1use std::sync::Arc;
5use std::time::Instant;
6
7use axum::body::Body;
8use axum::extract::{Request, State};
9use axum::http::HeaderMap;
10use axum::middleware::Next;
11use axum::response::{IntoResponse, Response};
12use axum::routing::{get, post};
13use axum::{Extension, Json, Router};
14use bytes::Bytes;
15use firstpass_core::features::{hour_bucket, token_bucket};
16use firstpass_core::hashchain::sha256_hex;
17use firstpass_core::{
18 Attempt, DeferredVerdict, Dialect, FEATURE_VERSION, Features, FinalOutcome, GENESIS_HASH, Mode,
19 ModelRef, PolicyRef, ProbeRegime, ProbeSignal, RequestInfo, RoutingMode, Score, ServedFrom,
20 TaskKind, Trace, Verdict,
21 features::{DifficultyHint, TrajectorySignals},
22};
23use serde::Deserialize;
24use serde_json::Value;
25use std::future::Future;
26use std::time::Duration;
27use tokio::sync::mpsc::error::TrySendError;
28use uuid::Uuid;
29
30use crate::config::ProxyConfig;
31use crate::error::ProxyError;
32use crate::gate::{GateHealthRegistry, aggregate_with_policy, resolve_gates};
33use crate::provider::{Auth, ChatMessage, ModelRequest, ModelResponse, ProviderRegistry};
34use crate::router::{EnforceCtx, EngineOutcome, route_enforce};
35use crate::store;
36use crate::tenant_auth::{TenantId, auth_middleware};
37use crate::upstream::{forward_anthropic, forward_anthropic_streaming};
38use firstpass_core::Route;
39use firstpass_core::trace::{RiskCertificate, ShadowSignal, ThresholdSource};
40
41#[derive(Clone)]
44pub struct AppState {
45 pub config: Arc<ProxyConfig>,
47 pub http: reqwest::Client,
49 pub providers: ProviderRegistry,
51 pub gate_health: Arc<GateHealthRegistry>,
53 pub shadow_ledger: Arc<crate::shadow::ShadowLedger>,
56 pub paired_ledger: Arc<crate::shadow::ShadowLedger>,
59 pub experiments: Arc<crate::experiment::ExperimentMonitor>,
62 pub guardrails: Arc<crate::guard::GuardrailRegistry>,
65 pub traces: store::TraceSender,
67 pub adaptive: Option<Arc<std::sync::Mutex<firstpass_core::conformal::AdaptiveConformal>>>,
71 pub eprocess: Option<Arc<std::sync::Mutex<firstpass_core::eprocess::EProcessRiskControl>>>,
76 pub bandit: Option<Arc<std::sync::Mutex<crate::bandit::StartRungBandit>>>,
81 pub promoter: Option<Arc<crate::affinity::SessionPromoter>>,
86 pub verified_cache: Option<Arc<dyn crate::verified_cache::CacheStore>>,
90 pub predictor: Option<Arc<std::sync::Mutex<firstpass_core::PassPredictor>>>,
96 pub prior: Option<Arc<crate::prior::PriorClient>>,
103 pub tenant_rate_limiter: Option<Arc<governor::DefaultKeyedRateLimiter<String>>>,
107 pub spill: Option<store::SpillHandle>,
111}
112
113#[must_use]
129fn routing_cfg_condense(state: &AppState) -> Option<&firstpass_core::config::CondenseConfig> {
130 state
131 .config
132 .routing
133 .as_ref()
134 .and_then(|r| r.escalation.condense.as_ref())
135}
136
137#[must_use]
142fn mode_now() -> jiff::Timestamp {
144 jiff::Timestamp::now()
145}
146
147fn cache_hit_trace(
155 ctx: &crate::router::EnforceCtx<'_>,
156 entry: &crate::verified_cache::Entry,
157 ts: jiff::Timestamp,
158) -> Trace {
159 Trace {
160 trace_id: uuid::Uuid::now_v7(),
161 prev_hash: firstpass_core::GENESIS_HASH.to_owned(),
162 tenant_id: ctx.tenant_id.clone(),
163 session_id: ctx.session_id.clone(),
164 ts,
165 mode: Mode::Enforce,
166 policy: PolicyRef {
167 id: format!("{}+cache", ctx.policy_id),
168 explore: false,
169 propensity: None,
170 mode_profile: None,
171 },
172 request: RequestInfo {
173 api: ctx.api.clone(),
174 prompt_hash: ctx.prompt_hash.clone(),
175 features: ctx.features.clone(),
176 },
177 attempts: Vec::new(),
178 final_: FinalOutcome {
179 served_rung: None,
180 served_from: ServedFrom::Cache,
181 total_cost_usd: 0.0,
185 gate_cost_usd: 0.0,
186 total_latency_ms: 0,
187 escalations: 0,
188 counterfactual_baseline_usd: entry.original_cost_usd,
189 savings_usd: entry.original_cost_usd,
190 cache_source: Some(entry.source),
191 },
192 deferred: Vec::new(),
193 predicted_pass: None,
194 decision_prior: None,
195 probe: None,
196 elastic: None,
197 risk: None,
198 rollout: None,
199 shadow: None,
200 route_ix: None,
201 experiment: None,
202 }
203}
204
205pub async fn build_verified_cache(
206 config: &ProxyConfig,
207) -> Result<Option<Arc<dyn crate::verified_cache::CacheStore>>, String> {
208 let Some(c) = config
209 .routing
210 .as_ref()
211 .and_then(|r| r.escalation.verified_cache.as_ref())
212 else {
213 return Ok(None);
214 };
215 if let Some(url) = c.redis_url.as_deref() {
216 #[cfg(feature = "redis-cache")]
217 {
218 let store = crate::verified_cache::RedisStore::connect(url, c.ttl_secs).await?;
219 tracing::info!("verified cache: shared via redis");
220 return Ok(Some(Arc::new(store)));
221 }
222 #[cfg(not(feature = "redis-cache"))]
223 {
224 let _ = url;
225 return Err(
226 "[escalation.verified_cache] redis_url is set, but this binary was built without \
227 the `redis-cache` feature. Rebuild with `--features redis-cache`, or remove \
228 redis_url to use the in-process cache."
229 .to_owned(),
230 );
231 }
232 }
233 Ok(Some(Arc::new(crate::verified_cache::VerifiedCache::new(
234 std::time::Duration::from_secs(c.ttl_secs),
235 c.max_entries,
236 ))))
237}
238
239pub async fn build_promoter_async(
240 config: &ProxyConfig,
241) -> Result<Option<Arc<crate::affinity::SessionPromoter>>, String> {
242 let Some(sp) = config
243 .routing
244 .as_ref()
245 .and_then(|r| r.escalation.session_promotion.as_ref())
246 else {
247 return Ok(None);
248 };
249 let promoter = crate::affinity::SessionPromoter::new(sp.clone()).map_err(|e| e.to_string())?;
250 if let Some(url) = sp.redis_url.as_deref() {
251 #[cfg(feature = "redis-cache")]
252 {
253 let ttl = sp.window_duration().map_err(|e| e.to_string())?.as_secs();
254 let store = crate::affinity::RedisPromotionStore::connect(url, ttl).await?;
255 tracing::info!("session promotion: shared via redis");
256 return Ok(Some(Arc::new(promoter.with_shared(Arc::new(store)))));
257 }
258 #[cfg(not(feature = "redis-cache"))]
259 {
260 let _ = url;
261 return Err(
262 "[escalation.session_promotion] redis_url is set, but this binary was \
263 built without the `redis-cache` feature. Rebuild with \
264 `--features redis-cache`, or remove redis_url to keep promotion \
265 in-process."
266 .to_owned(),
267 );
268 }
269 }
270 Ok(Some(Arc::new(promoter)))
271}
272
273#[must_use]
281pub fn build_paired_ledger(config: &ProxyConfig) -> crate::shadow::ShadowLedger {
282 let ledger = crate::shadow::ShadowLedger::new();
283 let any_paired = config.routing.as_ref().is_some_and(|r| {
284 r.routes
285 .iter()
286 .any(|route| route.challenger.as_ref().is_some_and(|c| c.is_paired()))
287 });
288 if !any_paired {
289 return ledger;
290 }
291 match crate::store::load_all_experiment_pairs(&config.db_path) {
292 Ok(rows) => {
293 let now = jiff::Timestamp::now();
294 for row in &rows {
295 ledger.seed(
296 &row.tenant,
297 row.route_ix,
298 row.challenger_cost_usd,
299 row.ts,
300 now,
301 );
302 }
303 }
304 Err(e) => tracing::warn!(%e, "paired ledger: could not read today's spend from the store"),
305 }
306 ledger
307}
308
309pub async fn build_experiment_log(
325 config: &ProxyConfig,
326) -> Result<Option<Arc<dyn crate::experiment::ExperimentLogBackend>>, String> {
327 use std::collections::HashSet;
328
329 let urls: HashSet<&str> = config
330 .routing
331 .iter()
332 .flat_map(|r| &r.routes)
333 .filter_map(|r| r.challenger.as_ref()?.redis_url.as_deref())
334 .collect();
335
336 let mut url_iter = urls.into_iter();
338 let Some(url) = url_iter.next() else {
339 return Ok(None); };
341 if let Some(other) = url_iter.next() {
342 let mut found = vec![url, other];
343 found.extend(url_iter);
344 found.sort_unstable();
345 return Err(format!(
346 "[route.challenger] multiple distinct redis_url values across routes are not \
347 supported — all experiment logs must share one Redis instance (keys are namespaced \
348 by tenant + route + fingerprint). URLs found: {found:?}"
349 ));
350 }
351
352 #[cfg(feature = "redis-cache")]
353 {
354 let log = crate::experiment::RedisExperimentLog::connect(url).await?;
356 tracing::info!("experiment log: shared via redis");
357 Ok(Some(
358 Arc::new(log) as Arc<dyn crate::experiment::ExperimentLogBackend>
359 ))
360 }
361
362 #[cfg(not(feature = "redis-cache"))]
363 {
364 let _ = url;
365 Err(
366 "[route.challenger] redis_url is set, but this binary was built without the \
367 `redis-cache` feature. Rebuild with `--features redis-cache`, or remove redis_url \
368 from all [route.challenger] sections to keep experiment state in-process."
369 .to_owned(),
370 )
371 }
372}
373
374#[must_use]
375pub fn build_tenant_rate_limiter(
376 config: &ProxyConfig,
377) -> Option<Arc<governor::DefaultKeyedRateLimiter<String>>> {
378 let per_sec = config.tenant_rate_per_sec?;
379 Some(Arc::new(governor::RateLimiter::keyed(
380 governor::Quota::per_second(per_sec),
381 )))
382}
383
384pub async fn tenant_rate_limit_middleware(
388 State(state): State<AppState>,
389 Extension(tenant): Extension<TenantId>,
390 req: Request,
391 next: Next,
392) -> Response {
393 if let Some(limiter) = &state.tenant_rate_limiter
394 && limiter.check_key(&tenant.0).is_err()
395 {
396 return ProxyError::RateLimited.into_response();
397 }
398 next.run(req).await
399}
400
401impl std::fmt::Debug for AppState {
402 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
403 f.debug_struct("AppState")
404 .field("config", &self.config)
405 .finish_non_exhaustive()
406 }
407}
408
409fn offer_trace(traces: &store::TraceSender, spill: Option<&store::SpillHandle>, trace: Trace) {
426 record_trace_metrics(&trace);
427 match traces.try_send(trace) {
428 Ok(()) => {}
429 Err(TrySendError::Full(t)) => {
430 if let Some(handle) = spill {
431 match store::append_to_spill(handle, &t) {
432 Ok(()) => {
433 metrics::counter!("firstpass_receipts_spilled_total").increment(1);
434 }
435 Err(e) => {
436 tracing::error!(%e, "durable mode: spill write failed; trace lost");
437 metrics::counter!("firstpass_traces_dropped_total").increment(1);
438 }
439 }
440 } else {
441 tracing::warn!("trace channel full; dropping trace (writer behind under load)");
442 metrics::counter!("firstpass_traces_dropped_total").increment(1);
443 }
444 }
445 Err(TrySendError::Closed(_)) => {
446 tracing::warn!("trace writer is gone; dropping trace");
447 }
448 }
449}
450
451fn risk_certificate(
457 eprocess: Option<(firstpass_core::eprocess::Certification, f64, f64)>,
458 aci: Option<(f64, f64)>,
459 fixed: Option<f64>,
460) -> Option<RiskCertificate> {
461 let cert = |source, threshold, alpha| RiskCertificate {
462 source,
463 threshold,
464 alpha,
465 delta: None,
466 e_value: None,
467 certified_at_round: None,
468 };
469 if let Some((c, alpha, delta)) = eprocess {
470 return Some(RiskCertificate {
471 delta: Some(delta),
472 e_value: Some(c.e_value),
473 certified_at_round: Some(c.certified_at_round),
474 ..cert(ThresholdSource::Eprocess, c.threshold, Some(alpha))
475 });
476 }
477 aci.map(|(t, alpha)| cert(ThresholdSource::Aci, t, Some(alpha)))
478 .or_else(|| fixed.map(|t| cert(ThresholdSource::Fixed, t, None)))
479}
480
481fn record_trace_metrics(trace: &Trace) {
482 if trace.mode == Mode::Enforce {
483 metrics::histogram!("firstpass_enforce_latency_ms")
484 .record(trace.final_.total_latency_ms as f64);
485 if trace.final_.escalations > 0 {
486 metrics::counter!("firstpass_escalations_total")
487 .increment(u64::from(trace.final_.escalations));
488 }
489 }
490 let served_from = match trace.final_.served_from {
491 ServedFrom::Attempt => "attempt",
492 ServedFrom::BestAttempt => "best_attempt",
493 ServedFrom::Error => "error",
494 ServedFrom::Cache => "cache",
498 };
499 metrics::counter!("firstpass_served_total", "served_from" => served_from).increment(1);
500 if trace.final_.served_from == ServedFrom::Error {
501 let provider = trace
506 .attempts
507 .last()
508 .map_or_else(|| "unknown".to_owned(), |a| a.provider.clone());
509 metrics::counter!("firstpass_upstream_failures_total", "provider" => provider).increment(1);
510 }
511
512 for a in &trace.attempts {
518 metrics::histogram!(
519 "firstpass_attempt_latency_ms",
520 "provider" => a.provider.clone(),
521 "rung" => a.rung.to_string()
522 )
523 .record(a.latency_ms as f64);
524 metrics::counter!(
525 "firstpass_attempt_total",
526 "provider" => a.provider.clone(),
527 "rung" => a.rung.to_string()
528 )
529 .increment(1);
530 metrics::gauge!(
531 "firstpass_attempt_cost_usd_total",
532 "provider" => a.provider.clone(),
533 "rung" => a.rung.to_string()
534 )
535 .increment(a.cost_usd);
536
537 for g in &a.gates {
541 let verdict = match g.verdict {
542 Verdict::Pass => "pass",
543 Verdict::Fail => "fail",
544 Verdict::Abstain => "abstain",
545 };
546 metrics::counter!(
547 "firstpass_gate_verdict_total",
548 "gate_id" => g.gate_id.clone(),
549 "verdict" => verdict
550 )
551 .increment(1);
552 metrics::histogram!("firstpass_gate_latency_ms", "gate_id" => g.gate_id.clone())
553 .record(g.ms as f64);
554 metrics::gauge!("firstpass_gate_cost_by_id_usd_total", "gate_id" => g.gate_id.clone())
555 .increment(g.cost_usd);
556 }
557 }
558 metrics::gauge!("firstpass_cost_usd_total").increment(trace.final_.total_cost_usd);
562 metrics::gauge!("firstpass_gate_cost_usd_total").increment(trace.final_.gate_cost_usd);
563 metrics::gauge!("firstpass_baseline_usd_total")
564 .increment(trace.final_.counterfactual_baseline_usd);
565 metrics::gauge!("firstpass_savings_usd_total").increment(trace.final_.savings_usd);
566 if let Some(rung) = trace.final_.served_rung {
568 let model = trace
569 .attempts
570 .iter()
571 .find(|a| a.rung == rung)
572 .map(|a| a.model.clone())
573 .unwrap_or_else(|| "unknown".to_owned());
574 metrics::counter!(
575 "firstpass_served_rung_total",
576 "rung" => rung.to_string(),
577 "model" => model
578 )
579 .increment(1);
580 }
581}
582
583const MAX_BODY_BYTES: usize = 16 * 1024 * 1024;
587
588pub(crate) fn u01(seed: u128) -> f64 {
599 let lo = splitmix64_finalise(seed as u64);
600 let hi = splitmix64_finalise((seed >> 64) as u64);
601 ((lo ^ hi) >> 11) as f64 * (1.0_f64 / (1u64 << 53) as f64)
603}
604
605fn splitmix64_finalise(mut z: u64) -> u64 {
606 z = z.wrapping_add(0x9E37_79B9_7F4A_7C15);
607 z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
608 z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
609 z ^ (z >> 31)
610}
611
612#[must_use]
619pub(crate) fn epsilon_propensity(chosen: u32, greedy: u32, epsilon: f64, k: usize) -> f64 {
620 let greedy_term = if chosen == greedy { 1.0 - epsilon } else { 0.0 };
621 greedy_term + epsilon / k as f64
622}
623
624pub fn app(state: AppState) -> Result<Router, ProxyError> {
631 crate::metrics::install()?;
632 let max_concurrency = state.config.max_concurrency;
633
634 let business = Router::new()
643 .route("/v1/messages", post(messages))
644 .route("/v1/chat/completions", post(chat_completions))
645 .route("/v1/responses", post(responses))
646 .route("/v1/feedback", post(feedback))
647 .route("/v1/capabilities", get(capabilities))
648 .route("/v1/models", get(models))
649 .route("/v1/receipts", get(receipts))
652 .route("/v1/experiments", get(experiments))
654 .layer(axum::middleware::from_fn_with_state(
655 state.clone(),
656 tenant_rate_limit_middleware,
657 ))
658 .layer(axum::middleware::from_fn_with_state(
659 state.clone(),
660 auth_middleware,
661 ));
662
663 Ok(Router::new()
664 .merge(business)
665 .route("/healthz", get(healthz))
666 .route("/metrics", get(crate::metrics::handler))
667 .route("/panel", get(panel))
670 .layer(axum::extract::DefaultBodyLimit::max(MAX_BODY_BYTES))
672 .layer(tower::limit::GlobalConcurrencyLimitLayer::new(
675 max_concurrency,
676 ))
677 .with_state(state))
678}
679
680async fn healthz() -> impl IntoResponse {
682 Json(serde_json::json!({
688 "status": "ok",
689 "service": "firstpass",
690 "version": env!("CARGO_PKG_VERSION"),
691 }))
692}
693
694const PANEL_HTML: &str = include_str!("panel.html");
698
699async fn receipts(
705 State(state): State<AppState>,
706 Extension(TenantId(tenant)): Extension<TenantId>,
707 axum::extract::Query(q): axum::extract::Query<ReceiptsQuery>,
708) -> Response {
709 let limit = q.limit.unwrap_or(20).clamp(1, 200);
711 let db = state.config.db_path.clone();
712 let (traces, summary) = match tokio::task::spawn_blocking(move || {
715 Ok::<_, crate::store::StoreError>((
716 crate::store::load_recent_tenant_traces(&db, &tenant, limit)?,
717 crate::store::tenant_receipts_summary(&db, &tenant)?,
718 ))
719 })
720 .await
721 {
722 Ok(Ok(t)) => t,
723 _ => (Vec::new(), crate::store::ReceiptsSummary::default()),
726 };
727 let summary = summary_json(&summary);
728 let items: Vec<serde_json::Value> = traces
729 .iter()
730 .map(|t| {
731 let f = &t.final_;
732 serde_json::json!({
733 "trace_id": t.trace_id,
734 "ts": t.ts,
735 "session_id": t.session_id,
736 "served_rung": f.served_rung,
737 "served_from": f.served_from,
738 "served_model": f.served_rung
743 .and_then(|r| t.attempts.iter().find(|a| a.rung == r))
744 .map(|a| a.model.clone()),
745 "attempts": t.attempts.iter().map(|a| serde_json::json!({
746 "rung": a.rung,
747 "model": a.model,
748 "verdict": a.verdict,
749 "cost_usd": a.cost_usd,
750 "latency_ms": a.latency_ms,
751 "gates": a.gates.iter().map(|g| serde_json::json!({
752 "gate_id": g.gate_id,
753 "verdict": g.verdict,
754 "score": g.score,
755 "reason": g.reason,
756 })).collect::<Vec<_>>(),
757 })).collect::<Vec<_>>(),
758 "escalations": f.escalations,
759 "total_latency_ms": f.total_latency_ms,
760 "total_cost_usd": f.total_cost_usd,
761 "baseline_usd": f.counterfactual_baseline_usd,
762 "savings_usd": f.savings_usd,
763 "prev_hash": t.prev_hash,
767 "hash": t.hash().ok(),
768 "experiment": t.experiment.as_ref().map(|e| serde_json::json!({
770 "arm": e.arm,
771 "percent": e.percent,
772 "bucket": e.bucket,
773 })),
774 "deferred": t.deferred.iter().map(|d| serde_json::json!({
775 "gate_id": d.gate_id,
776 "verdict": d.verdict,
777 "reporter": d.reporter,
778 })).collect::<Vec<_>>(),
779 })
780 })
781 .collect();
782 axum::Json(serde_json::json!({ "receipts": items, "summary": summary })).into_response()
783}
784
785fn summary_json(s: &crate::store::ReceiptsSummary) -> serde_json::Value {
788 let [pass, fail, abstain] = s.gates;
789 serde_json::json!({
790 "requests": s.requests,
791 "spent_usd": s.spent_usd,
792 "baseline_usd": s.baseline_usd,
793 "saved_usd": s.baseline_usd - s.spent_usd,
796 "first_pass": s.first_pass,
797 "escalated": s.escalated,
798 "gates": { "pass": pass, "fail": fail, "abstain": abstain },
799 "truncated": s.truncated,
800 })
801}
802
803async fn experiments(
812 State(state): State<AppState>,
813 Extension(TenantId(tenant)): Extension<TenantId>,
814) -> Response {
815 let Some(routing) = state.config.routing.as_ref() else {
816 return axum::Json(serde_json::json!({ "experiments": [], "configured": [] }))
817 .into_response();
818 };
819 let snaps: Vec<_> = state
820 .experiments
821 .reports()
822 .into_iter()
823 .filter(|s| s.tenant == tenant)
824 .collect();
825 let views = crate::experiment::views(snaps, routing);
826 let configured: Vec<serde_json::Value> = routing
827 .routes
828 .iter()
829 .enumerate()
830 .filter_map(|(ix, r)| r.challenger.as_ref().map(|c| (ix, r, c)))
831 .map(|(ix, r, c)| {
832 let status = state.experiments.status(&tenant, ix);
833 let paired = c.is_paired().then(|| {
836 serde_json::json!({
837 "sample_rate": c.sample_rate,
838 "max_usd_per_day": c.max_usd_per_day,
839 "spent_today_usd": state.paired_ledger.spent_today(
840 &tenant,
841 ix,
842 jiff::Timestamp::now(),
843 ),
844 })
845 });
846 serde_json::json!({
847 "route_ix": ix,
848 "incumbent_ladder": r.ladder,
849 "challenger_ladder": c.ladder,
850 "mode": if c.is_paired() { "paired" } else { "live" },
851 "paired": paired,
852 "percent": c.percent,
853 "margin": c.margin,
854 "alpha": c.alpha,
855 "unavailable": state.experiments.challenger_blocked(&tenant, ix)
856 && status != firstpass_core::ExperimentStatus::Halted,
857 })
858 })
859 .collect();
860 axum::Json(serde_json::json!({ "experiments": views, "configured": configured }))
861 .into_response()
862}
863
864#[derive(Debug, serde::Deserialize)]
866struct ReceiptsQuery {
867 limit: Option<usize>,
869}
870
871async fn panel() -> impl IntoResponse {
877 (
878 [
879 (
880 axum::http::header::CONTENT_TYPE,
881 "text/html; charset=utf-8".to_owned(),
882 ),
883 (
884 axum::http::header::CONTENT_SECURITY_POLICY,
885 PANEL_CSP.clone(),
886 ),
887 (
888 axum::http::header::X_CONTENT_TYPE_OPTIONS,
889 "nosniff".to_owned(),
890 ),
891 (
892 axum::http::header::REFERRER_POLICY,
893 "no-referrer".to_owned(),
894 ),
895 ],
896 PANEL_HTML,
897 )
898}
899
900static PANEL_CSP: std::sync::LazyLock<String> = std::sync::LazyLock::new(|| {
907 use sha2::Digest as _;
908 let script = PANEL_HTML
909 .split_once("<script>")
910 .and_then(|(_, rest)| rest.split_once("</script>"))
911 .map_or("", |(body, _)| body);
912 let digest = sha2::Sha256::digest(script.as_bytes());
913 format!(
914 "default-src 'none'; script-src 'sha256-{}'; style-src 'unsafe-inline'; \
915 connect-src 'self'; img-src 'self' data:; base-uri 'none'; form-action 'none'; \
916 frame-ancestors 'none'",
917 base64_std(&digest)
918 )
919});
920
921fn base64_std(bytes: &[u8]) -> String {
924 const ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
925 let mut out = String::with_capacity(bytes.len().div_ceil(3) * 4);
926 for chunk in bytes.chunks(3) {
927 let b = [
928 chunk[0],
929 chunk.get(1).copied().unwrap_or(0),
930 chunk.get(2).copied().unwrap_or(0),
931 ];
932 let n = (u32::from(b[0]) << 16) | (u32::from(b[1]) << 8) | u32::from(b[2]);
933 for (i, shift) in [18u32, 12, 6, 0].into_iter().enumerate() {
934 if i <= chunk.len() {
935 out.push(char::from(ALPHABET[((n >> shift) & 63) as usize]));
936 } else {
937 out.push('=');
938 }
939 }
940 }
941 out
942}
943
944async fn capabilities(State(state): State<AppState>) -> impl IntoResponse {
945 let (ladder, gates) = state
948 .config
949 .routing
950 .as_ref()
951 .and_then(|c| c.routes.iter().find(|r| r.mode == Mode::Enforce))
952 .map(|r| (r.ladder.clone(), r.gates.clone()))
953 .unwrap_or_default();
954 let routing_modes: Vec<serde_json::Value> = RoutingMode::ALL
955 .iter()
956 .map(|m| {
957 let p = m.preset();
958 serde_json::json!({
959 "name": m.as_str(),
960 "description": p.description,
961 "tradeoff": p.tradeoff,
962 })
963 })
964 .collect();
965 Json(serde_json::json!({
966 "service": "firstpass",
967 "version": env!("CARGO_PKG_VERSION"),
968 "feature_version": FEATURE_VERSION,
969 "modes": ["observe", "enforce"],
970 "routing_modes": routing_modes,
971 "wire_apis": ["anthropic.messages", "openai.chat_completions", "openai.responses"],
972 "ladder": ladder,
973 "gates": gates,
974 "feedback_api": "POST /v1/feedback",
975 "offboarding": "unset ANTHROPIC_BASE_URL (or OPENAI_BASE_URL for OpenAI clients)",
976 }))
977}
978
979async fn models(State(state): State<AppState>) -> impl IntoResponse {
991 Json(model_list(state.config.routing.as_ref()))
992}
993
994fn model_list(routing: Option<&firstpass_core::config::Config>) -> serde_json::Value {
997 let prices = routing.map(|c| c.price_table()).unwrap_or_default();
998 let mut seen = std::collections::HashSet::new();
1001 let data: Vec<serde_json::Value> = routing
1002 .map(|c| c.routes.as_slice())
1003 .unwrap_or_default()
1004 .iter()
1005 .flat_map(|r| r.ladder.iter())
1006 .filter(|id| seen.insert(id.as_str()))
1007 .map(|id| {
1008 let price = prices.get(id);
1009 serde_json::json!({
1010 "id": id,
1011 "object": "model",
1012 "created": 0,
1015 "owned_by": id.split_once('/').map_or("firstpass", |(p, _)| p),
1016 "firstpass": {
1017 "input_per_mtok": price.map(|p| p.input_per_mtok),
1018 "output_per_mtok": price.map(|p| p.output_per_mtok),
1019 },
1020 })
1021 })
1022 .collect();
1023 serde_json::json!({ "object": "list", "data": data })
1024}
1025
1026#[derive(Debug, Deserialize)]
1028struct FeedbackRequest {
1029 trace_id: String,
1031 gate_id: String,
1033 verdict: String,
1035 #[serde(default)]
1037 score: Option<f64>,
1038 reporter: String,
1040}
1041
1042async fn feedback(
1047 State(state): State<AppState>,
1048 Extension(TenantId(tenant)): Extension<TenantId>,
1049 body: Bytes,
1050) -> Response {
1051 let req: FeedbackRequest = match serde_json::from_slice(&body) {
1052 Ok(r) => r,
1053 Err(e) => {
1054 return ProxyError::BadRequest(format!("invalid feedback body: {e}")).into_response();
1055 }
1056 };
1057 let verdict = match req.verdict.as_str() {
1058 "pass" => Verdict::Pass,
1059 "fail" => Verdict::Fail,
1060 "abstain" => Verdict::Abstain,
1061 other => {
1062 return ProxyError::BadRequest(format!("unknown verdict {other:?}")).into_response();
1063 }
1064 };
1065 let score = match req.score {
1066 Some(s) => match Score::new(s) {
1067 Ok(sc) => Some(sc),
1068 Err(_) => {
1069 return ProxyError::BadRequest(format!("score {s} out of range [0,1]"))
1070 .into_response();
1071 }
1072 },
1073 None => None,
1074 };
1075
1076 let db = state.config.db_path.clone();
1077
1078 let (db_check, tenant_check, tid_check) = (db.clone(), tenant.clone(), req.trace_id.clone());
1083 match tokio::task::spawn_blocking(move || {
1084 store::trace_exists(&db_check, &tenant_check, &tid_check)
1085 })
1086 .await
1087 {
1088 Ok(Ok(true)) => {}
1089 Ok(Ok(false)) => {
1090 return ProxyError::NotFound(format!("unknown trace_id {:?}", req.trace_id))
1091 .into_response();
1092 }
1093 Ok(Err(e)) => {
1094 tracing::error!(%e, "feedback: trace_exists check failed");
1095 return ProxyError::Internal(e.to_string()).into_response();
1096 }
1097 Err(e) => {
1098 tracing::error!(%e, "feedback: trace_exists task panicked");
1099 return ProxyError::Internal(e.to_string()).into_response();
1100 }
1101 }
1102
1103 if verdict == Verdict::Fail
1108 && let Some(cache) = state.verified_cache.as_ref()
1109 {
1110 let reported = req.trace_id.parse::<uuid::Uuid>().ok();
1115 let origin = {
1116 let (d, t, i) = (db.clone(), tenant.clone(), req.trace_id.clone());
1117 tokio::task::spawn_blocking(move || store::trace_cache_source(&d, &t, &i))
1118 .await
1119 .ok()
1120 .and_then(Result::ok)
1121 .flatten()
1122 .flatten()
1123 };
1124 let mut dropped = 0usize;
1125 for id in reported.into_iter().chain(origin) {
1126 dropped += cache.retract(id).await;
1127 }
1128 if dropped > 0 {
1129 tracing::info!(
1130 trace = %req.trace_id,
1131 ?origin,
1132 dropped,
1133 "verified cache: retracted a disproven answer"
1134 );
1135 metrics::counter!("firstpass_cache_retractions_total").increment(dropped as u64);
1136 }
1137 }
1138
1139 let feedback_signal = match verdict {
1141 Verdict::Pass => Some(true),
1142 Verdict::Fail => Some(false),
1143 Verdict::Abstain => None,
1144 };
1145 let dv = DeferredVerdict {
1146 gate_id: req.gate_id,
1147 verdict,
1148 score,
1149 reported_at: jiff::Timestamp::now(),
1150 reporter: req.reporter,
1151 };
1152 let trace_id = req.trace_id.clone();
1153 match tokio::task::spawn_blocking(move || store::append_deferred(&db, &req.trace_id, &dv)).await
1154 {
1155 Ok(Ok(())) => {
1156 if let (Some(a), Some(correct)) = (state.adaptive.as_ref(), feedback_signal)
1158 && let Ok(mut g) = a.lock()
1159 {
1160 g.observe_served(correct);
1161 metrics::gauge!("firstpass_serve_threshold").set(g.threshold());
1162 metrics::gauge!("firstpass_realized_served_failure")
1163 .set(g.realized_served_failure());
1164 }
1165
1166 if let (Some(e), Some(correct)) = (state.eprocess.as_ref(), feedback_signal) {
1189 let db_s = state.config.db_path.clone();
1190 let (tenant_s, tid_s) = (tenant.clone(), trace_id.clone());
1191 let served = tokio::task::spawn_blocking(move || {
1192 store::load_trace_view(&db_s, &tenant_s, &tid_s)
1193 .ok()
1194 .flatten()
1195 .and_then(|t| {
1196 let rung = t.final_.served_rung?;
1197 let a = t.attempts.iter().find(|a| a.rung == rung)?;
1198 Some(crate::calibrate::gate_score(&a.gates, a.verdict))
1199 })
1200 })
1201 .await
1202 .ok()
1203 .flatten();
1204 if let (Some(sc), Ok(mut g)) = (served, e.lock()) {
1207 g.observe_served(sc, correct);
1208 metrics::counter!("firstpass_eprocess_rounds_total").increment(1);
1209 }
1210 }
1211
1212 let attributed_route = tokio::task::spawn_blocking({
1217 let (db, ten, tid) = (
1218 state.config.db_path.clone(),
1219 tenant.clone(),
1220 trace_id.clone(),
1221 );
1222 move || store::trace_route(&db, &ten, &tid)
1223 })
1224 .await
1225 .ok()
1226 .and_then(Result::ok)
1227 .flatten()
1228 .flatten()
1229 .unwrap_or(0) as usize;
1230
1231 if let (Some(cfg), Some(correct)) = (
1236 state.config.routing.as_ref().and_then(|r| r.guardrail),
1237 feedback_signal,
1238 ) {
1239 let cooldown = state
1240 .config
1241 .routing
1242 .as_ref()
1243 .map_or(3_600, |r| r.guardrail_cooldown_secs);
1244 let reaction = state.guardrails.record(
1245 &tenant,
1246 attributed_route,
1247 &cfg,
1248 correct,
1249 jiff::Timestamp::now().as_second(),
1250 cooldown,
1251 );
1252 match &reaction {
1253 crate::guard::Reaction::Demoted(v) => {
1254 metrics::counter!("firstpass_guardrail_demotions_total").increment(1);
1255 tracing::error!(
1256 tenant = %tenant,
1257 n = v.n,
1258 rate = v.rate,
1259 bound = v.bound,
1260 alpha = cfg.alpha,
1261 "GUARDRAIL: served-failure bound exceeded target — route demoted to observe; traffic now serves as it would without Firstpass"
1262 );
1263 }
1264 crate::guard::Reaction::Alarmed(v) => {
1265 metrics::counter!("firstpass_guardrail_breaches_total").increment(1);
1266 tracing::error!(
1267 tenant = %tenant,
1268 n = v.n,
1269 rate = v.rate,
1270 bound = v.bound,
1271 alpha = cfg.alpha,
1272 "GUARDRAIL: served-failure bound exceeded target (alarm only — routing unchanged)"
1273 );
1274 }
1275 crate::guard::Reaction::None => {}
1276 }
1277 }
1278 if state.experiments.is_active() && verdict != Verdict::Abstain {
1285 let (db_x, ten_x, tid_x) = (
1286 state.config.db_path.clone(),
1287 tenant.clone(),
1288 trace_id.clone(),
1289 );
1290 match tokio::task::spawn_blocking(move || {
1291 store::trace_experiment(&db_x, &ten_x, &tid_x)
1292 })
1293 .await
1294 {
1295 Ok(Ok(Some((session, route_ix, rec, cost_usd)))) => {
1296 state
1297 .experiments
1298 .record_outcome(
1299 &tenant,
1300 route_ix,
1301 &rec.fingerprint,
1302 rec.arm,
1303 &session,
1304 verdict,
1305 cost_usd,
1306 )
1307 .await;
1308 }
1309 Ok(Ok(None)) => {}
1310 Ok(Err(e)) => {
1311 tracing::warn!(%e, "feedback: experiment attribution lookup failed");
1312 }
1313 Err(e) => {
1314 tracing::warn!(%e, "feedback: experiment attribution task failed");
1315 }
1316 }
1317 }
1318 (
1319 axum::http::StatusCode::ACCEPTED,
1320 Json(serde_json::json!({ "status": "recorded", "trace_id": trace_id })),
1321 )
1322 .into_response()
1323 }
1324 Ok(Err(e)) => {
1325 tracing::error!(%e, "feedback: append_deferred failed");
1326 ProxyError::Internal(e.to_string()).into_response()
1327 }
1328 Err(e) => {
1329 tracing::error!(%e, "feedback: append_deferred task panicked");
1330 ProxyError::Internal(e.to_string()).into_response()
1331 }
1332 }
1333}
1334
1335const SESSION_HEADER: &str = "x-firstpass-session";
1338
1339const AGENT_HEADER: &str = "x-firstpass-agent";
1341const SUBAGENT_HEADER: &str = "x-firstpass-subagent";
1343#[expect(
1349 clippy::too_many_arguments,
1350 reason = "mirrors the enforce context it evaluates; a wrapper struct would only move the list"
1351)]
1352fn spawn_shadow(
1353 state: &AppState,
1354 route: &firstpass_core::Route,
1355 route_ix: usize,
1356 body: Bytes,
1357 auth: Auth,
1358 features: Features,
1359 tenant: String,
1360 session_id: String,
1361 api: &str,
1362) {
1363 let Some(shadow) = route.shadow else {
1364 return;
1365 };
1366 if !shadow.sampled(&state.config.prompt_salt, &session_id) {
1369 return;
1370 }
1371 let (state, route, api) = (state.clone(), route.clone(), api.to_owned());
1372 tokio::spawn(async move {
1373 let signal = evaluate_shadow(
1374 &state, &route, shadow, route_ix, &body, auth, &features, tenant, session_id, &api, 0.0,
1375 )
1376 .await;
1377 tracing::debug!(
1379 would_pass = signal.would_pass,
1380 projected_usd = signal.projected_cost_usd,
1381 skipped = ?signal.skipped,
1382 "shadow evaluation complete"
1383 );
1384 });
1385}
1386
1387#[expect(
1394 clippy::too_many_arguments,
1395 reason = "mirrors the enforce context; grouping them into a struct would only move the list"
1396)]
1397async fn evaluate_shadow(
1398 state: &AppState,
1399 route: &firstpass_core::Route,
1400 shadow: firstpass_core::rollout::Shadow,
1401 route_ix: usize,
1402 body: &Bytes,
1403 auth: Auth,
1404 features: &Features,
1405 tenant: String,
1406 session_id: String,
1407 api: &str,
1408 actual_cost_usd: f64,
1409) -> ShadowSignal {
1410 let skipped = |why: &str| ShadowSignal {
1411 would_serve_rung: None,
1412 would_pass: false,
1413 projected_cost_usd: 0.0,
1414 actual_cost_usd,
1415 skipped: Some(why.to_owned()),
1416 };
1417
1418 let now = jiff::Timestamp::now();
1419 if !state
1420 .shadow_ledger
1421 .may_spend(&tenant, route_ix, shadow.max_usd_per_day, now)
1422 {
1423 return skipped("budget_exhausted");
1426 }
1427
1428 let Some(base_request) = parse_model_request(body) else {
1429 return skipped("unparseable_request");
1430 };
1431 let gate_defs = state
1432 .config
1433 .routing
1434 .as_ref()
1435 .map_or(&[][..], |cfg| &cfg.gate_defs);
1436 let gates = resolve_gates(
1437 &route.gates,
1438 gate_defs,
1439 &state.providers,
1440 &auth,
1441 &state.config.prices,
1442 &state.http,
1443 );
1444 let (budget, max_rungs) = state
1445 .config
1446 .routing
1447 .as_ref()
1448 .map_or((None, u32::MAX), |cfg| {
1449 (
1450 cfg.budget.per_request_usd,
1451 cfg.escalation.max_rungs_per_request,
1452 )
1453 });
1454
1455 let mut base_request = base_request;
1456 base_request.cache_prefix = state
1460 .config
1461 .routing
1462 .as_ref()
1463 .is_some_and(|r| r.escalation.prompt_cache);
1464 let ctx = EnforceCtx {
1465 condense: routing_cfg_condense(state),
1466 ladder: &route.ladder,
1467 gates: &gates,
1468 health: &state.gate_health,
1469 base_request: &base_request,
1470 providers: &state.providers,
1471 auth: &auth,
1472 prices: &state.config.prices,
1473 budget_per_request_usd: budget,
1474 max_rungs,
1475 speculation: 0,
1478 serve_threshold: None,
1479 elastic: None,
1480 features: features.clone(),
1481 start_rung: 0,
1482 tenant_id: tenant.clone(),
1483 session_id,
1484 prompt_hash: prompt_hash(&state.config.prompt_salt, body),
1485 api: api.to_owned(),
1486 policy_id: "shadow".to_owned(),
1487 };
1488
1489 let (outcome, trace) = route_enforce(ctx).await;
1490 let spent = trace.final_.total_cost_usd;
1491 state.shadow_ledger.debit(&tenant, route_ix, spent, now);
1492
1493 let would_pass = matches!(outcome, EngineOutcome::Served(_))
1497 && trace.final_.served_from == firstpass_core::ServedFrom::Attempt;
1498 let would_serve_rung = if would_pass {
1499 trace.final_.served_rung
1500 } else {
1501 None
1502 };
1503 ShadowSignal {
1504 would_serve_rung,
1505 would_pass,
1506 projected_cost_usd: spent,
1507 actual_cost_usd,
1508 skipped: None,
1509 }
1510}
1511
1512fn pair_outcome(served_from: firstpass_core::ServedFrom) -> Option<f64> {
1517 match served_from {
1518 firstpass_core::ServedFrom::Attempt => Some(1.0),
1519 firstpass_core::ServedFrom::BestAttempt => Some(0.0),
1520 firstpass_core::ServedFrom::Cache | firstpass_core::ServedFrom::Error => None,
1521 }
1522}
1523
1524#[allow(clippy::too_many_arguments)]
1534fn spawn_paired_challenger(
1535 state: &AppState,
1536 route: &firstpass_core::Route,
1537 challenger: &firstpass_core::Challenger,
1538 fingerprint: String,
1539 route_ix: usize,
1540 body: bytes::Bytes,
1541 auth: Auth,
1542 features: firstpass_core::Features,
1543 tenant: String,
1544 session_id: String,
1545 api: &str,
1546 served_from: firstpass_core::ServedFrom,
1547 incumbent_cost_usd: f64,
1548) {
1549 let Some(x_inc) = pair_outcome(served_from) else {
1550 return;
1551 };
1552 if state.experiments.challenger_blocked(&tenant, route_ix) {
1555 return;
1556 }
1557
1558 let max_usd_per_day = match challenger.max_usd_per_day {
1559 Some(v) => v,
1560 None => {
1561 tracing::warn!(
1562 tenant,
1563 route = route_ix,
1564 "paired challenger is missing max_usd_per_day; skipping evaluation"
1565 );
1566 return;
1567 }
1568 };
1569
1570 let state = state.clone();
1571 let route = route.clone();
1572 let challenger = challenger.clone();
1573 let api = api.to_owned();
1574
1575 tokio::spawn(async move {
1576 let now = jiff::Timestamp::now();
1577 if !state
1578 .paired_ledger
1579 .may_spend(&tenant, route_ix, max_usd_per_day, now)
1580 {
1581 tracing::debug!(
1582 tenant,
1583 route = route_ix,
1584 "paired challenger: daily budget exhausted, skipping"
1585 );
1586 return;
1587 }
1588
1589 let Some(base_request) = parse_model_request(&body) else {
1590 return;
1591 };
1592 let gate_defs = state
1593 .config
1594 .routing
1595 .as_ref()
1596 .map_or(&[][..], |cfg| &cfg.gate_defs);
1597 let gates = resolve_gates(
1598 &route.gates,
1599 gate_defs,
1600 &state.providers,
1601 &auth,
1602 &state.config.prices,
1603 &state.http,
1604 );
1605 let (budget, max_rungs) = state
1606 .config
1607 .routing
1608 .as_ref()
1609 .map_or((None, u32::MAX), |cfg| {
1610 (
1611 cfg.budget.per_request_usd,
1612 cfg.escalation.max_rungs_per_request,
1613 )
1614 });
1615 let mut base_request = base_request;
1616 base_request.cache_prefix = state
1617 .config
1618 .routing
1619 .as_ref()
1620 .is_some_and(|r| r.escalation.prompt_cache);
1621
1622 let ctx = EnforceCtx {
1623 condense: routing_cfg_condense(&state),
1624 ladder: &challenger.ladder,
1625 gates: &gates,
1626 health: &state.gate_health,
1627 base_request: &base_request,
1628 providers: &state.providers,
1629 auth: &auth,
1630 prices: &state.config.prices,
1631 budget_per_request_usd: budget,
1632 max_rungs,
1633 speculation: 0,
1634 serve_threshold: None,
1635 elastic: None,
1636 features,
1637 start_rung: 0,
1638 tenant_id: tenant.clone(),
1639 session_id: session_id.clone(),
1640 prompt_hash: prompt_hash(&state.config.prompt_salt, &body),
1641 api: api.clone(),
1642 policy_id: "paired-challenger@v0".to_owned(),
1643 };
1644
1645 let (_, ch_trace) = route_enforce(ctx).await;
1646 let challenger_cost = ch_trace.final_.total_cost_usd;
1647 state
1648 .paired_ledger
1649 .debit(&tenant, route_ix, challenger_cost, now);
1650
1651 state.experiments.observe_health(
1652 &tenant,
1653 route_ix,
1654 &fingerprint,
1655 firstpass_core::Arm::Challenger,
1656 ch_trace.final_.served_from == firstpass_core::ServedFrom::Error,
1657 crate::experiment::truncated_attempts(&ch_trace),
1658 );
1659 let x_ch = match ch_trace.final_.served_from {
1664 firstpass_core::ServedFrom::Attempt => 1.0,
1665 firstpass_core::ServedFrom::BestAttempt | firstpass_core::ServedFrom::Error => 0.0,
1666 firstpass_core::ServedFrom::Cache => return,
1669 };
1670
1671 let pair_row = crate::store::ExperimentPairRow {
1673 tenant: tenant.clone(),
1674 route_ix,
1675 fingerprint: fingerprint.clone(),
1676 trace_id: ch_trace.trace_id,
1677 x_inc,
1678 x_ch,
1679 challenger_cost_usd: challenger_cost,
1680 ts: now,
1681 };
1682 if let Err(e) = crate::store::append_experiment_pair(&state.config.db_path, &pair_row) {
1683 tracing::warn!(%e, "paired challenger: failed to persist pair to store");
1684 }
1685
1686 state
1689 .experiments
1690 .record_pair(
1691 &tenant,
1692 route_ix,
1693 &fingerprint,
1694 x_inc,
1695 x_ch,
1696 incumbent_cost_usd,
1697 challenger_cost,
1698 )
1699 .await;
1700
1701 tracing::debug!(
1702 tenant,
1703 route = route_ix,
1704 x_inc,
1705 x_ch,
1706 challenger_cost_usd = challenger_cost,
1707 "paired challenger evaluation complete"
1708 );
1709 });
1710}
1711
1712const MODE_PROFILE_HEADER: &str = "x-firstpass-mode";
1715
1716fn resolve_mode(headers: &HeaderMap, route: &Route, config: &ProxyConfig) -> RoutingMode {
1725 if let Some(val) = header_str(headers, MODE_PROFILE_HEADER) {
1727 match val.trim().to_ascii_lowercase().as_str() {
1728 "observe" => return RoutingMode::Observe,
1729 "cost" => return RoutingMode::Cost,
1730 "balanced" => return RoutingMode::Balanced,
1731 "quality" => return RoutingMode::Quality,
1732 "latency" => return RoutingMode::Latency,
1733 "max" => return RoutingMode::Max,
1734 other => {
1735 tracing::warn!(
1736 value = other,
1737 "unknown x-firstpass-mode value; ignoring \
1738 (valid: observe|cost|balanced|quality|latency|max)"
1739 );
1740 }
1741 }
1742 }
1743 if let Some(m) = route.routing_mode {
1745 return m;
1746 }
1747 config.default_routing_mode
1749}
1750
1751async fn messages(
1756 State(state): State<AppState>,
1757 Extension(TenantId(tenant)): Extension<TenantId>,
1758 headers: HeaderMap,
1759 body: Bytes,
1760) -> Response {
1761 let session_header = header_str(&headers, SESSION_HEADER);
1762
1763 if let Some(routing) = state.config.routing.as_ref() {
1766 let features = extract_features(&headers, &body);
1767 if let Some(route) = routing
1768 .route_for(&features)
1769 .filter(|r| r.mode == Mode::Enforce && !r.ladder.is_empty())
1770 {
1771 let route_ix = routing
1774 .routes
1775 .iter()
1776 .position(|r| std::ptr::eq(r, route))
1777 .unwrap_or(0);
1778 let route = route.clone();
1781 let routing_mode = resolve_mode(&headers, &route, &state.config);
1783 if routing_mode == RoutingMode::Observe {
1785 return observe_passthrough(state, headers, body, session_header, tenant).await;
1786 }
1787 if state
1791 .guardrails
1792 .is_demoted(&tenant, route_ix, jiff::Timestamp::now().as_second())
1793 {
1794 return observe_passthrough(state, headers, body, session_header, tenant).await;
1795 }
1796 if let Some(rollout) = route.rollout.as_ref() {
1801 let key_value = match rollout.key {
1802 firstpass_core::RolloutKey::Session => session_header
1803 .clone()
1804 .unwrap_or_else(|| firstpass_core::rollout::request_identity(&body)),
1805 firstpass_core::RolloutKey::Request => {
1811 firstpass_core::rollout::request_identity(&body)
1812 }
1813 firstpass_core::RolloutKey::Tenant => tenant.to_string(),
1814 };
1815 let decision =
1816 firstpass_core::rollout::decide(&state.config.prompt_salt, rollout, &key_value);
1817 if !decision.enforced {
1818 let shadow_ctx = route.shadow.map(|_| {
1820 (
1821 body.clone(),
1822 Auth::from_headers(&headers),
1823 features.clone(),
1824 tenant.clone(),
1825 session_header
1826 .clone()
1827 .unwrap_or_else(|| Uuid::now_v7().to_string()),
1828 )
1829 });
1830 let resp =
1831 observe_passthrough(state.clone(), headers, body, session_header, tenant)
1832 .await;
1833 if let Some((b, auth, feats, ten, sess)) = shadow_ctx {
1836 spawn_shadow(
1837 &state,
1838 &route,
1839 route_ix,
1840 b,
1841 auth,
1842 feats,
1843 ten,
1844 sess,
1845 "anthropic",
1846 );
1847 }
1848 return resp;
1849 }
1850 }
1851 if enforce_can_handle_route(
1852 &features,
1853 &body,
1854 routing.escalation.enforce_structured,
1855 &route,
1856 &state.providers,
1857 Dialect::Anthropic,
1858 ) {
1859 return handle_enforce(
1860 &state,
1861 &headers,
1862 &body,
1863 features,
1864 &route,
1865 route_ix,
1866 session_header,
1867 tenant,
1868 routing_mode,
1869 )
1870 .await;
1871 }
1872 tracing::info!(
1876 "enforce route matched but structured request can't be routed faithfully (flag/ladder); serving via observe passthrough"
1877 );
1878 }
1879 }
1880 observe_passthrough(state, headers, body, session_header, tenant).await
1881}
1882
1883fn enforce_can_handle_route(
1900 features: &Features,
1901 body: &[u8],
1902 enforce_structured: bool,
1903 route: &Route,
1904 providers: &crate::provider::ProviderRegistry,
1905 inbound: Dialect,
1906) -> bool {
1907 enforce_can_handle(
1908 features,
1909 body,
1910 enforce_structured,
1911 &route.ladder,
1912 providers,
1913 inbound,
1914 ) && route.challenger.as_ref().is_none_or(|c| {
1915 enforce_can_handle(
1916 features,
1917 body,
1918 enforce_structured,
1919 &c.ladder,
1920 providers,
1921 inbound,
1922 )
1923 })
1924}
1925
1926fn enforce_can_handle(
1927 features: &Features,
1928 body: &[u8],
1929 enforce_structured: bool,
1930 ladder: &[String],
1931 providers: &crate::provider::ProviderRegistry,
1932 inbound: Dialect,
1933) -> bool {
1934 let structured = features.tool_count > 0
1935 || features.has_images
1936 || match inbound {
1937 Dialect::Anthropic => messages_have_tool_blocks(body),
1938 Dialect::Openai => openai_messages_have_tool_calls(body),
1939 Dialect::Gemini => false,
1940 };
1941 if !structured {
1942 return true;
1943 }
1944 if !enforce_structured {
1945 return false;
1946 }
1947 let all_verbatim = ladder.iter().all(|rung| {
1949 let provider_id = rung.split('/').next().unwrap_or_default();
1950 providers
1951 .get(provider_id)
1952 .is_some_and(|p| p.carries_structured_verbatim(inbound))
1953 });
1954 if all_verbatim {
1955 return true;
1956 }
1957 if inbound == Dialect::Openai && !openai_has_http_images(body) {
1961 let all_anthropic = ladder.iter().all(|rung| {
1962 let pid = rung.split('/').next().unwrap_or_default();
1963 providers
1964 .get(pid)
1965 .is_some_and(|p| p.carries_structured_verbatim(Dialect::Anthropic))
1966 });
1967 if all_anthropic {
1968 return true;
1969 }
1970 }
1971 false
1972}
1973
1974fn messages_have_tool_blocks(body: &[u8]) -> bool {
1977 serde_json::from_slice::<Value>(body)
1978 .ok()
1979 .and_then(|json| {
1980 json.get("messages")
1981 .and_then(Value::as_array)
1982 .map(|messages| messages.iter().any(message_has_tool_block))
1983 })
1984 .unwrap_or(false)
1985}
1986
1987fn message_has_tool_block(message: &Value) -> bool {
1989 message
1990 .get("content")
1991 .and_then(Value::as_array)
1992 .is_some_and(|blocks| {
1993 blocks.iter().any(|block| {
1994 matches!(
1995 block.get("type").and_then(Value::as_str),
1996 Some("tool_use" | "tool_result")
1997 )
1998 })
1999 })
2000}
2001
2002fn is_stream_request(body: &[u8]) -> bool {
2004 serde_json::from_slice::<Value>(body)
2005 .ok()
2006 .and_then(|json| json.get("stream").and_then(Value::as_bool))
2007 .unwrap_or(false)
2008}
2009
2010fn header_str(headers: &HeaderMap, name: &str) -> Option<String> {
2012 headers
2013 .get(name)
2014 .and_then(|v| v.to_str().ok())
2015 .map(str::to_owned)
2016}
2017
2018fn trajectory_signals(body: &[u8]) -> TrajectorySignals {
2032 let Ok(json) = serde_json::from_slice::<Value>(body) else {
2033 return TrajectorySignals::default();
2034 };
2035 let Some(messages) = json.get("messages").and_then(Value::as_array) else {
2036 return TrajectorySignals::default();
2037 };
2038
2039 const WINDOW: usize = 12;
2044 let recent = &messages[messages.len().saturating_sub(WINDOW)..];
2045
2046 let assistant_turns = u32::try_from(
2055 messages
2056 .iter()
2057 .filter(|m| m.get("role").and_then(Value::as_str) == Some("assistant"))
2058 .count(),
2059 )
2060 .unwrap_or(u32::MAX);
2061 let mut s = TrajectorySignals {
2062 assistant_turns,
2063 ..Default::default()
2064 };
2065 let mut seen: Vec<u64> = Vec::new();
2069
2070 for m in recent {
2071 match m.get("role").and_then(Value::as_str) {
2072 Some("assistant") => {}
2074 Some("tool") => {
2077 s.tool_results += 1;
2078 if openai_tool_content_looks_like_error(m) {
2079 s.tool_errors += 1;
2080 }
2081 }
2082 _ => {}
2083 }
2084
2085 if let Some(blocks) = m.get("content").and_then(Value::as_array) {
2088 for b in blocks {
2089 match b.get("type").and_then(Value::as_str) {
2090 Some("tool_result") => {
2091 s.tool_results += 1;
2092 if b.get("is_error").and_then(Value::as_bool).unwrap_or(false) {
2093 s.tool_errors += 1;
2094 }
2095 }
2096 Some("tool_use") => {
2097 let fp = tool_call_fingerprint(
2098 b.get("name").and_then(Value::as_str).unwrap_or_default(),
2099 b.get("input"),
2100 );
2101 if seen.contains(&fp) {
2102 s.repeated_tool_calls += 1;
2103 } else {
2104 seen.push(fp);
2105 }
2106 }
2107 _ => {}
2108 }
2109 }
2110 }
2111
2112 if let Some(calls) = m.get("tool_calls").and_then(Value::as_array) {
2114 for c in calls {
2115 let f = c.get("function");
2116 let fp = tool_call_fingerprint(
2117 f.and_then(|f| f.get("name"))
2118 .and_then(Value::as_str)
2119 .unwrap_or_default(),
2120 f.and_then(|f| f.get("arguments")),
2121 );
2122 if seen.contains(&fp) {
2123 s.repeated_tool_calls += 1;
2124 } else {
2125 seen.push(fp);
2126 }
2127 }
2128 }
2129 }
2130 s
2131}
2132
2133fn tool_call_fingerprint(name: &str, args: Option<&Value>) -> u64 {
2138 use std::hash::{Hash, Hasher};
2139 let mut h = std::collections::hash_map::DefaultHasher::new();
2140 name.hash(&mut h);
2141 if let Some(a) = args {
2146 a.to_string().hash(&mut h);
2147 }
2148 h.finish()
2149}
2150
2151fn openai_tool_content_looks_like_error(message: &Value) -> bool {
2160 let content = message.get("content");
2164 let text: &str = match content.and_then(Value::as_str) {
2169 Some(t) => t,
2170 None => {
2171 let Some(t) = content.and_then(Value::as_array).and_then(|parts| {
2172 parts
2173 .iter()
2174 .find_map(|p| p.get("text").and_then(Value::as_str))
2175 }) else {
2176 return false;
2177 };
2178 t
2179 }
2180 };
2181 let lowered: String = text
2197 .chars()
2198 .take(128)
2199 .collect::<String>()
2200 .trim_start()
2201 .chars()
2202 .take(64)
2203 .collect::<String>()
2204 .to_ascii_lowercase();
2205 lowered.starts_with("error")
2206 || lowered.starts_with("exception")
2207 || lowered.starts_with("traceback")
2208 || lowered.starts_with("failed")
2209 || lowered.starts_with("fatal")
2210}
2211
2212fn extract_features(headers: &HeaderMap, body: &[u8]) -> Features {
2213 let (_model, tool_count, has_images) = request_features(body);
2214 let mut f = Features::new(TaskKind::Other);
2215 f.agent = header_str(headers, AGENT_HEADER);
2216 f.subagent = header_str(headers, SUBAGENT_HEADER);
2217 f.tool_count = tool_count;
2218 f.has_images = has_images;
2219 f.prompt_token_bucket = token_bucket(body.len() as u64);
2222 f.hour_bucket = hour_bucket(jiff::Timestamp::now());
2223 f.difficulty_hint = DifficultyHint::score(trajectory_signals(body)).as_u8();
2227 f
2228}
2229
2230#[allow(clippy::too_many_arguments)]
2233async fn handle_enforce(
2234 state: &AppState,
2235 headers: &HeaderMap,
2236 body: &Bytes,
2237 features: Features,
2238 route: &Route,
2239 route_ix: usize,
2242 session_header: Option<String>,
2243 tenant: String,
2244 routing_mode: RoutingMode,
2245) -> Response {
2246 if is_stream_request(body) {
2254 let (tx, rx) = tokio::sync::oneshot::channel::<Result<Value, ProxyError>>();
2255 let (state_c, headers_c, body_c, route_c) =
2256 (state.clone(), headers.clone(), body.clone(), route.clone());
2257 tokio::spawn(async move {
2258 let out = enforce_pipeline(
2259 &state_c,
2260 &headers_c,
2261 &body_c,
2262 features,
2263 &route_c,
2264 route_ix,
2265 session_header,
2266 tenant,
2267 routing_mode,
2268 )
2269 .await;
2270 let _ = tx.send(out);
2271 });
2272 return sse_keepalive_response(rx, anthropic_sse_from_message);
2273 }
2274 match enforce_pipeline(
2275 state,
2276 headers,
2277 body,
2278 features,
2279 route,
2280 route_ix,
2281 session_header,
2282 tenant,
2283 routing_mode,
2284 )
2285 .await
2286 {
2287 Ok(message) => (axum::http::StatusCode::OK, Json(message)).into_response(),
2288 Err(e) => e.into_response(),
2289 }
2290}
2291
2292#[allow(clippy::too_many_arguments)] async fn enforce_pipeline_inner(
2297 state: &AppState,
2298 body: &Bytes,
2299 base_request: ModelRequest,
2300 auth: Auth,
2301 features: Features,
2302 route: &Route,
2303 session_header: Option<String>,
2304 tenant: String,
2305 api: &str,
2306 routing_mode: RoutingMode,
2307 route_ix: usize,
2308 allow_challenger: bool,
2312) -> Result<ModelResponse, ProxyError> {
2313 let gate_defs = state
2314 .config
2315 .routing
2316 .as_ref()
2317 .map_or(&[][..], |cfg| &cfg.gate_defs);
2318 let gates = resolve_gates(
2319 &route.gates,
2320 gate_defs,
2321 &state.providers,
2322 &auth,
2323 &state.config.prices,
2324 &state.http,
2325 );
2326 let session_id = session_header.unwrap_or_else(|| Uuid::now_v7().to_string());
2327
2328 let experiment = route
2337 .challenger
2338 .as_ref()
2339 .filter(|_| allow_challenger)
2340 .map(|ch| {
2341 let bucket = firstpass_core::Challenger::bucket(&state.config.prompt_salt, &session_id);
2342 let assigned = ch.arm_of(bucket);
2343 let arm = if ch.is_paired()
2344 || (assigned == firstpass_core::Arm::Challenger
2345 && state.experiments.challenger_blocked(&tenant, route_ix))
2346 {
2347 firstpass_core::Arm::Incumbent
2348 } else {
2349 assigned
2350 };
2351 (
2352 ch,
2353 firstpass_core::trace::ExperimentRecord {
2354 arm,
2355 percent: ch.percent,
2356 bucket,
2357 fingerprint: ch.fingerprint(&route.ladder),
2358 },
2359 )
2360 });
2361 let on_challenger = experiment
2362 .as_ref()
2363 .is_some_and(|(_, rec)| rec.arm == firstpass_core::Arm::Challenger);
2364 let ladder: &[String] = match &experiment {
2367 Some((ch, rec)) if rec.arm == firstpass_core::Arm::Challenger => &ch.ladder,
2368 _ => &route.ladder,
2369 };
2370 let (budget, max_rungs, speculation, serve_threshold, elastic) =
2371 match state.config.routing.as_ref() {
2372 Some(cfg) => (
2373 cfg.budget.per_request_usd,
2374 cfg.escalation.max_rungs_per_request,
2375 cfg.escalation.speculation,
2376 cfg.escalation.serve_threshold,
2377 cfg.escalation.elastic.as_ref(),
2378 ),
2379 None => (None, 3, 0, None, None),
2380 };
2381 let certified = state.eprocess.as_ref().and_then(|e| {
2396 e.lock()
2397 .ok()
2398 .and_then(|g| g.certified_threshold().map(|c| (c, g.alpha(), g.delta())))
2399 });
2400 if let Some((c, _, _)) = &certified {
2401 metrics::gauge!("firstpass_eprocess_certified_threshold").set(c.threshold);
2402 metrics::gauge!("firstpass_eprocess_e_value").set(c.e_value);
2403 }
2404 let aci = state
2405 .adaptive
2406 .as_ref()
2407 .and_then(|a| a.lock().ok().map(|g| (g.threshold(), g.alpha())));
2408 let risk = risk_certificate(certified, aci, serve_threshold);
2411 let serve_threshold = risk.as_ref().map(|r| r.threshold);
2412
2413 let preset = routing_mode.preset();
2416 let max_rungs = if let Some(delta) = preset.max_rungs_delta {
2417 (max_rungs as i32 + delta).max(1) as u32
2418 } else {
2419 max_rungs
2420 };
2421 let speculation = preset.speculation.unwrap_or(speculation);
2422
2423 let bandit_ctx = crate::bandit::ContextBucket::from_features(&features);
2427
2428 let decision_prior: Option<Vec<f64>> = match state.prior.as_ref().filter(|_| !on_challenger) {
2434 Some(prior_client) => {
2435 prior_client
2436 .fetch(&crate::prior::query_text(&base_request))
2437 .await
2438 }
2439 None => None,
2440 };
2441
2442 let (greedy_rung, base_policy_id, ts_propensity) = if on_challenger {
2453 (0, "challenger-ladder@v0".to_owned(), None)
2454 } else {
2455 let (chosen, ts_p) = match state.bandit.as_ref() {
2456 Some(bandit) => bandit
2457 .lock()
2458 .ok()
2459 .map(|mut b| {
2460 b.choose_start_with_prior(
2461 &bandit_ctx,
2462 &route.ladder,
2463 &state.config.prices,
2464 decision_prior.as_deref(),
2465 )
2466 })
2467 .unwrap_or((0, None)),
2468 None => match (state.prior.as_ref(), decision_prior.as_deref()) {
2471 (Some(prior_client), Some(prior)) => crate::bandit::StartRungBandit::new(0, 1.0)
2472 .with_prior_strength(prior_client.strength())
2473 .choose_start_with_prior(
2474 &bandit_ctx,
2475 &route.ladder,
2476 &state.config.prices,
2477 Some(prior),
2478 ),
2479 _ => (0, None),
2480 },
2481 };
2482 let policy = if decision_prior.is_some() {
2483 "bandit@v3-prior".to_owned()
2484 } else if ts_p.is_some() {
2485 "bandit@v2-ts".to_owned()
2486 } else if chosen > 0 {
2487 "bandit@v1".to_owned()
2488 } else {
2489 "static-ladder@v0".to_owned()
2490 };
2491 (chosen, policy, ts_p)
2492 };
2493
2494 let exploration_epsilon = state
2499 .config
2500 .routing
2501 .as_ref()
2502 .and_then(|cfg| cfg.escalation.exploration.as_ref())
2503 .map(|e| e.epsilon)
2504 .filter(|_| !on_challenger);
2505
2506 let (start_rung, policy_id, explore_flag, propensity) = if ts_propensity.is_some() {
2507 if exploration_epsilon.is_some() {
2510 tracing::warn!(
2511 "bandit.algorithm = thompson already logs propensities; \
2512 [escalation.exploration] epsilon is ignored"
2513 );
2514 }
2515 (greedy_rung, base_policy_id, false, ts_propensity)
2516 } else if let Some(epsilon) = exploration_epsilon {
2517 let k = ladder.len().max(1);
2518 let u = u01(Uuid::now_v7().as_u128());
2520 let (chosen, eps_branch) = if u < epsilon {
2521 let idx = ((u / epsilon) * k as f64) as u32;
2523 (idx.min(k as u32 - 1), true)
2524 } else {
2525 (greedy_rung, false)
2526 };
2527 let p = epsilon_propensity(chosen, greedy_rung, epsilon, k);
2528 (chosen, format!("{base_policy_id}+eps"), eps_branch, Some(p))
2529 } else {
2530 (greedy_rung, base_policy_id, false, None)
2531 };
2532
2533 let promotion = state.promoter.as_ref().filter(|_| !on_challenger).cloned();
2538 let promotion = match promotion {
2541 Some(p) => {
2542 p.decide_async(&tenant, &session_id, std::time::Instant::now())
2543 .await
2544 }
2545 None => crate::affinity::Decision::Cold,
2546 };
2547 let start_rung = start_rung.max(promotion.start_rung());
2548 let promo_key = state
2551 .promoter
2552 .as_ref()
2553 .filter(|_| !on_challenger)
2554 .map(|_| (tenant.clone(), session_id.clone()));
2555
2556 let start_rung = if preset.start_at_top {
2559 ladder.len().saturating_sub(1) as u32
2560 } else {
2561 start_rung
2562 };
2563
2564 let speculation = match state
2570 .config
2571 .routing
2572 .as_ref()
2573 .and_then(|cfg| cfg.escalation.speculation_band)
2574 {
2575 Some([lo, hi]) if speculation > 0 && !on_challenger => {
2576 let estimate = state
2577 .bandit
2578 .as_ref()
2579 .and_then(|b| b.lock().ok())
2580 .and_then(|b| b.pass_estimate(&bandit_ctx, start_rung));
2581 match estimate {
2582 Some(p) if p < lo || p > hi => {
2583 metrics::counter!("firstpass_speculation_skipped_total").increment(1);
2584 0
2585 }
2586 _ => speculation,
2587 }
2588 }
2589 _ => speculation,
2590 };
2591
2592 if state.bandit.is_some() && !on_challenger {
2594 metrics::counter!(
2595 "firstpass_bandit_start_rung",
2596 "rung" => start_rung.to_string()
2597 )
2598 .increment(1);
2599 }
2600
2601 let mut base_request = base_request;
2602 base_request.cache_prefix = state
2606 .config
2607 .routing
2608 .as_ref()
2609 .is_some_and(|r| r.escalation.prompt_cache);
2610 let paired_spawn_data: Option<(Features, String, String)> = experiment
2613 .as_ref()
2614 .filter(|(ch, _)| ch.is_paired())
2615 .map(|_| (features.clone(), tenant.clone(), session_id.clone()));
2616
2617 let ctx = EnforceCtx {
2618 condense: routing_cfg_condense(state),
2619 ladder,
2620 gates: &gates,
2621 health: &state.gate_health,
2622 base_request: &base_request,
2623 providers: &state.providers,
2624 auth: &auth,
2625 prices: &state.config.prices,
2626 budget_per_request_usd: budget,
2627 max_rungs,
2628 speculation,
2629 serve_threshold,
2630 elastic,
2631 features,
2632 start_rung,
2633 tenant_id: tenant,
2636 session_id,
2637 prompt_hash: prompt_hash(&state.config.prompt_salt, body),
2638 api: api.to_owned(),
2639 policy_id,
2640 };
2641
2642 if let Some(cache) = state.verified_cache.as_ref()
2647 && let Some(entry) = cache
2648 .get(
2649 &crate::verified_cache::VerifiedCache::compose_key(
2650 &ctx.tenant_id,
2651 &ctx.prompt_hash,
2652 ladder,
2655 ),
2656 std::time::Instant::now(),
2657 )
2658 .await
2659 {
2660 metrics::counter!("firstpass_cache_hits_total").increment(1);
2661 let mut trace = cache_hit_trace(&ctx, &entry, mode_now());
2662 if let Some((_, rec)) = &experiment {
2663 trace.route_ix = Some(u32::try_from(route_ix).unwrap_or(0));
2668 trace.experiment = Some(rec.clone());
2669 state.experiments.observe_cost(
2670 &trace.tenant_id,
2671 route_ix,
2672 &rec.fingerprint,
2673 rec.arm,
2674 trace.final_.total_cost_usd,
2675 );
2676 }
2677 offer_trace(&state.traces, state.spill.as_ref(), trace);
2678 return Ok(entry.response);
2679 }
2680
2681 let (outcome, mut trace) = route_enforce(ctx).await;
2682
2683 trace.route_ix = Some(u32::try_from(route_ix).unwrap_or(0));
2687 trace.policy.explore = explore_flag;
2690 trace.policy.propensity = propensity;
2691 if routing_mode != RoutingMode::Balanced {
2694 trace.policy.mode_profile = Some(routing_mode.as_str().to_owned());
2695 }
2696 trace.decision_prior = decision_prior;
2699 trace.risk = risk;
2700 if let Some((_, rec)) = &experiment {
2703 state.experiments.observe_cost(
2704 &trace.tenant_id,
2705 route_ix,
2706 &rec.fingerprint,
2707 rec.arm,
2708 trace.final_.total_cost_usd,
2709 );
2710 state.experiments.observe_health(
2711 &trace.tenant_id,
2712 route_ix,
2713 &rec.fingerprint,
2714 rec.arm,
2715 trace.final_.served_from == firstpass_core::ServedFrom::Error,
2716 crate::experiment::truncated_attempts(&trace),
2717 );
2718 trace.experiment = Some(rec.clone());
2719 }
2720
2721 if let Some(bandit) = state.bandit.as_ref().filter(|_| !on_challenger)
2725 && let Ok(mut b) = bandit.lock()
2726 {
2727 for attempt in &trace.attempts {
2728 b.observe(&bandit_ctx, attempt.rung, attempt.verdict);
2729 }
2730 }
2731
2732 if let Some(promoter) = state.promoter.as_ref()
2737 && let Some((tenant_key, session_key)) = promo_key.as_ref()
2738 && let Some(served) = trace.final_.served_rung
2739 {
2740 promoter
2741 .record_async(
2742 tenant_key,
2743 session_key,
2744 served,
2745 served > start_rung,
2746 std::time::Instant::now(),
2747 )
2748 .await;
2749 }
2750
2751 if let Some(predictor) = state.predictor.as_ref().filter(|_| !on_challenger)
2757 && let Ok(mut p) = predictor.lock()
2758 {
2759 let predicted = p.predict(&trace.request.features, start_rung);
2761 for attempt in &trace.attempts {
2762 match attempt.verdict {
2763 Verdict::Pass => p.update(&trace.request.features, attempt.rung, true),
2764 Verdict::Fail => p.update(&trace.request.features, attempt.rung, false),
2765 Verdict::Abstain => {} }
2767 }
2768 trace.predicted_pass = Some(predicted);
2769 metrics::histogram!("firstpass_predictor_pass_prob").record(predicted);
2770 }
2771
2772 if let Some(probe_cfg) = state
2779 .config
2780 .routing
2781 .as_ref()
2782 .and_then(|c| c.escalation.probe)
2783 && u01(Uuid::now_v7().as_u128()) < probe_cfg.sample_rate
2784 {
2785 let probe_rung = (start_rung as usize).min(ladder.len().saturating_sub(1));
2787 if let Some(probe_model_str) = ladder.get(probe_rung).cloned() {
2788 let probe_provider = ModelRef::parse(&probe_model_str)
2789 .ok()
2790 .and_then(|m| state.providers.get(&m.provider));
2791
2792 if let Some(probe_provider) = probe_provider {
2793 let mut join_set = tokio::task::JoinSet::new();
2796 for _ in 0..probe_cfg.k {
2797 let mut probe_req = base_request.clone();
2798 probe_req.model = probe_model_str.clone();
2799 let probe_auth = auth.clone();
2800 let prov = probe_provider.clone();
2801 join_set.spawn(async move { prov.complete(&probe_req, &probe_auth).await });
2802 }
2803
2804 let fail_closed_owned: std::collections::HashSet<String> = gates
2807 .iter()
2808 .filter(|g| g.abstain_fails_closed())
2809 .map(|g| g.id().to_owned())
2810 .collect();
2811
2812 let mut gate_pass_count = 0u32;
2813 let mut probe_cost_usd = 0.0f64;
2814
2815 while let Some(task_result) = join_set.join_next().await {
2816 let Ok(Ok(probe_resp)) = task_result else {
2817 continue; };
2819 probe_cost_usd += state
2822 .config
2823 .prices
2824 .cost_usd_with_cache(
2825 &probe_model_str,
2826 probe_resp.in_tokens,
2827 probe_resp.cache_write_tokens,
2828 probe_resp.cache_read_tokens,
2829 probe_resp.out_tokens,
2830 )
2831 .unwrap_or(0.0);
2832
2833 let mut probe_gate_req = base_request.clone();
2834 probe_gate_req.model = probe_model_str.clone();
2835
2836 let mut probe_gate_results = Vec::with_capacity(gates.len());
2839 for g in &gates {
2840 if !state.gate_health.enabled(&trace.tenant_id, g.id()) {
2842 continue;
2843 }
2844 let r = g.evaluate(&probe_gate_req, &probe_resp).await;
2845 probe_gate_results.push(r);
2847 }
2848
2849 let fail_closed_refs: std::collections::HashSet<&str> =
2850 fail_closed_owned.iter().map(|s| s.as_str()).collect();
2851 let verdict = aggregate_with_policy(&probe_gate_results, &fail_closed_refs);
2852 let passes = match serve_threshold {
2855 None => verdict == Verdict::Pass,
2856 Some(t) => crate::calibrate::gate_score(&probe_gate_results, verdict) >= t,
2857 };
2858 if passes {
2859 gate_pass_count += 1;
2860 }
2861 }
2862
2863 let regime = ProbeRegime::classify(gate_pass_count, probe_cfg.k);
2864 let regime_label = match regime {
2865 ProbeRegime::ConfidentPass => "confident_pass",
2866 ProbeRegime::ConfidentFail => "confident_fail",
2867 ProbeRegime::Ambiguous => "ambiguous",
2868 };
2869 metrics::counter!(
2870 "firstpass_probe_regime_total",
2871 "regime" => regime_label
2872 )
2873 .increment(1);
2874 metrics::gauge!("firstpass_probe_cost_usd_total").increment(probe_cost_usd);
2875 trace.probe = Some(ProbeSignal {
2876 k: probe_cfg.k,
2877 gate_pass_count,
2878 regime,
2879 probe_cost_usd,
2880 });
2881 }
2882 }
2883 }
2884 if let Some(cache) = state.verified_cache.as_ref()
2897 && let EngineOutcome::Served(resp) = &outcome
2898 && crate::verified_cache::is_cacheable(&trace)
2899 {
2900 let now = std::time::Instant::now();
2901 cache
2902 .put(
2903 &crate::verified_cache::VerifiedCache::compose_key(
2904 &trace.tenant_id,
2905 &trace.request.prompt_hash,
2906 ladder,
2907 ),
2908 trace.trace_id,
2909 crate::verified_cache::Entry::new(resp.clone(), &trace, now),
2910 now,
2911 )
2912 .await;
2913 metrics::counter!("firstpass_cache_stores_total").increment(1);
2914 }
2915
2916 if let Some((ch, rec)) = &experiment
2920 && ch.is_paired()
2921 {
2922 let trace_id_str = trace.trace_id.to_string();
2923 if ch.paired_sampled(&state.config.prompt_salt, &trace_id_str)
2924 && let Some((spawn_feats, spawn_tenant, spawn_session)) = paired_spawn_data
2925 {
2926 let served_from = trace.final_.served_from;
2927 let incumbent_cost = trace.final_.total_cost_usd;
2928 spawn_paired_challenger(
2929 state,
2930 route,
2931 ch,
2932 rec.fingerprint.clone(),
2933 route_ix,
2934 body.clone(),
2935 auth.clone(),
2936 spawn_feats,
2937 spawn_tenant,
2938 spawn_session,
2939 api,
2940 served_from,
2941 incumbent_cost,
2942 );
2943 }
2944 }
2945
2946 offer_trace(&state.traces, state.spill.as_ref(), trace);
2948
2949 match outcome {
2950 EngineOutcome::Served(resp) => Ok(resp),
2951 EngineOutcome::Failed(msg) => Err(ProxyError::Engine(msg)),
2952 }
2953}
2954
2955#[allow(clippy::too_many_arguments)]
2958async fn enforce_pipeline(
2959 state: &AppState,
2960 headers: &HeaderMap,
2961 body: &Bytes,
2962 features: Features,
2963 route: &Route,
2964 route_ix: usize,
2966 session_header: Option<String>,
2967 tenant: String,
2968 routing_mode: RoutingMode,
2969) -> Result<Value, ProxyError> {
2970 let Some(base_request) = parse_model_request(body) else {
2971 return Err(ProxyError::BadRequest(
2972 "request body is not a valid Anthropic Messages request".to_owned(),
2973 ));
2974 };
2975 let auth = Auth::from_headers(headers);
2976 let resp = enforce_pipeline_inner(
2977 state,
2978 body,
2979 base_request,
2980 auth,
2981 features,
2982 route,
2983 session_header,
2984 tenant,
2985 "anthropic.messages",
2986 routing_mode,
2987 route_ix,
2988 true,
2989 )
2990 .await?;
2991 Ok(anthropic_response_json(&resp))
2992}
2993
2994#[allow(clippy::too_many_arguments)]
2997async fn enforce_pipeline_openai(
2998 state: &AppState,
2999 headers: &HeaderMap,
3000 body: &Bytes,
3001 features: Features,
3002 route: &Route,
3003 route_ix: usize,
3005 session_header: Option<String>,
3006 tenant: String,
3007 routing_mode: RoutingMode,
3008) -> Result<Value, ProxyError> {
3009 enforce_pipeline_openai_as(
3010 state,
3011 headers,
3012 body,
3013 body,
3014 "openai.chat_completions",
3015 features,
3016 route,
3017 route_ix,
3018 session_header,
3019 tenant,
3020 routing_mode,
3021 )
3022 .await
3023}
3024
3025#[allow(clippy::too_many_arguments)]
3034async fn enforce_pipeline_openai_as(
3035 state: &AppState,
3036 headers: &HeaderMap,
3037 body: &Bytes,
3038 receipt_body: &Bytes,
3039 api: &str,
3040 features: Features,
3041 route: &Route,
3042 route_ix: usize,
3043 session_header: Option<String>,
3044 tenant: String,
3045 routing_mode: RoutingMode,
3046) -> Result<Value, ProxyError> {
3047 let providers = &state.providers;
3049 let all_openai_of = |ladder: &[String]| {
3050 ladder.iter().all(|rung| {
3051 let pid = rung.split('/').next().unwrap_or_default();
3052 providers
3053 .get(pid)
3054 .is_some_and(|p| p.carries_structured_verbatim(Dialect::Openai))
3055 })
3056 };
3057 let all_openai = all_openai_of(&route.ladder);
3058 let allow_challenger = route
3063 .challenger
3064 .as_ref()
3065 .is_none_or(|c| all_openai_of(&c.ladder) == all_openai);
3066 let Some(base_request) = parse_openai_request(body, all_openai) else {
3067 return Err(ProxyError::BadRequest(
3068 "request body is not a valid OpenAI Chat Completions request".to_owned(),
3069 ));
3070 };
3071 let auth = Auth::from_headers(headers);
3072 let resp = enforce_pipeline_inner(
3073 state,
3074 receipt_body,
3075 base_request,
3076 auth,
3077 features,
3078 route,
3079 session_header,
3080 tenant,
3081 api,
3082 routing_mode,
3083 route_ix,
3084 allow_challenger,
3085 )
3086 .await?;
3087 Ok(openai_response_json(&resp))
3088}
3089
3090const SSE_KEEPALIVE_EVERY: Duration = Duration::from_secs(5);
3092
3093fn sse_keepalive_response(
3102 rx: tokio::sync::oneshot::Receiver<Result<Value, ProxyError>>,
3103 format_message: fn(&Value) -> String,
3104) -> Response {
3105 let mut ticks = tokio::time::interval(SSE_KEEPALIVE_EVERY);
3106 ticks.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
3107 ticks.reset(); let stream = KeepaliveStream {
3109 rx: Some(rx),
3110 ticks,
3111 format_message,
3112 };
3113 (
3114 axum::http::StatusCode::OK,
3115 [(
3116 axum::http::header::CONTENT_TYPE,
3117 "text/event-stream; charset=utf-8",
3118 )],
3119 axum::body::Body::from_stream(stream),
3120 )
3121 .into_response()
3122}
3123
3124struct KeepaliveStream {
3127 rx: Option<tokio::sync::oneshot::Receiver<Result<Value, ProxyError>>>,
3129 ticks: tokio::time::Interval,
3130 format_message: fn(&Value) -> String,
3132}
3133
3134impl futures_core::Stream for KeepaliveStream {
3135 type Item = Result<Bytes, std::convert::Infallible>;
3136
3137 fn poll_next(
3138 mut self: std::pin::Pin<&mut Self>,
3139 cx: &mut std::task::Context<'_>,
3140 ) -> std::task::Poll<Option<Self::Item>> {
3141 use std::task::Poll;
3142 let Some(rx) = self.rx.as_mut() else {
3143 return Poll::Ready(None); };
3145 if let Poll::Ready(out) = std::pin::Pin::new(rx).poll(cx) {
3146 let fmt = self.format_message;
3147 let frame = match out {
3148 Ok(Ok(message)) => fmt(&message),
3149 Ok(Err(e)) => sse_error_event(&e),
3150 Err(_) => sse_error_event(&ProxyError::Internal(
3151 "enforce pipeline task dropped".to_owned(),
3152 )),
3153 };
3154 self.rx = None;
3155 return Poll::Ready(Some(Ok(Bytes::from(frame))));
3156 }
3157 if self.ticks.poll_tick(cx).is_ready() {
3158 return Poll::Ready(Some(Ok(Bytes::from_static(b": firstpass routing\n\n"))));
3159 }
3160 Poll::Pending
3161 }
3162}
3163
3164fn sse_error_event(e: &ProxyError) -> String {
3167 let mut out = String::new();
3168 sse_event(
3169 &mut out,
3170 "error",
3171 &serde_json::json!({
3172 "type": "error",
3173 "error": { "type": "api_error", "message": e.client_message() }
3174 }),
3175 );
3176 out
3177}
3178
3179fn parse_model_request(body: &[u8]) -> Option<ModelRequest> {
3188 let json: Value = serde_json::from_slice(body).ok()?;
3189 let raw = json.clone();
3190 let messages_json = json.get("messages")?.as_array()?;
3191 let messages = messages_json
3192 .iter()
3193 .map(|m| ChatMessage {
3194 role: m
3195 .get("role")
3196 .and_then(Value::as_str)
3197 .unwrap_or("user")
3198 .to_owned(),
3199 content: m
3200 .get("content")
3201 .cloned()
3202 .unwrap_or_else(|| Value::String(String::new())),
3203 })
3204 .collect();
3205 let system = json
3206 .get("system")
3207 .and_then(Value::as_str)
3208 .map(str::to_owned);
3209 let max_tokens = json
3210 .get("max_tokens")
3211 .and_then(Value::as_u64)
3212 .and_then(|n| u32::try_from(n).ok())
3213 .unwrap_or(1024);
3214 let tools = json.get("tools").cloned().unwrap_or(Value::Null);
3215 Some(ModelRequest {
3216 model: json
3217 .get("model")
3218 .and_then(Value::as_str)
3219 .unwrap_or_default()
3220 .to_owned(),
3221 system,
3222 messages,
3223 max_tokens,
3224 tools,
3225 raw,
3226 cache_prefix: false,
3227 })
3228}
3229
3230fn anthropic_response_json(resp: &ModelResponse) -> Value {
3240 let content = resp
3241 .raw
3242 .get("content")
3243 .filter(|c| c.is_array())
3244 .cloned()
3245 .unwrap_or_else(|| serde_json::json!([{ "type": "text", "text": resp.text }]));
3246 serde_json::json!({
3247 "id": format!("msg_{}", Uuid::now_v7()),
3248 "type": "message",
3249 "role": "assistant",
3250 "model": resp.model,
3251 "content": content,
3252 "usage": { "input_tokens": resp.in_tokens, "output_tokens": resp.out_tokens },
3253 })
3254}
3255
3256fn sse_event(out: &mut String, event: &str, data: &Value) {
3258 out.push_str("event: ");
3259 out.push_str(event);
3260 out.push_str("\ndata: ");
3261 out.push_str(&data.to_string());
3262 out.push_str("\n\n");
3263}
3264
3265fn anthropic_sse_from_message(message: &Value) -> String {
3272 let mut out = String::new();
3273
3274 let mut start_msg = message.clone();
3276 start_msg["content"] = Value::Array(Vec::new());
3277 sse_event(
3278 &mut out,
3279 "message_start",
3280 &serde_json::json!({ "type": "message_start", "message": start_msg }),
3281 );
3282
3283 let empty = Vec::new();
3284 let blocks = message
3285 .get("content")
3286 .and_then(Value::as_array)
3287 .unwrap_or(&empty);
3288 for (i, block) in blocks.iter().enumerate() {
3289 match block.get("type").and_then(Value::as_str) {
3290 Some("tool_use") => {
3291 let mut shell = block.clone();
3293 shell["input"] = serde_json::json!({});
3294 sse_event(
3295 &mut out,
3296 "content_block_start",
3297 &serde_json::json!({ "type": "content_block_start", "index": i, "content_block": shell }),
3298 );
3299 let input_json = block
3300 .get("input")
3301 .map_or_else(|| "{}".to_owned(), std::string::ToString::to_string);
3302 sse_event(
3303 &mut out,
3304 "content_block_delta",
3305 &serde_json::json!({ "type": "content_block_delta", "index": i,
3306 "delta": { "type": "input_json_delta", "partial_json": input_json } }),
3307 );
3308 }
3309 _ => {
3310 let text = block.get("text").and_then(Value::as_str).unwrap_or("");
3312 sse_event(
3313 &mut out,
3314 "content_block_start",
3315 &serde_json::json!({ "type": "content_block_start", "index": i,
3316 "content_block": { "type": "text", "text": "" } }),
3317 );
3318 sse_event(
3319 &mut out,
3320 "content_block_delta",
3321 &serde_json::json!({ "type": "content_block_delta", "index": i,
3322 "delta": { "type": "text_delta", "text": text } }),
3323 );
3324 }
3325 }
3326 sse_event(
3327 &mut out,
3328 "content_block_stop",
3329 &serde_json::json!({ "type": "content_block_stop", "index": i }),
3330 );
3331 }
3332
3333 let out_tokens = message
3334 .pointer("/usage/output_tokens")
3335 .cloned()
3336 .unwrap_or_else(|| Value::from(0));
3337 sse_event(
3338 &mut out,
3339 "message_delta",
3340 &serde_json::json!({ "type": "message_delta", "delta": { "stop_reason": "end_turn" },
3341 "usage": { "output_tokens": out_tokens } }),
3342 );
3343 sse_event(
3344 &mut out,
3345 "message_stop",
3346 &serde_json::json!({ "type": "message_stop" }),
3347 );
3348 out
3349}
3350
3351async fn observe_passthrough(
3353 state: AppState,
3354 headers: HeaderMap,
3355 body: Bytes,
3356 session_header: Option<String>,
3357 tenant: String,
3358) -> Response {
3359 if is_stream_request(&body) {
3361 return observe_stream(state, headers, body, session_header, tenant).await;
3362 }
3363 let start = Instant::now();
3364 let result = forward_anthropic(
3365 &state.http,
3366 &state.config.upstream_anthropic,
3367 &headers,
3368 body.clone(),
3369 )
3370 .await;
3371 let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
3372
3373 match result {
3374 Ok((status, resp_headers, resp_body)) => {
3375 spawn_trace(
3379 &state,
3380 body,
3381 Some(resp_body.clone()),
3382 latency_ms,
3383 session_header,
3384 tenant,
3385 );
3386 (status, resp_headers, resp_body).into_response()
3387 }
3388 Err(err) => {
3389 spawn_trace(&state, body, None, latency_ms, session_header, tenant);
3390 err.into_response()
3391 }
3392 }
3393}
3394
3395async fn observe_stream(
3404 state: AppState,
3405 headers: HeaderMap,
3406 body: Bytes,
3407 session_header: Option<String>,
3408 tenant: String,
3409) -> Response {
3410 let start = Instant::now();
3411 let result = forward_anthropic_streaming(
3412 &state.http,
3413 &state.config.upstream_anthropic,
3414 &headers,
3415 body.clone(),
3416 )
3417 .await;
3418 let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
3419
3420 match result {
3421 Ok((status, resp_headers, response)) => {
3422 spawn_stream_trace(&state, body, latency_ms, session_header, tenant);
3423 let stream_body = Body::from_stream(response.bytes_stream());
3424 (status, resp_headers, stream_body).into_response()
3425 }
3426 Err(err) => {
3427 spawn_trace(&state, body, None, latency_ms, session_header, tenant);
3428 err.into_response()
3429 }
3430 }
3431}
3432
3433fn spawn_stream_trace(
3435 state: &AppState,
3436 req_body: Bytes,
3437 latency_ms: u64,
3438 session_header: Option<String>,
3439 tenant: String,
3440) {
3441 let config = state.config.clone();
3442 let traces = state.traces.clone();
3443 let spill = state.spill.clone();
3444 tokio::spawn(async move {
3445 let mut trace =
3446 build_stream_trace(&config, &req_body, latency_ms, session_header.as_deref());
3447 trace.tenant_id = tenant;
3449 offer_trace(&traces, spill.as_ref(), trace);
3450 });
3451}
3452
3453fn spawn_trace(
3458 state: &AppState,
3459 req_body: Bytes,
3460 resp_body: Option<Bytes>,
3461 latency_ms: u64,
3462 session_header: Option<String>,
3463 tenant: String,
3464) {
3465 let config = state.config.clone();
3466 let traces = state.traces.clone();
3467 let spill = state.spill.clone();
3468 tokio::spawn(async move {
3469 let mut trace = match resp_body {
3470 Some(resp) => build_trace(
3471 &config,
3472 &req_body,
3473 &resp,
3474 latency_ms,
3475 session_header.as_deref(),
3476 ),
3477 None => build_error_trace(&config, &req_body, latency_ms, session_header.as_deref()),
3478 };
3479 trace.tenant_id = tenant;
3481 offer_trace(&traces, spill.as_ref(), trace);
3482 });
3483}
3484
3485fn session_id(session_header: Option<&str>, trace_id: Uuid) -> String {
3487 session_header
3488 .map(str::to_owned)
3489 .unwrap_or_else(|| trace_id.to_string())
3490}
3491
3492fn prompt_hash(salt: &str, body: &[u8]) -> String {
3495 let mut salted = Vec::with_capacity(salt.len() + body.len());
3496 salted.extend_from_slice(salt.as_bytes());
3497 salted.extend_from_slice(body);
3498 sha256_hex(&salted)
3499}
3500
3501fn request_features(body: &[u8]) -> (Option<String>, u32, bool) {
3505 let Ok(json) = serde_json::from_slice::<Value>(body) else {
3506 return (None, 0, false);
3507 };
3508 let model = json.get("model").and_then(Value::as_str).map(str::to_owned);
3509 let tool_count = json
3510 .get("tools")
3511 .and_then(Value::as_array)
3512 .map_or(0, |tools| u32::try_from(tools.len()).unwrap_or(u32::MAX));
3513 let has_images = json
3514 .get("messages")
3515 .and_then(Value::as_array)
3516 .is_some_and(|messages| messages.iter().any(message_has_image));
3517 (model, tool_count, has_images)
3518}
3519
3520fn message_has_image(message: &Value) -> bool {
3522 message
3523 .get("content")
3524 .and_then(Value::as_array)
3525 .is_some_and(|blocks| {
3526 blocks
3527 .iter()
3528 .any(|block| block.get("type").and_then(Value::as_str) == Some("image"))
3529 })
3530}
3531
3532#[derive(Debug, Default, Clone, PartialEq, Eq)]
3534struct ResponseUsage {
3535 model: Option<String>,
3536 in_tokens: u64,
3537 cache_write_tokens: u64,
3538 cache_read_tokens: u64,
3539 out_tokens: u64,
3540}
3541
3542fn response_usage(body: &[u8]) -> ResponseUsage {
3550 let Ok(json) = serde_json::from_slice::<Value>(body) else {
3551 return ResponseUsage::default();
3552 };
3553 let u = |ptr: &str| json.pointer(ptr).and_then(Value::as_u64).unwrap_or(0);
3554 ResponseUsage {
3555 model: json.get("model").and_then(Value::as_str).map(str::to_owned),
3556 in_tokens: u("/usage/input_tokens"),
3557 cache_write_tokens: u("/usage/cache_creation_input_tokens"),
3558 cache_read_tokens: u("/usage/cache_read_input_tokens"),
3559 out_tokens: u("/usage/output_tokens"),
3560 }
3561}
3562
3563fn build_trace(
3565 config: &ProxyConfig,
3566 req_body: &Bytes,
3567 resp_body: &Bytes,
3568 latency_ms: u64,
3569 session_header: Option<&str>,
3570) -> Trace {
3571 let (req_model, tool_count, has_images) = request_features(req_body);
3572 let usage = response_usage(resp_body);
3573 let (resp_model, in_tokens, out_tokens) = (usage.model, usage.in_tokens, usage.out_tokens);
3574 let (cache_write_tokens, cache_read_tokens) =
3575 (usage.cache_write_tokens, usage.cache_read_tokens);
3576 let model = resp_model
3577 .or(req_model)
3578 .unwrap_or_else(|| "unknown".to_owned());
3579
3580 let cost_usd = config
3581 .prices
3582 .cost_usd_with_cache(
3583 &format!("anthropic/{model}"),
3584 in_tokens,
3585 cache_write_tokens,
3586 cache_read_tokens,
3587 out_tokens,
3588 )
3589 .unwrap_or(0.0);
3590
3591 let attempt = Attempt {
3592 rung: 0,
3593 model,
3594 provider: "anthropic".to_owned(),
3595 in_tokens,
3596 cache_write_tokens,
3597 cache_read_tokens,
3598 out_tokens,
3599 cost_usd,
3600 latency_ms,
3601 gates: Vec::new(),
3602 verdict: Verdict::Pass,
3603 truncated: serde_json::from_slice::<Value>(resp_body)
3604 .is_ok_and(|v| crate::provider::hit_output_limit(&v)),
3605 };
3606
3607 let mut trace = base_trace(config, req_body, latency_ms, session_header);
3608 trace.request.features.prompt_token_bucket = token_bucket(in_tokens);
3609 trace.request.features.tool_count = tool_count;
3610 trace.request.features.has_images = has_images;
3611 trace.attempts.push(attempt);
3612 trace.final_ = FinalOutcome {
3613 served_rung: Some(0),
3614 served_from: ServedFrom::Attempt,
3615 total_cost_usd: cost_usd,
3616 gate_cost_usd: 0.0,
3617 total_latency_ms: latency_ms,
3618 escalations: 0,
3619 counterfactual_baseline_usd: cost_usd,
3620 savings_usd: 0.0,
3621 cache_source: None,
3622 };
3623 trace.recompute_savings();
3624 trace
3625}
3626
3627fn build_stream_trace(
3632 config: &ProxyConfig,
3633 req_body: &Bytes,
3634 latency_ms: u64,
3635 session_header: Option<&str>,
3636) -> Trace {
3637 let (req_model, tool_count, has_images) = request_features(req_body);
3638 let model = req_model.unwrap_or_else(|| "unknown".to_owned());
3639
3640 let attempt = Attempt {
3641 rung: 0,
3642 model,
3643 provider: "anthropic".to_owned(),
3644 in_tokens: 0,
3645 cache_write_tokens: 0,
3647 cache_read_tokens: 0,
3648 out_tokens: 0,
3649 cost_usd: 0.0,
3650 latency_ms,
3651 gates: Vec::new(),
3652 verdict: Verdict::Pass,
3653 truncated: false,
3654 };
3655
3656 let mut trace = base_trace(config, req_body, latency_ms, session_header);
3657 trace.request.features.tool_count = tool_count;
3658 trace.request.features.has_images = has_images;
3659 trace.attempts.push(attempt);
3660 trace.final_ = FinalOutcome {
3661 served_rung: Some(0),
3662 served_from: ServedFrom::Attempt,
3663 total_cost_usd: 0.0,
3664 gate_cost_usd: 0.0,
3665 total_latency_ms: latency_ms,
3666 escalations: 0,
3667 counterfactual_baseline_usd: 0.0,
3668 savings_usd: 0.0,
3669 cache_source: None,
3670 };
3671 trace.recompute_savings();
3672 trace
3673}
3674
3675fn build_error_trace(
3679 config: &ProxyConfig,
3680 req_body: &Bytes,
3681 latency_ms: u64,
3682 session_header: Option<&str>,
3683) -> Trace {
3684 let (_, tool_count, has_images) = request_features(req_body);
3685 let mut trace = base_trace(config, req_body, latency_ms, session_header);
3686 trace.request.features.tool_count = tool_count;
3687 trace.request.features.has_images = has_images;
3688 trace.final_ = FinalOutcome {
3689 served_rung: None,
3690 served_from: ServedFrom::Error,
3691 total_cost_usd: 0.0,
3692 gate_cost_usd: 0.0,
3693 total_latency_ms: latency_ms,
3694 escalations: 0,
3695 counterfactual_baseline_usd: 0.0,
3696 savings_usd: 0.0,
3697 cache_source: None,
3698 };
3699 trace.recompute_savings();
3700 trace
3701}
3702
3703fn base_trace(
3706 config: &ProxyConfig,
3707 req_body: &Bytes,
3708 latency_ms: u64,
3709 session_header: Option<&str>,
3710) -> Trace {
3711 let trace_id = Uuid::now_v7();
3712 let mut features = Features::new(TaskKind::Other);
3713 features.hour_bucket = hour_bucket(jiff::Timestamp::now());
3714
3715 Trace {
3716 trace_id,
3717 prev_hash: GENESIS_HASH.to_owned(),
3718 tenant_id: config.tenant_id.clone(),
3719 session_id: session_id(session_header, trace_id),
3720 ts: jiff::Timestamp::now(),
3721 mode: Mode::Observe,
3722 policy: PolicyRef {
3723 id: "observe-passthrough@v0".to_owned(),
3724 explore: false,
3725 propensity: None,
3726 mode_profile: None,
3727 },
3728 request: RequestInfo {
3729 api: "anthropic.messages".to_owned(),
3730 prompt_hash: prompt_hash(&config.prompt_salt, req_body),
3731 features,
3732 },
3733 attempts: Vec::new(),
3734 deferred: Vec::new(),
3735 final_: FinalOutcome {
3736 served_rung: None,
3737 served_from: ServedFrom::Error,
3738 total_cost_usd: 0.0,
3739 gate_cost_usd: 0.0,
3740 total_latency_ms: latency_ms,
3741 escalations: 0,
3742 counterfactual_baseline_usd: 0.0,
3743 savings_usd: 0.0,
3744 cache_source: None,
3745 },
3746 probe: None,
3747 rollout: None,
3748 shadow: None,
3749 route_ix: None,
3750 predicted_pass: None,
3751 decision_prior: None,
3752 elastic: None,
3753 risk: None,
3754 experiment: None,
3755 }
3756}
3757
3758fn openai_messages_have_tool_calls(body: &[u8]) -> bool {
3763 serde_json::from_slice::<Value>(body)
3764 .ok()
3765 .and_then(|json| {
3766 json.get("messages").and_then(Value::as_array).map(|msgs| {
3767 msgs.iter().any(|m| {
3768 m.get("tool_calls").is_some()
3769 || m.get("role").and_then(Value::as_str) == Some("tool")
3770 })
3771 })
3772 })
3773 .unwrap_or(false)
3774}
3775
3776fn openai_has_http_images(body: &[u8]) -> bool {
3780 serde_json::from_slice::<Value>(body)
3781 .ok()
3782 .and_then(|json| {
3783 json.get("messages").and_then(Value::as_array).map(|msgs| {
3784 msgs.iter().any(|m| {
3785 m.get("content")
3786 .and_then(Value::as_array)
3787 .is_some_and(|parts| {
3788 parts.iter().any(|p| {
3789 p.get("type").and_then(Value::as_str) == Some("image_url")
3790 && p.pointer("/image_url/url")
3791 .and_then(Value::as_str)
3792 .is_some_and(|u| {
3793 u.starts_with("http://") || u.starts_with("https://")
3794 })
3795 })
3796 })
3797 })
3798 })
3799 })
3800 .unwrap_or(false)
3801}
3802
3803fn openai_messages_have_images(body: &[u8]) -> bool {
3806 serde_json::from_slice::<Value>(body)
3807 .ok()
3808 .and_then(|json| {
3809 json.get("messages").and_then(Value::as_array).map(|msgs| {
3810 msgs.iter().any(|m| {
3811 m.get("content")
3812 .and_then(Value::as_array)
3813 .is_some_and(|parts| {
3814 parts
3815 .iter()
3816 .any(|p| p.get("type").and_then(Value::as_str) == Some("image_url"))
3817 })
3818 })
3819 })
3820 })
3821 .unwrap_or(false)
3822}
3823
3824fn extract_openai_features(headers: &HeaderMap, body: &[u8]) -> Features {
3827 let Ok(json) = serde_json::from_slice::<Value>(body) else {
3828 let mut f = Features::new(TaskKind::Other);
3829 f.hour_bucket = hour_bucket(jiff::Timestamp::now());
3830 return f;
3831 };
3832 let tool_count = json
3833 .get("tools")
3834 .and_then(Value::as_array)
3835 .map_or(0, |tools| u32::try_from(tools.len()).unwrap_or(u32::MAX));
3836 let has_images = openai_messages_have_images(body);
3837 let mut f = Features::new(TaskKind::Other);
3838 f.agent = header_str(headers, AGENT_HEADER);
3839 f.subagent = header_str(headers, SUBAGENT_HEADER);
3840 f.tool_count = tool_count;
3841 f.has_images = has_images;
3842 f.prompt_token_bucket = token_bucket(body.len() as u64);
3843 f.hour_bucket = hour_bucket(jiff::Timestamp::now());
3844 f.difficulty_hint = DifficultyHint::score(trajectory_signals(body)).as_u8();
3848 f
3849}
3850
3851fn parse_data_url(url: &str) -> Option<(&str, &str)> {
3855 let rest = url.strip_prefix("data:")?;
3856 let (meta, data) = rest.split_once(',')?;
3857 let media_type = meta.strip_suffix(";base64")?;
3858 Some((media_type, data))
3859}
3860
3861fn translate_openai_user_content(content: &Value) -> Option<Value> {
3864 match content {
3865 Value::String(_) => Some(content.clone()),
3867 Value::Array(parts) => {
3868 let mut blocks: Vec<Value> = Vec::with_capacity(parts.len());
3869 for part in parts {
3870 match part.get("type").and_then(Value::as_str) {
3871 Some("text") => {
3872 let text = part.get("text").and_then(Value::as_str).unwrap_or("");
3873 blocks.push(serde_json::json!({ "type": "text", "text": text }));
3874 }
3875 Some("image_url") => {
3876 let url = part.pointer("/image_url/url").and_then(Value::as_str)?;
3877 if url.starts_with("http://") || url.starts_with("https://") {
3878 return None; }
3880 let (media_type, data) = parse_data_url(url)?;
3882 blocks.push(serde_json::json!({
3883 "type": "image",
3884 "source": { "type": "base64", "media_type": media_type, "data": data }
3885 }));
3886 }
3887 _ => {} }
3889 }
3890 Some(Value::Array(blocks))
3891 }
3892 _ => Some(Value::String(String::new())),
3893 }
3894}
3895
3896fn translate_openai_tools(tools: &Value) -> Value {
3900 let Some(arr) = tools.as_array() else {
3901 return Value::Null;
3902 };
3903 let converted: Vec<Value> = arr
3904 .iter()
3905 .map(|tool| {
3906 let func = tool.get("function").unwrap_or(&Value::Null);
3907 let mut out = serde_json::json!({
3908 "name": func.get("name").cloned().unwrap_or(Value::String(String::new())),
3909 "input_schema": func.get("parameters").cloned()
3910 .unwrap_or_else(|| serde_json::json!({ "type": "object" })),
3911 });
3912 if let Some(desc) = func.get("description") {
3913 out["description"] = desc.clone();
3914 }
3915 out
3916 })
3917 .collect();
3918 Value::Array(converted)
3919}
3920
3921fn translate_openai_tool_choice(tc: &Value) -> Value {
3923 match tc {
3924 Value::String(s) => match s.as_str() {
3925 "auto" => serde_json::json!({ "type": "auto" }),
3926 "required" => serde_json::json!({ "type": "any" }),
3927 _ => serde_json::json!({ "type": "auto" }),
3929 },
3930 Value::Object(_) => {
3931 if tc.get("type").and_then(Value::as_str) == Some("function") {
3933 let name = tc.pointer("/function/name").cloned().unwrap_or(Value::Null);
3934 serde_json::json!({ "type": "tool", "name": name })
3935 } else {
3936 serde_json::json!({ "type": "auto" })
3937 }
3938 }
3939 _ => serde_json::json!({ "type": "auto" }),
3940 }
3941}
3942
3943pub fn parse_openai_request(body: &[u8], carry_raw: bool) -> Option<ModelRequest> {
3956 let json: Value = serde_json::from_slice(body).ok()?;
3957 let raw = if carry_raw { json.clone() } else { Value::Null };
3958
3959 let messages_json = json.get("messages")?.as_array()?;
3960
3961 let mut system: Option<String> = None;
3962 let mut messages: Vec<ChatMessage> = Vec::with_capacity(messages_json.len());
3963 let mut tools = Value::Null;
3964 let mut tool_choice_override: Option<Value> = None;
3965
3966 for msg in messages_json {
3967 let role = msg.get("role").and_then(Value::as_str).unwrap_or("user");
3968 match role {
3969 "system" => {
3970 if let Some(s) = msg.get("content").and_then(Value::as_str) {
3975 system = Some(s.to_owned());
3976 }
3977 }
3978 "user" => {
3979 let content_val = msg.get("content").unwrap_or(&Value::Null);
3980 let translated = translate_openai_user_content(content_val)?;
3981 messages.push(ChatMessage {
3982 role: "user".to_owned(),
3983 content: translated,
3984 });
3985 }
3986 "assistant" => {
3987 if let Some(tc_arr) = msg.get("tool_calls").and_then(Value::as_array) {
3988 let mut blocks: Vec<Value> = Vec::new();
3990 if let Some(text) = msg.get("content").and_then(Value::as_str)
3992 && !text.is_empty()
3993 {
3994 blocks.push(serde_json::json!({ "type": "text", "text": text }));
3995 }
3996 for tc in tc_arr {
3997 let id = tc.get("id").and_then(Value::as_str).unwrap_or("");
3998 let func = tc.get("function").unwrap_or(&Value::Null);
3999 let name = func.get("name").and_then(Value::as_str).unwrap_or("");
4000 let args_str = func
4001 .get("arguments")
4002 .and_then(Value::as_str)
4003 .unwrap_or("{}");
4004 let input: Value = serde_json::from_str(args_str)
4005 .unwrap_or_else(|_| serde_json::json!({}));
4006 blocks.push(serde_json::json!({
4007 "type": "tool_use",
4008 "id": id,
4009 "name": name,
4010 "input": input,
4011 }));
4012 }
4013 messages.push(ChatMessage {
4014 role: "assistant".to_owned(),
4015 content: Value::Array(blocks),
4016 });
4017 } else {
4018 let content = msg
4020 .get("content")
4021 .cloned()
4022 .unwrap_or_else(|| Value::String(String::new()));
4023 messages.push(ChatMessage {
4024 role: "assistant".to_owned(),
4025 content,
4026 });
4027 }
4028 }
4029 "tool" => {
4030 let tool_call_id = msg
4032 .get("tool_call_id")
4033 .and_then(Value::as_str)
4034 .unwrap_or("");
4035 let content = msg
4036 .get("content")
4037 .cloned()
4038 .unwrap_or_else(|| Value::String(String::new()));
4039 let result_block = serde_json::json!({
4040 "type": "tool_result",
4041 "tool_use_id": tool_call_id,
4042 "content": content,
4043 });
4044 messages.push(ChatMessage {
4045 role: "user".to_owned(),
4046 content: Value::Array(vec![result_block]),
4047 });
4048 }
4049 _ => {} }
4051 }
4052
4053 if !carry_raw {
4055 if let Some(t) = json.get("tools") {
4056 tools = translate_openai_tools(t);
4057 }
4058 if let Some(tc) = json.get("tool_choice") {
4059 tool_choice_override = Some(translate_openai_tool_choice(tc));
4060 }
4061 } else {
4062 tools = json.get("tools").cloned().unwrap_or(Value::Null);
4063 }
4064
4065 let max_tokens = json
4066 .get("max_tokens")
4067 .and_then(Value::as_u64)
4068 .and_then(|n| u32::try_from(n).ok())
4069 .unwrap_or(1024);
4070
4071 let model = json
4072 .get("model")
4073 .and_then(Value::as_str)
4074 .unwrap_or_default()
4075 .to_owned();
4076
4077 let _ = tool_choice_override; Some(ModelRequest {
4088 model,
4089 system,
4090 messages,
4091 max_tokens,
4092 tools,
4093 raw,
4094 cache_prefix: false,
4095 })
4096}
4097
4098fn extract_openai_content_and_tools(raw: &Value, text: &str) -> (Value, Option<Value>) {
4105 if let Some(blocks) = raw.get("content").and_then(Value::as_array) {
4107 let mut text_parts: Vec<&str> = Vec::new();
4108 let mut tool_calls: Vec<Value> = Vec::new();
4109 for block in blocks {
4110 match block.get("type").and_then(Value::as_str) {
4111 Some("text") => {
4112 if let Some(t) = block.get("text").and_then(Value::as_str) {
4113 text_parts.push(t);
4114 }
4115 }
4116 Some("tool_use") => {
4117 let id = block.get("id").and_then(Value::as_str).unwrap_or("");
4118 let name = block.get("name").and_then(Value::as_str).unwrap_or("");
4119 let input_str = block
4120 .get("input")
4121 .map_or_else(|| "{}".to_owned(), std::string::ToString::to_string);
4122 tool_calls.push(serde_json::json!({
4123 "id": id,
4124 "type": "function",
4125 "function": { "name": name, "arguments": input_str },
4126 }));
4127 }
4128 _ => {}
4129 }
4130 }
4131 let content_text = if tool_calls.is_empty() || !text_parts.is_empty() {
4132 Value::String(text_parts.join(""))
4133 } else {
4134 Value::Null };
4136 let tc = if tool_calls.is_empty() {
4137 None
4138 } else {
4139 Some(Value::Array(tool_calls))
4140 };
4141 return (content_text, tc);
4142 }
4143
4144 if let Some(msg) = raw.pointer("/choices/0/message") {
4146 let content = msg
4147 .get("content")
4148 .cloned()
4149 .unwrap_or(Value::String(text.to_owned()));
4150 let tc = msg.get("tool_calls").cloned();
4151 return (content, tc);
4152 }
4153
4154 (Value::String(text.to_owned()), None)
4156}
4157
4158fn openai_response_json(resp: &ModelResponse) -> Value {
4161 let (content_text, tool_calls) = extract_openai_content_and_tools(&resp.raw, &resp.text);
4162 let finish_reason = if tool_calls.is_some() {
4163 "tool_calls"
4164 } else {
4165 "stop"
4166 };
4167 let mut message = serde_json::json!({
4168 "role": "assistant",
4169 "content": content_text,
4170 });
4171 if let Some(tc) = tool_calls {
4172 message["tool_calls"] = tc;
4173 }
4174 serde_json::json!({
4175 "id": format!("chatcmpl-{}", Uuid::now_v7()),
4176 "object": "chat.completion",
4177 "created": jiff::Timestamp::now().as_second(),
4178 "model": resp.model,
4179 "choices": [{
4180 "index": 0,
4181 "message": message,
4182 "finish_reason": finish_reason,
4183 }],
4184 "usage": {
4185 "prompt_tokens": resp.in_tokens,
4186 "completion_tokens": resp.out_tokens,
4187 "total_tokens": resp.in_tokens + resp.out_tokens,
4188 }
4189 })
4190}
4191
4192fn openai_sse_from_message(message: &Value) -> String {
4196 let id = message
4197 .get("id")
4198 .and_then(Value::as_str)
4199 .unwrap_or("chatcmpl-unknown")
4200 .to_owned();
4201 let created = message.get("created").cloned().unwrap_or(Value::from(0));
4202 let model = message
4203 .get("model")
4204 .and_then(Value::as_str)
4205 .unwrap_or("unknown");
4206
4207 let choices = message.get("choices").and_then(Value::as_array);
4208 let msg = choices
4209 .and_then(|c| c.first())
4210 .and_then(|c| c.get("message"));
4211 let content = msg
4212 .and_then(|m| m.get("content"))
4213 .cloned()
4214 .unwrap_or(Value::Null);
4215 let tool_calls = msg.and_then(|m| m.get("tool_calls")).cloned();
4216 let finish_reason = choices
4217 .and_then(|c| c.first())
4218 .and_then(|c| c.get("finish_reason"))
4219 .cloned()
4220 .unwrap_or_else(|| Value::String("stop".to_owned()));
4221
4222 let mut out = String::new();
4223 let chunk = |delta: Value| {
4224 serde_json::json!({
4225 "id": id,
4226 "object": "chat.completion.chunk",
4227 "created": created,
4228 "model": model,
4229 "choices": [{ "index": 0, "delta": delta, "finish_reason": Value::Null }]
4230 })
4231 };
4232
4233 let role_chunk = chunk(serde_json::json!({ "role": "assistant", "content": "" }));
4235 out.push_str("data: ");
4236 out.push_str(&role_chunk.to_string());
4237 out.push_str("\n\n");
4238
4239 if let Value::String(text) = &content
4241 && !text.is_empty()
4242 {
4243 let content_chunk = chunk(serde_json::json!({ "content": text }));
4244 out.push_str("data: ");
4245 out.push_str(&content_chunk.to_string());
4246 out.push_str("\n\n");
4247 }
4248
4249 if let Some(tc) = tool_calls {
4251 let tc_chunk = chunk(serde_json::json!({ "tool_calls": tc }));
4252 out.push_str("data: ");
4253 out.push_str(&tc_chunk.to_string());
4254 out.push_str("\n\n");
4255 }
4256
4257 let finish_chunk = serde_json::json!({
4259 "id": id,
4260 "object": "chat.completion.chunk",
4261 "created": created,
4262 "model": model,
4263 "choices": [{ "index": 0, "delta": {}, "finish_reason": finish_reason }]
4264 });
4265 out.push_str("data: ");
4266 out.push_str(&finish_chunk.to_string());
4267 out.push_str("\n\n");
4268
4269 out.push_str("data: [DONE]\n\n");
4270 out
4271}
4272
4273#[allow(clippy::too_many_arguments)]
4278async fn handle_enforce_openai(
4279 state: &AppState,
4280 headers: &HeaderMap,
4281 body: &Bytes,
4282 features: Features,
4283 route: &Route,
4284 route_ix: usize,
4287 session_header: Option<String>,
4288 tenant: String,
4289 routing_mode: RoutingMode,
4290) -> Response {
4291 if is_stream_request(body) {
4292 let (tx, rx) = tokio::sync::oneshot::channel::<Result<Value, ProxyError>>();
4293 let (state_c, headers_c, body_c, route_c) =
4294 (state.clone(), headers.clone(), body.clone(), route.clone());
4295 tokio::spawn(async move {
4296 let out = enforce_pipeline_openai(
4297 &state_c,
4298 &headers_c,
4299 &body_c,
4300 features,
4301 &route_c,
4302 route_ix,
4303 session_header,
4304 tenant,
4305 routing_mode,
4306 )
4307 .await;
4308 let _ = tx.send(out);
4309 });
4310 return sse_keepalive_response(rx, openai_sse_from_message);
4311 }
4312 match enforce_pipeline_openai(
4313 state,
4314 headers,
4315 body,
4316 features,
4317 route,
4318 route_ix,
4319 session_header,
4320 tenant,
4321 routing_mode,
4322 )
4323 .await
4324 {
4325 Ok(message) => (axum::http::StatusCode::OK, Json(message)).into_response(),
4326 Err(e) => e.into_response(),
4327 }
4328}
4329
4330async fn observe_passthrough_openai(
4336 state: AppState,
4337 headers: HeaderMap,
4338 body: Bytes,
4339 session_header: Option<String>,
4340 tenant: String,
4341) -> Response {
4342 observe_passthrough_openai_path(
4343 state,
4344 headers,
4345 body,
4346 session_header,
4347 tenant,
4348 crate::upstream::OPENAI_CHAT_PATH,
4349 )
4350 .await
4351}
4352
4353async fn observe_passthrough_openai_path(
4359 state: AppState,
4360 headers: HeaderMap,
4361 body: Bytes,
4362 session_header: Option<String>,
4363 tenant: String,
4364 path: &'static str,
4365) -> Response {
4366 if is_stream_request(&body) {
4367 return observe_stream_openai_path(state, headers, body, session_header, tenant, path)
4368 .await;
4369 }
4370 let start = Instant::now();
4371 let result = crate::upstream::forward_openai_path(
4372 &state.http,
4373 &state.config.upstream_openai,
4374 path,
4375 &headers,
4376 body.clone(),
4377 )
4378 .await;
4379 let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
4380 match result {
4381 Ok((status, resp_headers, resp_body)) => {
4382 spawn_trace(
4383 &state,
4384 body,
4385 Some(resp_body.clone()),
4386 latency_ms,
4387 session_header,
4388 tenant,
4389 );
4390 (status, resp_headers, resp_body).into_response()
4391 }
4392 Err(err) => {
4393 spawn_trace(&state, body, None, latency_ms, session_header, tenant);
4394 err.into_response()
4395 }
4396 }
4397}
4398
4399async fn observe_stream_openai_path(
4403 state: AppState,
4404 headers: HeaderMap,
4405 body: Bytes,
4406 session_header: Option<String>,
4407 tenant: String,
4408 path: &'static str,
4409) -> Response {
4410 let start = Instant::now();
4411 let result = crate::upstream::forward_openai_streaming_path(
4412 &state.http,
4413 &state.config.upstream_openai,
4414 path,
4415 &headers,
4416 body.clone(),
4417 )
4418 .await;
4419 let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
4420 match result {
4421 Ok((status, resp_headers, response)) => {
4422 spawn_stream_trace(&state, body, latency_ms, session_header, tenant);
4423 let stream_body = Body::from_stream(response.bytes_stream());
4424 (status, resp_headers, stream_body).into_response()
4425 }
4426 Err(err) => {
4427 spawn_trace(&state, body, None, latency_ms, session_header, tenant);
4428 err.into_response()
4429 }
4430 }
4431}
4432
4433async fn chat_completions(
4438 State(state): State<AppState>,
4439 Extension(TenantId(tenant)): Extension<TenantId>,
4440 headers: HeaderMap,
4441 body: Bytes,
4442) -> Response {
4443 let session_header = header_str(&headers, SESSION_HEADER);
4444
4445 if let Some(routing) = state.config.routing.as_ref() {
4446 let features = extract_openai_features(&headers, &body);
4447 if let Some(route) = routing
4448 .route_for(&features)
4449 .filter(|r| r.mode == Mode::Enforce && !r.ladder.is_empty())
4450 {
4451 let route_ix = routing
4453 .routes
4454 .iter()
4455 .position(|r| std::ptr::eq(r, route))
4456 .unwrap_or(0);
4457 let route = route.clone();
4458 let routing_mode = resolve_mode(&headers, &route, &state.config);
4460 if routing_mode == RoutingMode::Observe {
4462 return observe_passthrough_openai(state, headers, body, session_header, tenant)
4463 .await;
4464 }
4465 if enforce_can_handle_route(
4466 &features,
4467 &body,
4468 routing.escalation.enforce_structured,
4469 &route,
4470 &state.providers,
4471 Dialect::Openai,
4472 ) {
4473 return handle_enforce_openai(
4474 &state,
4475 &headers,
4476 &body,
4477 features,
4478 &route,
4479 route_ix,
4480 session_header,
4481 tenant,
4482 routing_mode,
4483 )
4484 .await;
4485 }
4486 tracing::info!(
4487 "enforce route matched but OpenAI structured request can't be routed faithfully (flag/ladder); serving via observe passthrough"
4488 );
4489 }
4490 }
4491 observe_passthrough_openai(state, headers, body, session_header, tenant).await
4492}
4493
4494async fn responses(
4503 State(state): State<AppState>,
4504 Extension(TenantId(tenant)): Extension<TenantId>,
4505 headers: HeaderMap,
4506 body: Bytes,
4507) -> Response {
4508 let session_header = header_str(&headers, SESSION_HEADER);
4509
4510 if is_stream_request(&body) {
4514 return observe_passthrough_openai_path(
4515 state,
4516 headers,
4517 body,
4518 session_header,
4519 tenant,
4520 crate::upstream::OPENAI_RESPONSES_PATH,
4521 )
4522 .await;
4523 }
4524
4525 let Ok(parsed) = serde_json::from_slice::<Value>(&body) else {
4526 return observe_passthrough_openai_path(
4527 state,
4528 headers,
4529 body,
4530 session_header,
4531 tenant,
4532 crate::upstream::OPENAI_RESPONSES_PATH,
4533 )
4534 .await;
4535 };
4536 if crate::responses::has_untranslatable_content(&parsed) {
4540 return observe_passthrough_openai_path(
4541 state,
4542 headers,
4543 body,
4544 session_header,
4545 tenant,
4546 crate::upstream::OPENAI_RESPONSES_PATH,
4547 )
4548 .await;
4549 }
4550 let chat_body = Bytes::from(crate::responses::request_to_chat(&parsed).to_string());
4551
4552 if let Some(routing) = state.config.routing.as_ref() {
4553 let features = extract_openai_features(&headers, &chat_body);
4554 if let Some(route) = routing
4555 .route_for(&features)
4556 .filter(|r| r.mode == Mode::Enforce && !r.ladder.is_empty())
4557 {
4558 let route_ix = routing
4559 .routes
4560 .iter()
4561 .position(|r| std::ptr::eq(r, route))
4562 .unwrap_or(0);
4563 let route = route.clone();
4564 let routing_mode = resolve_mode(&headers, &route, &state.config);
4565 if routing_mode != RoutingMode::Observe
4566 && enforce_can_handle_route(
4567 &features,
4568 &chat_body,
4569 routing.escalation.enforce_structured,
4570 &route,
4571 &state.providers,
4572 Dialect::Openai,
4573 )
4574 {
4575 return match enforce_pipeline_openai_as(
4576 &state,
4577 &headers,
4578 &chat_body,
4579 &body,
4580 "openai.responses",
4581 features,
4582 &route,
4583 route_ix,
4584 session_header,
4585 tenant,
4586 routing_mode,
4587 )
4588 .await
4589 {
4590 Ok(chat) => Json(crate::responses::response_from_chat(&chat)).into_response(),
4591 Err(e) => e.into_response(),
4592 };
4593 }
4594 }
4595 }
4596 observe_passthrough_openai_path(
4597 state,
4598 headers,
4599 body,
4600 session_header,
4601 tenant,
4602 crate::upstream::OPENAI_RESPONSES_PATH,
4603 )
4604 .await
4605}
4606
4607#[cfg(test)]
4608mod tests {
4609 use bytes::Bytes;
4610
4611 #[test]
4614 fn risk_certificate_follows_adr_0011_precedence() {
4615 use super::{ThresholdSource, risk_certificate};
4616 let c = firstpass_core::eprocess::Certification {
4617 threshold: 0.6,
4618 e_value: 30.0,
4619 certified_at_round: 200,
4620 };
4621 let r = risk_certificate(Some((c, 0.1, 0.05)), Some((0.4, 0.1)), Some(0.9)).unwrap();
4622 assert_eq!(r.source, ThresholdSource::Eprocess);
4623 assert_eq!(
4624 (r.threshold, r.e_value, r.delta, r.certified_at_round),
4625 (0.6, Some(30.0), Some(0.05), Some(200))
4626 );
4627
4628 let r = risk_certificate(None, Some((0.4, 0.1)), Some(0.9)).unwrap();
4629 assert_eq!((r.source, r.threshold), (ThresholdSource::Aci, 0.4));
4630 assert_eq!((r.alpha, r.e_value), (Some(0.1), None));
4631
4632 let r = risk_certificate(None, None, Some(0.9)).unwrap();
4633 assert_eq!(
4634 (r.source, r.threshold, r.alpha),
4635 (ThresholdSource::Fixed, 0.9, None)
4636 );
4637
4638 assert!(risk_certificate(None, None, None).is_none());
4639 }
4640
4641 use super::*;
4642
4643 #[test]
4647 fn anthropic_tool_errors_are_extracted() {
4648 let body = br#"{"messages":[
4649 {"role":"assistant","content":[{"type":"tool_use","name":"bash","input":{"cmd":"a"}}]},
4650 {"role":"user","content":[{"type":"tool_result","is_error":true,"content":"boom"}]},
4651 {"role":"assistant","content":[{"type":"tool_use","name":"bash","input":{"cmd":"b"}}]},
4652 {"role":"user","content":[{"type":"tool_result","is_error":true,"content":"boom"}]}
4653 ]}"#;
4654 let s = trajectory_signals(body);
4655 assert_eq!(
4656 s.tool_errors, 2,
4657 "both failing tool_results must be counted"
4658 );
4659 assert_eq!(s.tool_results, 2);
4660 assert_eq!(s.assistant_turns, 2);
4661 assert!(
4662 DifficultyHint::score(s) >= DifficultyHint::Medium,
4663 "an all-failing window must not read as an easy session"
4664 );
4665 }
4666
4667 #[test]
4672 fn openai_tool_errors_are_extracted_from_a_different_shape() {
4673 let body = br#"{"messages":[
4674 {"role":"assistant","tool_calls":[{"function":{"name":"run","arguments":"{\"x\":1}"}}]},
4675 {"role":"tool","content":"Error: command not found"},
4676 {"role":"assistant","tool_calls":[{"function":{"name":"run","arguments":"{\"x\":2}"}}]},
4677 {"role":"tool","content":"Traceback (most recent call last): ..."}
4678 ]}"#;
4679 let s = trajectory_signals(body);
4680 assert_eq!(s.tool_results, 2, "role:tool messages are tool results");
4681 assert_eq!(
4682 s.tool_errors, 2,
4683 "both error-shaped tool outputs must count"
4684 );
4685 assert_eq!(s.assistant_turns, 2);
4686 }
4687
4688 #[test]
4695 fn a_flood_of_leading_whitespace_does_not_defeat_the_bound() {
4696 let flood = format!("{}Error: boom", " ".repeat(4_000_000));
4697 let small = format!("{}Error: boom", " ".repeat(400_000));
4698 let m_big = serde_json::json!({ "role": "tool", "content": flood });
4699 let m_small = serde_json::json!({ "role": "tool", "content": small });
4700
4701 let t = |m: &serde_json::Value| {
4702 let start = std::time::Instant::now();
4703 for _ in 0..20 {
4704 std::hint::black_box(openai_tool_content_looks_like_error(m));
4705 }
4706 start.elapsed()
4707 };
4708 let _warm = t(&m_small);
4709 let small_t = t(&m_small);
4710 let big_t = t(&m_big);
4711 assert!(
4712 big_t < small_t * 5 + std::time::Duration::from_millis(5),
4713 "10x the leading whitespace took {big_t:?} vs {small_t:?} — trimming is scanning the \
4714 whole payload before the bound applies"
4715 );
4716
4717 let indented = serde_json::json!({ "role": "tool", "content": " Error: boom" });
4719 assert!(
4720 openai_tool_content_looks_like_error(&indented),
4721 "ordinary leading whitespace must still be trimmed"
4722 );
4723 }
4724
4725 #[test]
4734 fn a_long_real_conversation_can_actually_reach_the_deep_signal() {
4735 let mut msgs = Vec::new();
4737 for i in 0..40 {
4738 msgs.push(format!(
4739 r#"{{"role":"assistant","content":[{{"type":"tool_use","name":"b","input":{{"i":{i}}}}}]}}"#
4740 ));
4741 msgs.push(
4742 r#"{"role":"user","content":[{"type":"tool_result","is_error":true}]}"#.to_owned(),
4743 );
4744 }
4745 let body = format!(r#"{{"messages":[{}]}}"#, msgs.join(","));
4746 let s = trajectory_signals(body.as_bytes());
4747 assert!(
4748 s.assistant_turns >= 8,
4749 "a 40-turn failing conversation must register as deep, got {} assistant turns — the \
4750 window is counting turns it cannot see",
4751 s.assistant_turns
4752 );
4753 assert_eq!(
4754 DifficultyHint::score(s),
4755 DifficultyHint::High,
4756 "the hardest possible conversation must reach the top bucket, or High is dead code"
4757 );
4758 }
4759
4760 #[test]
4767 fn openai_array_form_tool_content_is_read() {
4768 let arr = serde_json::json!({
4769 "role": "tool",
4770 "content": [{"type": "text", "text": "Error: command not found"}]
4771 });
4772 assert!(
4773 openai_tool_content_looks_like_error(&arr),
4774 "an error in array-form content must be detected"
4775 );
4776
4777 let clean = serde_json::json!({
4778 "role": "tool",
4779 "content": [{"type": "text", "text": "All 42 tests passed"}]
4780 });
4781 assert!(
4782 !openai_tool_content_looks_like_error(&clean),
4783 "clean array-form content must not be a failure"
4784 );
4785
4786 for weird in [
4788 serde_json::json!({"role": "tool", "content": []}),
4789 serde_json::json!({"role": "tool", "content": [{"type": "image"}]}),
4790 serde_json::json!({"role": "tool", "content": [null, 42]}),
4791 ] {
4792 assert!(!openai_tool_content_looks_like_error(&weird));
4793 }
4794
4795 let body = br#"{"messages":[
4797 {"role":"tool","content":[{"type":"text","text":"Error: boom"}]},
4798 {"role":"tool","content":[{"type":"text","text":"Error: boom again"}]},
4799 {"role":"tool","content":[{"type":"text","text":"Error: still broken"}]}
4800 ]}"#;
4801 let s = trajectory_signals(body);
4802 assert_eq!(s.tool_errors, 3, "array-form errors must reach the signals");
4803 assert!(DifficultyHint::score(s) >= DifficultyHint::Medium);
4804 }
4805
4806 #[test]
4812 fn tool_output_merely_mentioning_errors_is_not_counted_as_failure() {
4813 let body = br#"{"messages":[
4814 {"role":"tool","content":"0 errors, 0 warnings"},
4815 {"role":"tool","content":"All 42 tests passed (no failures)"},
4816 {"role":"tool","content":"lint: checked 12 files for errors"}
4817 ]}"#;
4818 let s = trajectory_signals(body);
4819 assert_eq!(s.tool_results, 3);
4820 assert_eq!(
4821 s.tool_errors, 0,
4822 "clean output that mentions the word 'error' must not be scored as a failure"
4823 );
4824 }
4825
4826 #[test]
4836 fn a_multibyte_character_does_not_defeat_the_prefix_bound() {
4837 let padded = format!("{}é{}", "x".repeat(63), "y".repeat(200_000));
4839 let msg = serde_json::json!({ "role": "tool", "content": padded });
4840 assert!(
4841 !openai_tool_content_looks_like_error(&msg),
4842 "a huge non-error payload must not be classified as a failure"
4843 );
4844
4845 let big = format!("{}é{}", "x".repeat(63), "y".repeat(2_000_000));
4853 let small = format!("{}é{}", "x".repeat(63), "y".repeat(200_000));
4854 let m_big = serde_json::json!({ "role": "tool", "content": big });
4855 let m_small = serde_json::json!({ "role": "tool", "content": small });
4856
4857 let t = |m: &serde_json::Value| {
4858 let start = std::time::Instant::now();
4859 for _ in 0..20 {
4860 std::hint::black_box(openai_tool_content_looks_like_error(m));
4861 }
4862 start.elapsed()
4863 };
4864 let _warm = t(&m_small);
4865 let small_t = t(&m_small);
4866 let big_t = t(&m_big);
4867 assert!(
4868 big_t < small_t * 5 + std::time::Duration::from_millis(5),
4869 "scanning a 10x larger payload took {big_t:?} vs {small_t:?} — the prefix bound is not \
4870 holding, so work scales with attacker-controlled input size"
4871 );
4872
4873 let err = format!("Error: boom 💥{}", "z".repeat(200_000));
4875 let msg2 = serde_json::json!({ "role": "tool", "content": err });
4876 assert!(
4877 openai_tool_content_looks_like_error(&msg2),
4878 "bounding the scan must not cost the detection it exists to perform"
4879 );
4880
4881 let late = format!("{}Error: too late", "w".repeat(200));
4884 let msg3 = serde_json::json!({ "role": "tool", "content": late });
4885 assert!(
4886 !openai_tool_content_looks_like_error(&msg3),
4887 "the scan must stay bounded to the head of the message"
4888 );
4889 }
4890
4891 #[test]
4894 fn identical_repeated_tool_calls_are_detected() {
4895 let body = br#"{"messages":[
4896 {"role":"assistant","content":[{"type":"tool_use","name":"ls","input":{"p":"/x"}}]},
4897 {"role":"assistant","content":[{"type":"tool_use","name":"ls","input":{"p":"/x"}}]},
4898 {"role":"assistant","content":[{"type":"tool_use","name":"ls","input":{"p":"/x"}}]},
4899 {"role":"user","content":[{"type":"tool_result","content":"ok"}]}
4900 ]}"#;
4901 let s = trajectory_signals(body);
4902 assert_eq!(
4903 s.repeated_tool_calls, 2,
4904 "three identical calls are two repeats"
4905 );
4906 let distinct = br#"{"messages":[
4908 {"role":"assistant","content":[{"type":"tool_use","name":"ls","input":{"p":"/x"}}]},
4909 {"role":"assistant","content":[{"type":"tool_use","name":"ls","input":{"p":"/y"}}]}
4910 ]}"#;
4911 assert_eq!(trajectory_signals(distinct).repeated_tool_calls, 0);
4912 }
4913
4914 #[test]
4917 fn malformed_bodies_yield_no_signal_and_never_panic() {
4918 for body in [
4919 &b""[..],
4920 b"not json at all",
4921 b"{}",
4922 br#"{"messages":"not-an-array"}"#,
4923 br#"{"messages":[{"role":"assistant","content":"plain text"}]}"#,
4924 br#"{"messages":[null,42,"x"]}"#,
4925 ] {
4926 let s = trajectory_signals(body);
4927 assert_eq!(
4928 DifficultyHint::score(s),
4929 DifficultyHint::None,
4930 "a body with no usable trajectory must score None, not a difficulty level"
4931 );
4932 }
4933
4934 let quiet_success = br#"{"messages":[{"content":[{"type":"tool_result"}]}]}"#;
4940 let s = trajectory_signals(quiet_success);
4941 assert_eq!(s.tool_results, 1);
4942 assert_eq!(s.tool_errors, 0);
4943 assert_eq!(
4944 DifficultyHint::score(s),
4945 DifficultyHint::Low,
4946 "a tool result that reported no error is evidence of health, not absence of evidence"
4947 );
4948 }
4949
4950 #[test]
4954 fn only_the_recent_window_counts_so_old_failures_do_not_ratchet() {
4955 let mut msgs = String::from("{\"messages\":[");
4956 for _ in 0..30 {
4958 msgs.push_str(r#"{"role":"user","content":[{"type":"tool_result","is_error":true}]},"#);
4959 }
4960 for i in 0..12 {
4962 msgs.push_str(&format!(
4963 r#"{{"role":"user","content":[{{"type":"tool_result","is_error":false,"content":"ok{i}"}}]}}{}"#,
4964 if i == 11 { "" } else { "," }
4965 ));
4966 }
4967 msgs.push_str("]}");
4968 let s = trajectory_signals(msgs.as_bytes());
4969 assert_eq!(
4970 s.tool_errors, 0,
4971 "failures outside the window must not count; a recovered session is not a hard one"
4972 );
4973 assert!(s.tool_results > 0, "the recent clean results must be seen");
4974 }
4975
4976 #[test]
4979 fn the_hint_reaches_the_feature_vector_for_both_dialects() {
4980 let hard = br#"{"messages":[
4981 {"role":"assistant","content":[{"type":"tool_use","name":"b","input":{"c":1}}]},
4982 {"role":"user","content":[{"type":"tool_result","is_error":true}]},
4983 {"role":"assistant","content":[{"type":"tool_use","name":"b","input":{"c":1}}]},
4984 {"role":"user","content":[{"type":"tool_result","is_error":true}]},
4985 {"role":"assistant","content":[{"type":"tool_use","name":"b","input":{"c":1}}]},
4986 {"role":"user","content":[{"type":"tool_result","is_error":true}]}
4987 ]}"#;
4988 let h = HeaderMap::new();
4989 assert!(
4990 extract_features(&h, hard).difficulty_hint > 0,
4991 "a visibly struggling Anthropic conversation must carry a non-zero hint"
4992 );
4993 assert!(
4994 extract_openai_features(&h, hard).difficulty_hint > 0,
4995 "the OpenAI extractor must populate the hint too, not silently leave it zero"
4996 );
4997 let plain = br#"{"messages":[{"role":"user","content":"hello"}]}"#;
4998 assert_eq!(
4999 extract_features(&h, plain).difficulty_hint,
5000 0,
5001 "a single-shot request must be unchanged by this feature"
5002 );
5003 }
5004
5005 fn test_config() -> ProxyConfig {
5006 ProxyConfig::from_lookup(|_| None).unwrap()
5007 }
5008
5009 #[test]
5010 fn build_trace_maps_request_and_response_fields() {
5011 let config = test_config();
5012 let req = Bytes::from_static(
5013 br#"{"model":"claude-haiku-4-5","tools":[{"name":"a"}],"messages":[]}"#,
5014 );
5015 let resp = Bytes::from_static(
5016 br#"{"model":"claude-haiku-4-5","usage":{"input_tokens":1200,"output_tokens":300}}"#,
5017 );
5018
5019 let trace = build_trace(&config, &req, &resp, 42, Some("sess-1"));
5020
5021 assert_eq!(trace.request.api, "anthropic.messages");
5022 assert_eq!(trace.session_id, "sess-1");
5023 assert_eq!(trace.attempts.len(), 1);
5024 let attempt = &trace.attempts[0];
5025 assert_eq!(attempt.model, "claude-haiku-4-5");
5026 assert_eq!(attempt.provider, "anthropic");
5027 assert_eq!(attempt.in_tokens, 1200);
5028 assert_eq!(attempt.out_tokens, 300);
5029 assert!(attempt.cost_usd > 0.0);
5030 assert_eq!(trace.request.features.tool_count, 1);
5031 assert!(!trace.request.features.has_images);
5032 assert_eq!(trace.final_.served_rung, Some(0));
5033 }
5034
5035 #[test]
5036 fn build_trace_falls_back_to_trace_id_session_when_header_absent() {
5037 let config = test_config();
5038 let req = Bytes::from_static(b"{}");
5039 let resp = Bytes::from_static(b"{}");
5040
5041 let trace = build_trace(&config, &req, &resp, 1, None);
5042
5043 assert_eq!(trace.session_id, trace.trace_id.to_string());
5044 }
5045
5046 #[test]
5047 fn build_error_trace_has_no_attempts_and_served_from_error() {
5048 let config = test_config();
5049 let req = Bytes::from_static(br#"{"model":"claude-haiku-4-5"}"#);
5050
5051 let trace = build_error_trace(&config, &req, 7, None);
5052
5053 assert!(trace.attempts.is_empty());
5054 assert_eq!(trace.final_.served_from, ServedFrom::Error);
5055 assert_eq!(trace.final_.served_rung, None);
5056 }
5057
5058 #[test]
5059 fn message_with_image_block_sets_has_images() {
5060 let req = br#"{"messages":[{"role":"user","content":[{"type":"image"}]}]}"#;
5061 let (_, _, has_images) = request_features(req);
5062 assert!(has_images);
5063 }
5064
5065 #[test]
5066 fn prompt_hash_never_contains_raw_prompt_text() {
5067 let hash = prompt_hash("salt", b"super secret prompt");
5068 assert!(!hash.contains("secret"));
5069 assert_eq!(hash.len(), 64);
5070 }
5071
5072 #[test]
5073 fn parse_model_request_preserves_content_verbatim_and_projects_text() {
5074 let body = br#"{"model":"m","system":"sys","max_tokens":50,
5075 "messages":[{"role":"user","content":[{"type":"text","text":"a"},{"type":"text","text":"b"}]},
5076 {"role":"assistant","content":"c"}]}"#;
5077 let req = parse_model_request(body).unwrap();
5078 assert_eq!(req.system.as_deref(), Some("sys"));
5079 assert_eq!(req.max_tokens, 50);
5080 assert_eq!(req.messages.len(), 2);
5081 assert_eq!(
5083 req.messages[0].content,
5084 serde_json::json!([{"type":"text","text":"a"},{"type":"text","text":"b"}])
5085 );
5086 assert_eq!(req.messages[1].content, Value::String("c".to_owned()));
5088 assert_eq!(req.messages[0].text_view(), "a\nb");
5090 assert_eq!(req.messages[1].text_view(), "c");
5091 }
5092
5093 #[test]
5094 fn tool_and_image_blocks_survive_the_request_round_trip() {
5095 let body = br#"{"model":"m","max_tokens":50,"messages":[
5097 {"role":"assistant","content":[{"type":"tool_use","id":"t1","name":"calc","input":{"x":1}}]},
5098 {"role":"user","content":[
5099 {"type":"tool_result","tool_use_id":"t1","content":"2"},
5100 {"type":"image","source":{"type":"base64","media_type":"image/png","data":"AA=="}}
5101 ]}]}"#;
5102 let req = parse_model_request(body).unwrap();
5103 let round_tripped = serde_json::to_value(&req.messages).unwrap();
5104 assert_eq!(
5105 round_tripped,
5106 serde_json::json!([
5107 {"role":"assistant","content":[{"type":"tool_use","id":"t1","name":"calc","input":{"x":1}}]},
5108 {"role":"user","content":[
5109 {"type":"tool_result","tool_use_id":"t1","content":"2"},
5110 {"type":"image","source":{"type":"base64","media_type":"image/png","data":"AA=="}}
5111 ]}
5112 ])
5113 );
5114 }
5115
5116 #[test]
5117 fn text_message_serializes_byte_identical_to_a_plain_string() {
5118 let m = ChatMessage::text("user", "hello");
5120 assert_eq!(
5121 serde_json::to_string(&m).unwrap(),
5122 r#"{"role":"user","content":"hello"}"#
5123 );
5124 }
5125
5126 #[test]
5127 fn parse_model_request_rejects_non_message_bodies() {
5128 assert!(parse_model_request(b"not json").is_none());
5129 assert!(parse_model_request(br#"{"no":"messages"}"#).is_none());
5130 }
5131
5132 use crate::provider::{MockProvider, ModelResponse, Provider, ProviderError, ProviderRegistry};
5135 use axum::extract::State;
5136 use std::collections::HashMap;
5137 use std::sync::Arc;
5138 use tokio::sync::mpsc;
5139
5140 fn model_resp(model: &str, text: &str) -> ModelResponse {
5141 ModelResponse {
5142 model: model.to_owned(),
5143 text: text.to_owned(),
5144 in_tokens: 1000,
5145 cache_write_tokens: 0,
5146 cache_read_tokens: 0,
5147 out_tokens: 400,
5148 raw: serde_json::Value::Null,
5149 }
5150 }
5151
5152 fn enforce_state(
5155 ladder: &[&str],
5156 gates: &[&str],
5157 outcomes: Vec<(&str, Result<ModelResponse, ProviderError>)>,
5158 ) -> (AppState, mpsc::Receiver<Trace>) {
5159 let toml = format!(
5160 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\nladder = [{}]\ngates = [{}]\n",
5161 ladder
5162 .iter()
5163 .map(|m| format!("\"{m}\""))
5164 .collect::<Vec<_>>()
5165 .join(", "),
5166 gates
5167 .iter()
5168 .map(|g| format!("\"{g}\""))
5169 .collect::<Vec<_>>()
5170 .join(", "),
5171 );
5172 let config = ProxyConfig::from_lookup(|k| match k {
5173 "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
5174 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
5175 _ => None,
5176 })
5177 .unwrap();
5178
5179 let mut outs = HashMap::new();
5180 for (model, out) in outcomes {
5181 outs.insert(model.to_owned(), out);
5182 }
5183 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
5184 map.insert(
5185 "anthropic".to_owned(),
5186 Arc::new(MockProvider::new("anthropic", outs)),
5187 );
5188 let providers = ProviderRegistry::from_map(map);
5189
5190 let (traces, rx) = mpsc::channel(64);
5191 let state = AppState {
5192 config: Arc::new(config),
5193 http: reqwest::Client::new(),
5194 providers,
5195 gate_health: Arc::new(GateHealthRegistry::new()),
5196 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
5197 paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
5198 experiments: Arc::default(),
5199 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
5200 traces,
5201 adaptive: None,
5202 eprocess: None,
5203 bandit: None,
5204 promoter: None,
5205 verified_cache: None,
5206 predictor: None,
5207 prior: None,
5208 tenant_rate_limiter: None,
5209 spill: None,
5210 };
5211 (state, rx)
5212 }
5213
5214 fn user_body() -> Bytes {
5215 Bytes::from_static(
5216 br#"{"model":"ignored","max_tokens":64,"messages":[{"role":"user","content":"hi"}]}"#,
5217 )
5218 }
5219
5220 async fn body_json(resp: Response) -> Value {
5221 let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
5222 .await
5223 .unwrap();
5224 serde_json::from_slice(&bytes).unwrap()
5225 }
5226
5227 #[tokio::test]
5228 async fn mode_profile_stamped_on_trace_when_non_balanced() {
5229 let (state, mut rx) = enforce_state(
5230 &["anthropic/claude-haiku-4-5"],
5231 &["non-empty"],
5232 vec![(
5233 "anthropic/claude-haiku-4-5",
5234 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
5235 )],
5236 );
5237 let mut headers = HeaderMap::new();
5238 headers.insert("x-firstpass-mode", "quality".parse().unwrap());
5239 let resp = messages(
5240 State(state),
5241 Extension(TenantId("default".to_owned())),
5242 headers,
5243 user_body(),
5244 )
5245 .await;
5246 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5247 let trace = rx.try_recv().expect("trace enqueued");
5248 assert_eq!(
5249 trace.policy.mode_profile.as_deref(),
5250 Some("quality"),
5251 "mode_profile must be stamped when quality mode is active"
5252 );
5253 }
5254
5255 #[tokio::test]
5256 async fn mode_profile_absent_from_trace_when_balanced() {
5257 let (state, mut rx) = enforce_state(
5258 &["anthropic/claude-haiku-4-5"],
5259 &["non-empty"],
5260 vec![(
5261 "anthropic/claude-haiku-4-5",
5262 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
5263 )],
5264 );
5265 let resp = messages(
5267 State(state),
5268 Extension(TenantId("default".to_owned())),
5269 HeaderMap::new(),
5270 user_body(),
5271 )
5272 .await;
5273 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5274 let trace = rx.try_recv().expect("trace enqueued");
5275 assert!(
5276 trace.policy.mode_profile.is_none(),
5277 "mode_profile must be absent when Balanced (byte-identical invariant)"
5278 );
5279 }
5280
5281 #[tokio::test]
5282 async fn enforce_serves_first_pass_and_returns_anthropic_shape() {
5283 let (state, mut rx) = enforce_state(
5284 &["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"],
5285 &["non-empty"],
5286 vec![(
5287 "anthropic/claude-haiku-4-5",
5288 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
5289 )],
5290 );
5291 let resp = messages(
5292 State(state),
5293 Extension(TenantId("default".to_owned())),
5294 HeaderMap::new(),
5295 user_body(),
5296 )
5297 .await;
5298 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5299 let json = body_json(resp).await;
5300 assert_eq!(json["type"], "message");
5301 assert_eq!(json["content"][0]["text"], "hello");
5302 assert_eq!(json["model"], "anthropic/claude-haiku-4-5");
5303
5304 let trace = rx.try_recv().expect("a trace was enqueued");
5305 assert_eq!(trace.mode, Mode::Enforce);
5306 assert_eq!(trace.final_.served_rung, Some(0));
5307 assert_eq!(trace.attempts.len(), 1);
5308 }
5309
5310 #[tokio::test]
5311 async fn enforce_escalates_then_serves_and_traces_two_attempts() {
5312 let (state, mut rx) = enforce_state(
5313 &["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"],
5314 &["non-empty"],
5315 vec![
5316 (
5317 "anthropic/claude-haiku-4-5",
5318 Ok(model_resp("anthropic/claude-haiku-4-5", " ")),
5319 ), (
5321 "anthropic/claude-sonnet-5",
5322 Ok(model_resp("anthropic/claude-sonnet-5", "answer")),
5323 ),
5324 ],
5325 );
5326 let resp = messages(
5327 State(state),
5328 Extension(TenantId("default".to_owned())),
5329 HeaderMap::new(),
5330 user_body(),
5331 )
5332 .await;
5333 let json = body_json(resp).await;
5334 assert_eq!(json["content"][0]["text"], "answer");
5335
5336 let trace = rx.try_recv().expect("trace enqueued");
5337 assert_eq!(trace.attempts.len(), 2);
5338 assert_eq!(trace.final_.escalations, 1);
5339 assert_eq!(trace.final_.served_rung, Some(1));
5340 }
5341
5342 #[tokio::test]
5343 async fn enforce_all_rungs_error_returns_502() {
5344 let (state, mut rx) = enforce_state(
5345 &["anthropic/claude-haiku-4-5"],
5346 &["non-empty"],
5347 vec![(
5348 "anthropic/claude-haiku-4-5",
5349 Err(ProviderError::Transport("down".into())),
5350 )],
5351 );
5352 let resp = messages(
5353 State(state),
5354 Extension(TenantId("default".to_owned())),
5355 HeaderMap::new(),
5356 user_body(),
5357 )
5358 .await;
5359 assert_eq!(resp.status(), axum::http::StatusCode::BAD_GATEWAY);
5360 assert!(rx.try_recv().is_ok());
5362 }
5363
5364 #[tokio::test]
5365 async fn no_routing_config_falls_through_to_observe_not_enforce() {
5366 let config = ProxyConfig::from_lookup(|k| match k {
5370 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
5371 _ => None,
5372 })
5373 .unwrap();
5374 let (traces, _rx) = mpsc::channel(64);
5375 let state = AppState {
5376 config: Arc::new(config),
5377 http: reqwest::Client::new(),
5378 providers: ProviderRegistry::new("http://127.0.0.1:1", "http://127.0.0.1:1"),
5379 gate_health: Arc::new(GateHealthRegistry::new()),
5380 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
5381 paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
5382 experiments: Arc::default(),
5383 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
5384 traces,
5385 adaptive: None,
5386 eprocess: None,
5387 bandit: None,
5388 promoter: None,
5389 verified_cache: None,
5390 predictor: None,
5391 prior: None,
5392 tenant_rate_limiter: None,
5393 spill: None,
5394 };
5395 let resp = messages(
5396 State(state),
5397 Extension(TenantId("default".to_owned())),
5398 HeaderMap::new(),
5399 user_body(),
5400 )
5401 .await;
5402 assert_ne!(resp.status(), axum::http::StatusCode::OK);
5404 }
5405
5406 fn bare_enforce_route() -> Route {
5409 use firstpass_core::config::{Match, Mode};
5410 Route {
5411 match_: Match::default(),
5412 mode: Mode::Enforce,
5413 ladder: vec!["anthropic/claude-haiku-4-5".to_owned()],
5414 gates: vec![],
5415 deferred_gates: vec![],
5416 routing_mode: None,
5417 rollout: None,
5418 shadow: None,
5419 challenger: None,
5420 }
5421 }
5422
5423 #[test]
5424 fn balanced_preset_application_is_noop() {
5425 let base_max_rungs = 3u32;
5427 let base_speculation = 2u32;
5428 let preset = RoutingMode::Balanced.preset();
5429 let max_rungs = if let Some(d) = preset.max_rungs_delta {
5430 (base_max_rungs as i32 + d).max(1) as u32
5431 } else {
5432 base_max_rungs
5433 };
5434 let speculation = preset.speculation.unwrap_or(base_speculation);
5435 let start_at_top = preset.start_at_top;
5436 assert_eq!(max_rungs, 3, "Balanced must not change max_rungs");
5437 assert_eq!(speculation, 2, "Balanced must not change speculation");
5438 assert!(!start_at_top, "Balanced must not set start_at_top");
5439 }
5440
5441 #[test]
5442 fn resolve_mode_header_wins_over_route_and_global() {
5443 let mut headers = HeaderMap::new();
5444 headers.insert("x-firstpass-mode", "cost".parse().unwrap());
5445 let mut route = bare_enforce_route();
5446 route.routing_mode = Some(RoutingMode::Quality); let mut config = test_config();
5448 config.default_routing_mode = RoutingMode::Max; assert_eq!(
5450 resolve_mode(&headers, &route, &config),
5451 RoutingMode::Cost,
5452 "header must win"
5453 );
5454 }
5455
5456 #[test]
5457 fn resolve_mode_route_wins_over_global_when_no_header() {
5458 let mut route = bare_enforce_route();
5459 route.routing_mode = Some(RoutingMode::Latency);
5460 let mut config = test_config();
5461 config.default_routing_mode = RoutingMode::Max;
5462 assert_eq!(
5463 resolve_mode(&HeaderMap::new(), &route, &config),
5464 RoutingMode::Latency,
5465 "route must beat global default"
5466 );
5467 }
5468
5469 #[test]
5470 fn resolve_mode_global_when_header_and_route_absent() {
5471 let mut config = test_config();
5472 config.default_routing_mode = RoutingMode::Quality;
5473 assert_eq!(
5474 resolve_mode(&HeaderMap::new(), &bare_enforce_route(), &config),
5475 RoutingMode::Quality
5476 );
5477 }
5478
5479 #[test]
5480 fn resolve_mode_unknown_header_falls_through_to_route() {
5481 let mut headers = HeaderMap::new();
5482 headers.insert("x-firstpass-mode", "turbo-mode".parse().unwrap());
5484 let mut route = bare_enforce_route();
5485 route.routing_mode = Some(RoutingMode::Cost);
5486 let config = test_config();
5487 assert_eq!(
5488 resolve_mode(&headers, &route, &config),
5489 RoutingMode::Cost,
5490 "unknown header value must fall through to route"
5491 );
5492 }
5493
5494 #[test]
5495 fn resolve_mode_header_case_insensitive() {
5496 let mut headers = HeaderMap::new();
5497 headers.insert("x-firstpass-mode", "QUALITY".parse().unwrap());
5498 assert_eq!(
5499 resolve_mode(&headers, &bare_enforce_route(), &test_config()),
5500 RoutingMode::Quality
5501 );
5502 }
5503
5504 #[test]
5505 fn resolve_mode_no_mode_set_returns_balanced() {
5506 assert_eq!(
5508 resolve_mode(&HeaderMap::new(), &bare_enforce_route(), &test_config()),
5509 RoutingMode::Balanced
5510 );
5511 }
5512
5513 #[test]
5514 fn capabilities_json_includes_routing_modes() {
5515 let modes: Vec<&'static str> = RoutingMode::ALL.iter().map(|m| m.as_str()).collect();
5516 assert!(modes.contains(&"balanced"));
5517 assert!(modes.contains(&"cost"));
5518 assert!(modes.contains(&"quality"));
5519 assert!(modes.contains(&"latency"));
5520 assert!(modes.contains(&"max"));
5521 assert!(modes.contains(&"observe"));
5522 }
5523
5524 #[test]
5525 fn model_list_dedups_across_ladders_keeping_cheapest_first_order() {
5526 let config = firstpass_core::config::Config::parse(
5529 r#"
5530 [[route]]
5531 mode = "enforce"
5532 ladder = ["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"]
5533 [[route]]
5534 mode = "enforce"
5535 ladder = ["anthropic/claude-sonnet-5", "anthropic/claude-opus-4-8"]
5536 "#,
5537 )
5538 .expect("test config parses");
5539
5540 let out = model_list(Some(&config));
5541
5542 let ids: Vec<&str> = out["data"]
5543 .as_array()
5544 .expect("data is an array")
5545 .iter()
5546 .map(|m| m["id"].as_str().expect("id is a string"))
5547 .collect();
5548 assert_eq!(
5549 ids,
5550 [
5551 "anthropic/claude-haiku-4-5",
5552 "anthropic/claude-sonnet-5",
5553 "anthropic/claude-opus-4-8",
5554 ],
5555 "sonnet appears once, and ladder order is preserved"
5556 );
5557 assert_eq!(out["object"], "list");
5558 assert_eq!(out["data"][0]["owned_by"], "anthropic");
5559 let haiku_in = out["data"][0]["firstpass"]["input_per_mtok"]
5562 .as_f64()
5563 .expect("a first-party rung has a price");
5564 assert!(haiku_in > 0.0, "expected a real price, got {haiku_in}");
5565 let opus_in = out["data"][2]["firstpass"]["input_per_mtok"]
5567 .as_f64()
5568 .expect("a first-party rung has a price");
5569 assert!(haiku_in < opus_in, "{haiku_in} should undercut {opus_in}");
5570 }
5571
5572 #[tokio::test]
5573 async fn healthz_names_the_service_so_a_client_can_tell_us_from_a_stranger() {
5574 let body = healthz().await.into_response();
5578 let bytes = axum::body::to_bytes(body.into_body(), 64 * 1024)
5579 .await
5580 .expect("healthz body");
5581 let json: serde_json::Value = serde_json::from_slice(&bytes).expect("healthz is json");
5582 assert_eq!(json["status"], "ok");
5583 assert_eq!(json["service"], "firstpass");
5584 }
5585
5586 #[test]
5587 fn model_list_is_empty_without_routing_config() {
5588 let out = model_list(None);
5589 assert_eq!(out["object"], "list");
5590 assert_eq!(
5591 out["data"].as_array().expect("data is an array").len(),
5592 0,
5593 "an observe-only deployment advertises no models"
5594 );
5595 }
5596
5597 #[test]
5598 fn detects_stream_requests() {
5599 assert!(is_stream_request(br#"{"stream": true}"#));
5600 assert!(!is_stream_request(br#"{"stream": false}"#));
5601 assert!(!is_stream_request(br#"{"model":"m"}"#));
5602 assert!(!is_stream_request(b"not json"));
5603 }
5604
5605 #[test]
5606 fn detects_tool_blocks_in_messages() {
5607 let with =
5608 br#"{"messages":[{"role":"user","content":[{"type":"tool_result","content":"42"}]}]}"#;
5609 let without = br#"{"messages":[{"role":"user","content":"hi"}]}"#;
5610 assert!(messages_have_tool_blocks(with));
5611 assert!(!messages_have_tool_blocks(without));
5612 }
5613
5614 #[test]
5615 fn enforce_only_handles_plain_text() {
5616 let plain =
5617 Bytes::from_static(br#"{"model":"m","messages":[{"role":"user","content":"hi"}]}"#);
5618 let tools = Bytes::from_static(
5619 br#"{"model":"m","tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
5620 );
5621 let f_plain = extract_features(&HeaderMap::new(), &plain);
5622 let f_tools = extract_features(&HeaderMap::new(), &tools);
5623 let anthropic_ladder = vec!["anthropic/claude-haiku-4-5".to_owned()];
5624 let providers = test_registry();
5625 assert!(enforce_can_handle(
5627 &f_plain,
5628 &plain,
5629 false,
5630 &anthropic_ladder,
5631 &providers,
5632 Dialect::Anthropic,
5633 ));
5634 assert!(!enforce_can_handle(
5635 &f_tools,
5636 &tools,
5637 false,
5638 &anthropic_ladder,
5639 &providers,
5640 Dialect::Anthropic,
5641 ));
5642 }
5643
5644 #[test]
5645 fn structured_enforce_routes_tools_and_streaming() {
5646 let tools = Bytes::from_static(
5649 br#"{"model":"m","tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
5650 );
5651 let streaming_tools = Bytes::from_static(
5652 br#"{"model":"m","stream":true,"tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
5653 );
5654 let f = extract_features(&HeaderMap::new(), &tools);
5655 let anthropic_ladder = vec![
5656 "anthropic/claude-haiku-4-5".to_owned(),
5657 "anthropic/claude-sonnet-5".to_owned(),
5658 ];
5659 let providers = test_registry();
5660 assert!(enforce_can_handle(
5661 &f,
5662 &tools,
5663 true,
5664 &anthropic_ladder,
5665 &providers,
5666 Dialect::Anthropic,
5667 ));
5668 assert!(enforce_can_handle(
5669 &f,
5670 &streaming_tools,
5671 true,
5672 &anthropic_ladder,
5673 &providers,
5674 Dialect::Anthropic,
5675 ));
5676 }
5677
5678 fn test_registry() -> crate::provider::ProviderRegistry {
5680 crate::provider::ProviderRegistry::new("http://localhost", "http://localhost")
5681 }
5682
5683 #[test]
5684 fn fidelity_guard_blocks_structured_on_non_verbatim_ladder() {
5685 let tools = Bytes::from_static(
5689 br#"{"model":"m","tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
5690 );
5691 let plain =
5692 Bytes::from_static(br#"{"model":"m","messages":[{"role":"user","content":"hi"}]}"#);
5693 let f_tools = extract_features(&HeaderMap::new(), &tools);
5694 let f_plain = extract_features(&HeaderMap::new(), &plain);
5695 let providers = test_registry();
5696 let mixed_ladder = vec![
5697 "openai/gpt-4.1-mini".to_owned(),
5698 "anthropic/claude-sonnet-5".to_owned(),
5699 ];
5700 assert!(!enforce_can_handle(
5701 &f_tools,
5702 &tools,
5703 true,
5704 &mixed_ladder,
5705 &providers,
5706 Dialect::Anthropic,
5707 ));
5708 assert!(enforce_can_handle(
5709 &f_plain,
5710 &plain,
5711 true,
5712 &mixed_ladder,
5713 &providers,
5714 Dialect::Anthropic,
5715 ));
5716 }
5717
5718 #[test]
5719 fn enforce_sse_reemission_preserves_text_and_tool_use() {
5720 let resp = ModelResponse {
5723 model: "anthropic/claude-haiku-4-5".to_owned(),
5724 text: "let me check".to_owned(),
5725 in_tokens: 5,
5726 cache_write_tokens: 0,
5727 cache_read_tokens: 0,
5728 out_tokens: 7,
5729 raw: serde_json::json!({
5730 "content": [
5731 { "type": "text", "text": "let me check" },
5732 { "type": "tool_use", "id": "tu_1", "name": "get_weather", "input": { "city": "Paris" } }
5733 ]
5734 }),
5735 };
5736 let sse = anthropic_sse_from_message(&anthropic_response_json(&resp));
5737
5738 let frames: Vec<Value> = sse
5740 .lines()
5741 .filter_map(|l| l.strip_prefix("data: "))
5742 .map(|d| serde_json::from_str::<Value>(d).expect("each SSE data frame is valid JSON"))
5743 .collect();
5744
5745 assert_eq!(frames.first().unwrap()["type"], "message_start");
5747 assert_eq!(frames.last().unwrap()["type"], "message_stop");
5748 assert!(frames.iter().any(|f| f["delta"]["type"] == "text_delta"
5750 && f["delta"]["text"] == "let me check"));
5751 assert!(
5754 frames
5755 .iter()
5756 .any(|f| f["content_block"]["type"] == "tool_use"
5757 && f["content_block"]["name"] == "get_weather"
5758 && f["content_block"]["id"] == "tu_1")
5759 );
5760 assert!(
5761 frames
5762 .iter()
5763 .any(|f| f["delta"]["type"] == "input_json_delta"
5764 && f["delta"]["partial_json"] == r#"{"city":"Paris"}"#)
5765 );
5766 }
5767
5768 #[tokio::test]
5773 async fn enforce_falls_back_to_observe_for_tool_requests() {
5774 let toml = "[[price]]\nmodel = \"anthropic/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/m\"]\ngates = [\"non-empty\"]\n";
5775 let config = ProxyConfig::from_lookup(|k| match k {
5776 "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
5777 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
5778 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
5779 _ => None,
5780 })
5781 .unwrap();
5782 let mut outs = HashMap::new();
5783 outs.insert(
5784 "anthropic/m".to_owned(),
5785 Ok(model_resp("anthropic/m", "hello")),
5786 );
5787 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
5788 map.insert(
5789 "anthropic".to_owned(),
5790 Arc::new(MockProvider::new("anthropic", outs)),
5791 );
5792 let (traces, _rx) = mpsc::channel(64);
5793 let state = AppState {
5794 config: Arc::new(config),
5795 http: reqwest::Client::new(),
5796 providers: ProviderRegistry::from_map(map),
5797 gate_health: Arc::new(GateHealthRegistry::new()),
5798 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
5799 paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
5800 experiments: Arc::default(),
5801 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
5802 traces,
5803 adaptive: None,
5804 eprocess: None,
5805 bandit: None,
5806 promoter: None,
5807 verified_cache: None,
5808 predictor: None,
5809 prior: None,
5810 tenant_rate_limiter: None,
5811 spill: None,
5812 };
5813
5814 let plain =
5816 Bytes::from_static(br#"{"model":"m","messages":[{"role":"user","content":"hi"}]}"#);
5817 let resp = messages(
5818 State(state.clone()),
5819 Extension(TenantId("default".to_owned())),
5820 HeaderMap::new(),
5821 plain,
5822 )
5823 .await;
5824 assert_eq!(
5825 resp.status(),
5826 axum::http::StatusCode::OK,
5827 "plain text should enforce"
5828 );
5829
5830 let tools = Bytes::from_static(
5833 br#"{"model":"m","tools":[{"name":"get_weather"}],"messages":[{"role":"user","content":"hi"}]}"#,
5834 );
5835 let resp = messages(
5836 State(state.clone()),
5837 Extension(TenantId("default".to_owned())),
5838 HeaderMap::new(),
5839 tools.clone(),
5840 )
5841 .await;
5842 assert_eq!(
5843 resp.status(),
5844 axum::http::StatusCode::OK,
5845 "tool request must route through enforce by default (ADR 0005 default-on)"
5846 );
5847
5848 let toml_off = "[[price]]\nmodel = \"anthropic/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/m\"]\ngates = [\"non-empty\"]\n[escalation]\nenforce_structured = false\n";
5851 let config_off = ProxyConfig::from_lookup(|k| match k {
5852 "FIRSTPASS_CONFIG_TOML" => Some(toml_off.to_owned()),
5853 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
5854 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
5855 _ => None,
5856 })
5857 .unwrap();
5858 let state_off = AppState {
5859 config: Arc::new(config_off),
5860 ..state.clone()
5861 };
5862 let resp = messages(
5863 State(state_off),
5864 Extension(TenantId("default".to_owned())),
5865 HeaderMap::new(),
5866 tools,
5867 )
5868 .await;
5869 assert_ne!(
5870 resp.status(),
5871 axum::http::StatusCode::OK,
5872 "with enforce_structured = false a tool request must fall back to observe"
5873 );
5874
5875 let toolres = Bytes::from_static(
5877 br#"{"model":"m","messages":[{"role":"user","content":[{"type":"tool_result","tool_use_id":"x","content":"42"}]}]}"#,
5878 );
5879 let resp = messages(
5880 State(state),
5881 Extension(TenantId("default".to_owned())),
5882 HeaderMap::new(),
5883 toolres,
5884 )
5885 .await;
5886 assert_eq!(
5887 resp.status(),
5888 axum::http::StatusCode::OK,
5889 "tool_result blocks route through enforce by default too (verbatim carry)"
5890 );
5891 }
5892
5893 #[tokio::test]
5897 async fn panel_is_public_markup_but_receipts_stay_behind_auth() {
5898 use tower::ServiceExt;
5899 let (state, _rx) = enforce_state(
5900 &["anthropic/claude-haiku-4-5"],
5901 &["non-empty"],
5902 vec![(
5903 "anthropic/claude-haiku-4-5",
5904 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
5905 )],
5906 );
5907 let router = app(state).expect("prometheus recorder installs");
5908
5909 let page = router
5910 .clone()
5911 .oneshot(
5912 axum::http::Request::builder()
5913 .uri("/panel")
5914 .body(Body::empty())
5915 .unwrap(),
5916 )
5917 .await
5918 .unwrap();
5919 assert_eq!(page.status(), axum::http::StatusCode::OK);
5920 let html = axum::body::to_bytes(page.into_body(), 1 << 20)
5921 .await
5922 .unwrap();
5923 let html = String::from_utf8_lossy(&html);
5924 assert!(
5925 html.contains("firstpass") && html.contains("/v1/receipts"),
5926 "the page must fetch its data rather than embed it"
5927 );
5928 assert!(
5931 !html.contains("http://") && !html.contains("https://"),
5932 "the panel must not reference any external origin"
5933 );
5934 assert!(html.contains("const esc ="), "the escaper must exist");
5939 for raw in [
5940 "${a.model}",
5941 "${a.rung}",
5942 "${a.verdict}",
5943 "${r.trace_id}",
5944 "${r.served_rung}",
5945 ] {
5946 assert!(
5947 !html.contains(raw),
5948 "receipt field interpolated unescaped into innerHTML: {raw}"
5949 );
5950 }
5951
5952 let data = router
5953 .oneshot(
5954 axum::http::Request::builder()
5955 .uri("/v1/receipts?limit=5")
5956 .body(Body::empty())
5957 .unwrap(),
5958 )
5959 .await
5960 .unwrap();
5961 assert_eq!(data.status(), axum::http::StatusCode::OK);
5965 let body = axum::body::to_bytes(data.into_body(), 1 << 20)
5966 .await
5967 .unwrap();
5968 let v: Value = serde_json::from_slice(&body).unwrap();
5969 assert!(v.get("receipts").and_then(Value::as_array).is_some());
5970 }
5971
5972 async fn feedback_state() -> (AppState, std::path::PathBuf, String) {
5976 feedback_state_with(false).await
5977 }
5978
5979 async fn feedback_state_with(served: bool) -> (AppState, std::path::PathBuf, String) {
5985 let db = std::env::temp_dir().join(format!("firstpass-feedback-{}.db", Uuid::now_v7()));
5986 let (tx, handle) = crate::store::open(&db).unwrap();
5987
5988 let mut trace = build_error_trace(
5989 &ProxyConfig::from_lookup(|_| None).unwrap(),
5990 &Bytes::from_static(b"{}"),
5991 5,
5992 Some("sess-fb"),
5993 );
5994 trace.attempts.push(Attempt {
5995 rung: 0,
5996 model: "anthropic/claude-haiku-4-5".into(),
5997 provider: "anthropic".into(),
5998 in_tokens: 10,
5999 cache_write_tokens: 0,
6000 cache_read_tokens: 0,
6001 out_tokens: 5,
6002 cost_usd: 0.001,
6003 latency_ms: 5,
6004 gates: vec![],
6005 verdict: Verdict::Pass,
6006 truncated: false,
6007 });
6008 let trace_id = trace.trace_id.to_string();
6009 if served {
6010 trace.final_.served_rung = Some(0);
6011 }
6012 tx.try_send(trace).unwrap();
6013 drop(tx);
6014 handle.await.unwrap();
6015
6016 let db_str = db.to_string_lossy().into_owned();
6017 let config = ProxyConfig::from_lookup(move |k| match k {
6018 "FIRSTPASS_DB" => Some(db_str.clone()),
6019 _ => None,
6020 })
6021 .unwrap();
6022 let (traces, _rx) = mpsc::channel(64);
6023 let state = AppState {
6024 config: Arc::new(config),
6025 http: reqwest::Client::new(),
6026 providers: ProviderRegistry::new("http://127.0.0.1:1", "http://127.0.0.1:1"),
6027 gate_health: Arc::new(GateHealthRegistry::new()),
6028 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
6029 paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
6030 experiments: Arc::default(),
6031 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
6032 traces,
6033 adaptive: None,
6034 eprocess: None,
6035 bandit: None,
6036 promoter: None,
6037 verified_cache: None,
6038 predictor: None,
6039 prior: None,
6040 tenant_rate_limiter: None,
6041 spill: None,
6042 };
6043 (state, db, trace_id)
6044 }
6045
6046 #[tokio::test]
6060 async fn feedback_without_a_score_still_advances_the_eprocess() {
6061 use firstpass_core::eprocess::{DEFAULT_BET, EProcessRiskControl};
6062 let (mut state, _db, trace_id) = feedback_state_with(true).await;
6063 let grid: Vec<f64> = (0..=20).map(|i| f64::from(i) / 20.0).collect();
6064 let ep = Arc::new(std::sync::Mutex::new(EProcessRiskControl::new(
6065 0.2,
6066 0.05,
6067 DEFAULT_BET,
6068 &grid,
6069 )));
6070 state.eprocess = Some(ep.clone());
6071 assert_eq!(ep.lock().unwrap().rounds(), 0);
6072
6073 let body = Bytes::from(
6075 serde_json::json!({
6076 "trace_id": trace_id, "gate_id": "tests", "verdict": "pass", "reporter": "ci"
6077 })
6078 .to_string(),
6079 );
6080 let resp = feedback(
6081 State(state.clone()),
6082 Extension(TenantId("default".to_owned())),
6083 body,
6084 )
6085 .await
6086 .into_response();
6087 assert_eq!(resp.status(), axum::http::StatusCode::ACCEPTED);
6089
6090 assert_eq!(
6091 ep.lock().unwrap().rounds(),
6092 1,
6093 "a scoreless feedback call must still advance the controller — keying on the optional \
6094 payload score left it permanently inert"
6095 );
6096 }
6097
6098 #[tokio::test]
6099 async fn feedback_nudges_the_adaptive_threshold() {
6100 use firstpass_core::conformal::AdaptiveConformal;
6101 let (mut state, _db, trace_id) = feedback_state().await;
6102 let aci = Arc::new(std::sync::Mutex::new(AdaptiveConformal::new(0.1, 0.2, 0.5)));
6103 state.adaptive = Some(aci.clone());
6104 let before = aci.lock().unwrap().threshold();
6105
6106 let fail = Bytes::from(
6108 serde_json::json!({ "trace_id": trace_id, "gate_id": "tests", "verdict": "fail", "reporter": "ci" })
6109 .to_string(),
6110 );
6111 assert_eq!(
6112 feedback(
6113 State(state.clone()),
6114 Extension(TenantId("default".to_owned())),
6115 fail
6116 )
6117 .await
6118 .status(),
6119 axum::http::StatusCode::ACCEPTED
6120 );
6121 let after_fail = aci.lock().unwrap().threshold();
6122 assert!(
6123 after_fail > before,
6124 "served fail should raise the live threshold: {before} -> {after_fail}"
6125 );
6126
6127 let pass = Bytes::from(
6129 serde_json::json!({ "trace_id": trace_id, "gate_id": "tests", "verdict": "pass", "reporter": "ci" })
6130 .to_string(),
6131 );
6132 let _ = feedback(
6133 State(state),
6134 Extension(TenantId("default".to_owned())),
6135 pass,
6136 )
6137 .await;
6138 assert!(aci.lock().unwrap().threshold() < after_fail);
6139 }
6140
6141 #[tokio::test]
6142 async fn feedback_records_a_deferred_verdict_without_breaking_the_chain() {
6143 let (state, db, trace_id) = feedback_state().await;
6144 let body = Bytes::from(
6145 serde_json::json!({
6146 "trace_id": trace_id,
6147 "gate_id": "tests",
6148 "verdict": "pass",
6149 "score": 1.0,
6150 "reporter": "ci",
6151 })
6152 .to_string(),
6153 );
6154 let resp = feedback(
6155 State(state),
6156 Extension(TenantId("default".to_owned())),
6157 body,
6158 )
6159 .await;
6160 assert_eq!(resp.status(), axum::http::StatusCode::ACCEPTED);
6161
6162 let view = crate::store::load_trace_view(&db, "default", &trace_id)
6164 .unwrap()
6165 .unwrap();
6166 assert_eq!(view.deferred.len(), 1);
6167 assert_eq!(view.deferred[0].gate_id, "tests");
6168 let traces = crate::store::load_all_traces(&db).unwrap();
6170 firstpass_core::verify_chain(&traces, GENESIS_HASH).unwrap();
6171
6172 let _ = std::fs::remove_file(&db);
6173 }
6174
6175 #[tokio::test]
6176 async fn feedback_for_unknown_trace_is_404() {
6177 let (state, db, _trace_id) = feedback_state().await;
6178 let body = Bytes::from(
6179 serde_json::json!({
6180 "trace_id": "does-not-exist",
6181 "gate_id": "tests",
6182 "verdict": "pass",
6183 "reporter": "ci",
6184 })
6185 .to_string(),
6186 );
6187 let resp = feedback(
6188 State(state),
6189 Extension(TenantId("default".to_owned())),
6190 body,
6191 )
6192 .await;
6193 assert_eq!(resp.status(), axum::http::StatusCode::NOT_FOUND);
6194 let _ = std::fs::remove_file(&db);
6195 }
6196
6197 #[tokio::test]
6200 async fn feedback_across_tenants_is_404_not_403() {
6201 let (state, db, trace_id) = feedback_state().await;
6202 let body = Bytes::from(
6203 serde_json::json!({
6204 "trace_id": trace_id,
6205 "gate_id": "tests",
6206 "verdict": "pass",
6207 "score": 1.0,
6208 "reporter": "attacker",
6209 })
6210 .to_string(),
6211 );
6212 let resp = feedback(
6214 State(state),
6215 Extension(TenantId("tenant-b".to_owned())),
6216 body,
6217 )
6218 .await;
6219 assert_eq!(
6220 resp.status(),
6221 axum::http::StatusCode::NOT_FOUND,
6222 "cross-tenant feedback must look exactly like a missing trace"
6223 );
6224 let _ = std::fs::remove_file(&db);
6225 }
6226
6227 #[tokio::test]
6228 async fn feedback_rejects_bad_verdict_and_score() {
6229 let (state, db, trace_id) = feedback_state().await;
6230 let bad_verdict = Bytes::from(
6231 serde_json::json!({ "trace_id": trace_id, "gate_id": "g", "verdict": "maybe", "reporter": "x" })
6232 .to_string(),
6233 );
6234 assert_eq!(
6235 feedback(
6236 State(state.clone()),
6237 Extension(TenantId("default".to_owned())),
6238 bad_verdict
6239 )
6240 .await
6241 .status(),
6242 axum::http::StatusCode::BAD_REQUEST
6243 );
6244 let bad_score = Bytes::from(
6245 serde_json::json!({ "trace_id": trace_id, "gate_id": "g", "verdict": "pass", "score": 9.0, "reporter": "x" })
6246 .to_string(),
6247 );
6248 assert_eq!(
6249 feedback(
6250 State(state),
6251 Extension(TenantId("default".to_owned())),
6252 bad_score
6253 )
6254 .await
6255 .status(),
6256 axum::http::StatusCode::BAD_REQUEST
6257 );
6258 let _ = std::fs::remove_file(&db);
6259 }
6260
6261 #[tokio::test]
6262 async fn metrics_endpoint_renders_after_a_real_request() {
6263 use tower::ServiceExt;
6264
6265 let (state, mut rx) = enforce_state(
6266 &["anthropic/claude-haiku-4-5"],
6267 &["non-empty"],
6268 vec![(
6269 "anthropic/claude-haiku-4-5",
6270 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
6271 )],
6272 );
6273 let router = app(state).expect("prometheus recorder installs");
6274
6275 let req = axum::http::Request::builder()
6276 .method("POST")
6277 .uri("/v1/messages")
6278 .header("content-type", "application/json")
6279 .body(Body::from(user_body()))
6280 .unwrap();
6281 let resp = router.clone().oneshot(req).await.unwrap();
6282 assert_eq!(resp.status(), axum::http::StatusCode::OK);
6283 rx.try_recv().expect("a trace was enqueued");
6284
6285 let metrics_req = axum::http::Request::builder()
6286 .method("GET")
6287 .uri("/metrics")
6288 .body(Body::empty())
6289 .unwrap();
6290 let metrics_resp = router.oneshot(metrics_req).await.unwrap();
6291 assert_eq!(metrics_resp.status(), axum::http::StatusCode::OK);
6292 let bytes = axum::body::to_bytes(metrics_resp.into_body(), 1 << 20)
6293 .await
6294 .unwrap();
6295 let body = String::from_utf8(bytes.to_vec()).unwrap();
6296 assert!(
6297 body.contains("firstpass_enforce_latency_ms"),
6298 "metrics body missing enforce latency histogram: {body}"
6299 );
6300 assert!(
6301 body.contains("firstpass_served_total"),
6302 "metrics body missing served counter: {body}"
6303 );
6304 }
6305
6306 #[tokio::test]
6312 async fn metrics_are_dimensioned_by_provider_rung_and_gate() {
6313 use tower::ServiceExt;
6314
6315 let (state, mut rx) = enforce_state(
6316 &["anthropic/claude-haiku-4-5"],
6317 &["non-empty"],
6318 vec![(
6319 "anthropic/claude-haiku-4-5",
6320 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
6321 )],
6322 );
6323 let router = app(state).expect("prometheus recorder installs");
6324
6325 let req = axum::http::Request::builder()
6326 .method("POST")
6327 .uri("/v1/messages")
6328 .header("content-type", "application/json")
6329 .body(Body::from(user_body()))
6330 .unwrap();
6331 assert_eq!(
6332 router.clone().oneshot(req).await.unwrap().status(),
6333 axum::http::StatusCode::OK
6334 );
6335 rx.try_recv().expect("a trace was enqueued");
6336
6337 let metrics_req = axum::http::Request::builder()
6338 .method("GET")
6339 .uri("/metrics")
6340 .body(Body::empty())
6341 .unwrap();
6342 let bytes = axum::body::to_bytes(
6343 router.oneshot(metrics_req).await.unwrap().into_body(),
6344 1 << 20,
6345 )
6346 .await
6347 .unwrap();
6348 let body = String::from_utf8(bytes.to_vec()).unwrap();
6349
6350 assert!(
6352 body.contains(r#"firstpass_attempt_total{provider="anthropic",rung="0"}"#),
6353 "attempt counter is not dimensioned by provider+rung: {body}"
6354 );
6355 assert!(
6356 body.contains("firstpass_attempt_latency_ms")
6357 && body.contains(r#"provider="anthropic""#),
6358 "attempt latency is not dimensioned by provider: {body}"
6359 );
6360 assert!(
6361 body.contains("firstpass_attempt_cost_usd_total"),
6362 "per-attempt cost series missing: {body}"
6363 );
6364 assert!(
6366 body.contains(r#"firstpass_gate_verdict_total{gate_id="non-empty",verdict="pass"}"#),
6367 "gate verdicts are not dimensioned by gate_id+verdict: {body}"
6368 );
6369 }
6370
6371 fn auth_state(require_auth: bool, keys_json: Option<String>) -> AppState {
6375 auth_state_rated(require_auth, keys_json, None)
6376 }
6377
6378 fn auth_state_rated(
6381 require_auth: bool,
6382 keys_json: Option<String>,
6383 rate_per_sec: Option<u32>,
6384 ) -> AppState {
6385 let config = ProxyConfig::from_lookup(|k| match k {
6386 "FIRSTPASS_REQUIRE_AUTH" => require_auth.then(|| "true".to_owned()),
6387 "FIRSTPASS_TENANT_KEYS_JSON" => keys_json.clone(),
6388 "FIRSTPASS_TENANT_RATE_PER_SEC" => rate_per_sec.map(|n| n.to_string()),
6389 _ => None,
6390 })
6391 .unwrap();
6392 let (traces, _rx) = mpsc::channel(64);
6393 std::mem::forget(_rx);
6396 let providers: HashMap<String, Arc<dyn Provider>> = HashMap::new();
6397 let tenant_rate_limiter = build_tenant_rate_limiter(&config);
6398 AppState {
6399 config: Arc::new(config),
6400 http: reqwest::Client::new(),
6401 providers: ProviderRegistry::from_map(providers),
6402 gate_health: Arc::new(GateHealthRegistry::new()),
6403 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
6404 paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
6405 experiments: Arc::default(),
6406 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
6407 traces,
6408 adaptive: None,
6409 eprocess: None,
6410 bandit: None,
6411 promoter: None,
6412 verified_cache: None,
6413 predictor: None,
6414 prior: None,
6415 tenant_rate_limiter,
6416 spill: None,
6417 }
6418 }
6419
6420 fn cap_request(auth_header: Option<&str>) -> axum::http::Request<Body> {
6421 let mut b = axum::http::Request::builder()
6422 .method("GET")
6423 .uri("/v1/capabilities");
6424 if let Some(h) = auth_header {
6425 b = b.header("authorization", h);
6426 }
6427 b.body(Body::empty()).unwrap()
6428 }
6429
6430 #[tokio::test]
6431 async fn auth_off_allows_unauthenticated_request() {
6432 use tower::ServiceExt;
6433 let router = app(auth_state(false, None)).expect("router");
6434 let resp = router.oneshot(cap_request(None)).await.unwrap();
6435 assert_eq!(resp.status(), axum::http::StatusCode::OK);
6437 }
6438
6439 #[tokio::test]
6440 async fn auth_on_missing_key_is_401_opaque() {
6441 use tower::ServiceExt;
6442 let hash = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
6443 let keys = format!("{{\"tenant-a\": {hash:?}}}");
6444 let router = app(auth_state(true, Some(keys))).expect("router");
6445
6446 let resp = router.oneshot(cap_request(None)).await.unwrap();
6447 assert_eq!(resp.status(), axum::http::StatusCode::UNAUTHORIZED);
6448 let json = body_json(resp).await;
6449 assert_eq!(json["error"]["type"], "unauthorized");
6450 let msg = json["error"]["message"].as_str().unwrap();
6452 assert!(!msg.contains("tenant"), "no tenant oracle in body: {msg}");
6453 }
6454
6455 #[tokio::test]
6456 async fn auth_on_invalid_key_is_401() {
6457 use tower::ServiceExt;
6458 let hash = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
6459 let keys = format!("{{\"tenant-a\": {hash:?}}}");
6460 let router = app(auth_state(true, Some(keys))).expect("router");
6461
6462 let resp = router
6463 .oneshot(cap_request(Some("Bearer wrong-key")))
6464 .await
6465 .unwrap();
6466 assert_eq!(resp.status(), axum::http::StatusCode::UNAUTHORIZED);
6467 }
6468
6469 #[tokio::test]
6470 async fn auth_on_valid_key_proceeds() {
6471 use tower::ServiceExt;
6472 let hash = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
6473 let keys = format!("{{\"tenant-a\": {hash:?}}}");
6474 let router = app(auth_state(true, Some(keys))).expect("router");
6475
6476 let resp = router
6478 .oneshot(cap_request(Some("Bearer tenant-a.key-a")))
6479 .await
6480 .unwrap();
6481 assert_eq!(resp.status(), axum::http::StatusCode::OK);
6483 }
6484
6485 fn two_tenant_state(rate_per_sec: Option<u32>) -> AppState {
6487 let hash_a = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
6488 let hash_b = crate::tenant_auth::TenantKeys::hash_key("key-b").unwrap();
6489 let keys = format!("{{\"tenant-a\": {hash_a:?}, \"tenant-b\": {hash_b:?}}}");
6490 let mut state = auth_state_rated(true, Some(keys), rate_per_sec);
6491 if let Some(n) = rate_per_sec.and_then(std::num::NonZeroU32::new) {
6496 state.tenant_rate_limiter = Some(Arc::new(governor::RateLimiter::keyed(
6497 governor::Quota::per_hour(n),
6498 )));
6499 }
6500 state
6501 }
6502
6503 #[test]
6504 fn tenant_rate_limiter_allows_a_burst_of_exactly_the_configured_rate() {
6505 let config = ProxyConfig::from_lookup(|k| {
6506 (k == "FIRSTPASS_TENANT_RATE_PER_SEC").then(|| "2".to_owned())
6507 })
6508 .unwrap();
6509 let limiter = build_tenant_rate_limiter(&config).expect("rate configured");
6510 let t = "tenant-a".to_owned();
6511 assert!(limiter.check_key(&t).is_ok());
6512 assert!(limiter.check_key(&t).is_ok());
6513 assert!(
6514 limiter.check_key(&t).is_err(),
6515 "third in the same instant is over the burst"
6516 );
6517 assert!(
6518 limiter.check_key(&"tenant-b".to_owned()).is_ok(),
6519 "buckets are per tenant"
6520 );
6521 }
6522
6523 #[tokio::test]
6524 async fn tenant_exceeding_rate_limit_gets_429_opaque() {
6525 use tower::ServiceExt;
6526 let router = app(two_tenant_state(Some(1))).expect("router");
6529
6530 let (r1, r2, r3, r4) = tokio::join!(
6531 router
6532 .clone()
6533 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
6534 router
6535 .clone()
6536 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
6537 router
6538 .clone()
6539 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
6540 router
6541 .clone()
6542 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
6543 );
6544 let responses = [r1.unwrap(), r2.unwrap(), r3.unwrap(), r4.unwrap()];
6545 let ok = responses
6546 .iter()
6547 .filter(|r| r.status() == axum::http::StatusCode::OK)
6548 .count();
6549 assert!(ok >= 1, "the burst's first request must pass");
6550 let limited: Vec<_> = responses
6551 .into_iter()
6552 .filter(|r| r.status() == axum::http::StatusCode::TOO_MANY_REQUESTS)
6553 .collect();
6554 assert!(
6555 !limited.is_empty(),
6556 "a 4-request burst against 1 req/sec must trip the limiter"
6557 );
6558
6559 let json = body_json(limited.into_iter().next().unwrap()).await;
6560 assert_eq!(json["error"]["type"], "rate_limited");
6561 let msg = json["error"]["message"].as_str().unwrap();
6563 assert!(!msg.contains('1'), "no limit value in body: {msg}");
6564 }
6565
6566 #[tokio::test]
6567 async fn rate_limit_buckets_are_independent_per_tenant() {
6568 use tower::ServiceExt;
6569 let router = app(two_tenant_state(Some(1))).expect("router");
6570
6571 let (a1, a2, a3) = tokio::join!(
6573 router
6574 .clone()
6575 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
6576 router
6577 .clone()
6578 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
6579 router
6580 .clone()
6581 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
6582 );
6583 let a_limited = [a1.unwrap(), a2.unwrap(), a3.unwrap()]
6584 .iter()
6585 .filter(|r| r.status() == axum::http::StatusCode::TOO_MANY_REQUESTS)
6586 .count();
6587 assert!(a_limited >= 1, "tenant A's burst must trip its limiter");
6588
6589 let b1 = router
6591 .clone()
6592 .oneshot(cap_request(Some("Bearer tenant-b.key-b")))
6593 .await
6594 .unwrap();
6595 assert_eq!(b1.status(), axum::http::StatusCode::OK);
6596 }
6597
6598 #[tokio::test]
6599 async fn key_verify_does_not_stall_the_async_worker() {
6600 use std::sync::atomic::{AtomicUsize, Ordering};
6601 use tower::ServiceExt;
6602 let router = app(two_tenant_state(None)).expect("router");
6605 let ticks = Arc::new(AtomicUsize::new(0));
6606 let ticker = tokio::spawn({
6607 let ticks = ticks.clone();
6608 async move {
6609 loop {
6610 tokio::time::sleep(std::time::Duration::from_millis(1)).await;
6611 ticks.fetch_add(1, Ordering::Relaxed);
6612 }
6613 }
6614 });
6615 let resp = router
6616 .oneshot(cap_request(Some("Bearer tenant-a.key-a")))
6617 .await
6618 .unwrap();
6619 let during = ticks.load(Ordering::Relaxed);
6620 ticker.abort();
6621 assert_eq!(resp.status(), axum::http::StatusCode::OK);
6622 assert!(
6623 during >= 2,
6624 "worker was blocked during the verify ({during} ticks)"
6625 );
6626 }
6627
6628 #[tokio::test]
6629 async fn rate_limit_unset_never_429s() {
6630 use tower::ServiceExt;
6631 let router = app(two_tenant_state(None)).expect("router");
6634 for _ in 0..20 {
6635 let resp = router
6636 .clone()
6637 .oneshot(cap_request(Some("Bearer tenant-a.key-a")))
6638 .await
6639 .unwrap();
6640 assert_eq!(resp.status(), axum::http::StatusCode::OK);
6641 }
6642 }
6643
6644 #[test]
6647 fn u01_is_deterministic_and_in_range() {
6648 let s1 = u01(0xDEAD_BEEF_CAFE_1234_u128);
6649 let s2 = u01(0xDEAD_BEEF_CAFE_1234_u128);
6650 assert_eq!(s1, s2, "u01 must be deterministic for the same seed");
6651 assert!((0.0..1.0).contains(&s1), "u01 must return [0, 1), got {s1}");
6652
6653 let s3 = u01(0x1234_5678_9ABC_DEF0_u128);
6655 assert_ne!(s1, s3, "different seeds should give different values");
6656
6657 for i in 0u64..256 {
6659 let v = u01(i as u128);
6660 assert!((0.0..1.0).contains(&v), "seed {i}: u01={v} out of [0,1)");
6661 }
6662 }
6663
6664 #[test]
6665 fn epsilon_propensity_formula() {
6666 let epsilon = 0.2_f64;
6667 let k = 3_usize;
6668 let greedy = 1_u32;
6669
6670 let p_greedy = epsilon_propensity(greedy, greedy, epsilon, k);
6672 let expected_greedy = (1.0 - epsilon) + epsilon / k as f64;
6673 assert!(
6674 (p_greedy - expected_greedy).abs() < 1e-12,
6675 "{p_greedy} != {expected_greedy}"
6676 );
6677
6678 let p_other = epsilon_propensity(0, greedy, epsilon, k);
6680 let expected_other = epsilon / k as f64;
6681 assert!(
6682 (p_other - expected_other).abs() < 1e-12,
6683 "{p_other} != {expected_other}"
6684 );
6685
6686 for chosen in 0..k as u32 {
6688 let p = epsilon_propensity(chosen, greedy, epsilon, k);
6689 assert!(
6690 p > 0.0 && p <= 1.0,
6691 "propensity {p} out of (0,1] for chosen={chosen}"
6692 );
6693 }
6694 }
6695
6696 #[test]
6697 fn epsilon_branch_and_greedy_branch_both_occur_over_many_seeds() {
6698 let epsilon = 0.3_f64;
6700 let mut saw_explore = false;
6701 let mut saw_greedy = false;
6702 for i in 0u64..200 {
6703 let u = u01(i as u128);
6704 if u < epsilon {
6705 saw_explore = true;
6706 } else {
6707 saw_greedy = true;
6708 }
6709 if saw_explore && saw_greedy {
6710 break;
6711 }
6712 }
6713 assert!(
6714 saw_explore,
6715 "epsilon branch must fire with epsilon=0.3 over 200 seeds"
6716 );
6717 assert!(
6718 saw_greedy,
6719 "greedy branch must occur with epsilon=0.3 over 200 seeds"
6720 );
6721 }
6722
6723 #[test]
6724 fn epsilon_propensity_sums_to_one_over_all_rungs() {
6725 let epsilon = 0.15_f64;
6727 let k = 4_usize;
6728 let greedy = 2_u32;
6729 let total: f64 = (0..k as u32)
6730 .map(|r| epsilon_propensity(r, greedy, epsilon, k))
6731 .sum();
6732 assert!(
6733 (total - 1.0).abs() < 1e-12,
6734 "propensities must sum to 1, got {total}"
6735 );
6736 }
6737
6738 #[tokio::test(start_paused = true)]
6741 async fn keepalive_stream_ticks_then_emits_final_frame() {
6742 let (tx, rx) = tokio::sync::oneshot::channel::<Result<Value, ProxyError>>();
6743 let mut ticks = tokio::time::interval(SSE_KEEPALIVE_EVERY);
6744 ticks.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
6745 ticks.reset();
6746 let mut stream = KeepaliveStream {
6747 rx: Some(rx),
6748 ticks,
6749 format_message: anthropic_sse_from_message,
6750 };
6751 async fn next(
6753 stream: &mut KeepaliveStream,
6754 ) -> Option<Option<Result<Bytes, std::convert::Infallible>>> {
6755 std::future::poll_fn(|cx| {
6756 std::task::Poll::Ready(
6757 match futures_core::Stream::poll_next(std::pin::Pin::new(&mut *stream), cx) {
6758 std::task::Poll::Ready(item) => Some(item),
6759 std::task::Poll::Pending => None,
6760 },
6761 )
6762 })
6763 .await
6764 }
6765
6766 assert!(
6768 next(&mut stream).await.is_none(),
6769 "no frame before an interval"
6770 );
6771
6772 tokio::time::advance(SSE_KEEPALIVE_EVERY + Duration::from_millis(1)).await;
6774 let frame = next(&mut stream)
6775 .await
6776 .expect("keepalive due")
6777 .unwrap()
6778 .unwrap();
6779 assert!(
6780 frame.starts_with(b": "),
6781 "keepalive must be an SSE comment (ignored by every conforming parser)"
6782 );
6783
6784 let message = serde_json::json!({
6786 "id": "msg_1", "type": "message", "role": "assistant", "model": "m",
6787 "content": [{ "type": "text", "text": "done" }],
6788 "usage": { "input_tokens": 1, "output_tokens": 1 }
6789 });
6790 tx.send(Ok(message)).unwrap();
6791 let frame = next(&mut stream)
6792 .await
6793 .expect("final frame")
6794 .unwrap()
6795 .unwrap();
6796 let text = String::from_utf8(frame.to_vec()).unwrap();
6797 assert!(text.contains("event: message_start"));
6798 assert!(text.contains("event: message_stop"));
6799 let eos = next(&mut stream)
6800 .await
6801 .expect("stream must end after the final frame");
6802 assert!(eos.is_none(), "end-of-stream after the final frame");
6803 }
6804
6805 #[tokio::test]
6808 async fn streaming_enforce_serves_full_sse_sequence() {
6809 let toml = "[[price]]\nmodel = \"anthropic/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/m\"]\ngates = [\"non-empty\"]\n";
6810 let config = ProxyConfig::from_lookup(|k| match k {
6811 "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
6812 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
6813 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
6814 _ => None,
6815 })
6816 .unwrap();
6817 let mut outs = HashMap::new();
6818 outs.insert(
6819 "anthropic/m".to_owned(),
6820 Ok(model_resp("anthropic/m", "gated answer")),
6821 );
6822 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
6823 map.insert(
6824 "anthropic".to_owned(),
6825 Arc::new(MockProvider::new("anthropic", outs)),
6826 );
6827 let (traces, _rx) = mpsc::channel(64);
6828 let state = AppState {
6829 config: Arc::new(config),
6830 http: reqwest::Client::new(),
6831 providers: ProviderRegistry::from_map(map),
6832 gate_health: Arc::new(GateHealthRegistry::new()),
6833 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
6834 paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
6835 experiments: Arc::default(),
6836 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
6837 traces,
6838 adaptive: None,
6839 eprocess: None,
6840 bandit: None,
6841 promoter: None,
6842 verified_cache: None,
6843 predictor: None,
6844 prior: None,
6845 tenant_rate_limiter: None,
6846 spill: None,
6847 };
6848 let body = Bytes::from_static(
6849 br#"{"model":"m","stream":true,"messages":[{"role":"user","content":"hi"}]}"#,
6850 );
6851 let resp = messages(
6852 State(state),
6853 Extension(TenantId("default".to_owned())),
6854 HeaderMap::new(),
6855 body,
6856 )
6857 .await;
6858 assert_eq!(resp.status(), axum::http::StatusCode::OK);
6859 assert!(
6860 resp.headers()
6861 .get(axum::http::header::CONTENT_TYPE)
6862 .and_then(|v| v.to_str().ok())
6863 .is_some_and(|ct| ct.starts_with("text/event-stream")),
6864 "streaming client must get SSE"
6865 );
6866 let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
6867 .await
6868 .unwrap();
6869 let text = String::from_utf8(bytes.to_vec()).unwrap();
6870 assert!(text.contains("event: message_start"));
6871 assert!(text.contains("gated answer"));
6872 assert!(text.contains("event: message_stop"));
6873 }
6874
6875 #[test]
6880 fn parse_openai_request_plain_text() {
6881 let body = br#"{"model":"gpt-4o","max_tokens":256,"messages":[{"role":"user","content":"hello"}]}"#;
6883 let req = parse_openai_request(body, false).expect("must parse");
6884 assert_eq!(req.model, "gpt-4o");
6885 assert_eq!(req.max_tokens, 256);
6886 assert_eq!(req.messages.len(), 1);
6887 assert_eq!(req.messages[0].role, "user");
6888 assert_eq!(req.messages[0].content, Value::String("hello".to_owned()));
6889 assert!(req.system.is_none());
6890 assert_eq!(req.raw, Value::Null);
6892 }
6893
6894 #[test]
6895 fn parse_openai_request_system_message() {
6896 let body = br#"{"model":"gpt-4o","messages":[{"role":"system","content":"be concise"},{"role":"user","content":"hi"}]}"#;
6897 let req = parse_openai_request(body, false).expect("must parse");
6898 assert_eq!(req.system.as_deref(), Some("be concise"));
6899 assert_eq!(req.messages.len(), 1);
6900 assert_eq!(req.messages[0].role, "user");
6901 }
6902
6903 #[test]
6904 fn parse_openai_request_tool_calls_translate_to_tool_use() {
6905 let body = br#"{
6906 "model":"gpt-4o",
6907 "messages":[
6908 {"role":"user","content":"what's the weather?"},
6909 {"role":"assistant","content":null,"tool_calls":[
6910 {"id":"call_1","type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Paris\"}"}}
6911 ]},
6912 {"role":"tool","tool_call_id":"call_1","content":"15C, cloudy"}
6913 ]
6914 }"#;
6915 let req = parse_openai_request(body, false).expect("must parse");
6916 assert_eq!(req.messages.len(), 3);
6918 let asst = &req.messages[1];
6920 assert_eq!(asst.role, "assistant");
6921 let blocks = asst.content.as_array().expect("content array");
6922 assert_eq!(blocks[0]["type"], "tool_use");
6923 assert_eq!(blocks[0]["name"], "get_weather");
6924 assert_eq!(blocks[0]["id"], "call_1");
6925 assert_eq!(blocks[0]["input"]["city"], "Paris");
6926 let tool_msg = &req.messages[2];
6928 assert_eq!(tool_msg.role, "user");
6929 let result_blocks = tool_msg.content.as_array().expect("result blocks");
6930 assert_eq!(result_blocks[0]["type"], "tool_result");
6931 assert_eq!(result_blocks[0]["tool_use_id"], "call_1");
6932 }
6933
6934 #[test]
6935 fn parse_openai_request_tools_translate_to_anthropic_format() {
6936 let body = br#"{
6937 "model":"gpt-4o",
6938 "messages":[{"role":"user","content":"use a tool"}],
6939 "tools":[{"type":"function","function":{"name":"search","description":"web search","parameters":{"type":"object","properties":{"q":{"type":"string"}}}}}]
6940 }"#;
6941 let req = parse_openai_request(body, false).expect("must parse");
6942 let tools = req.tools.as_array().expect("tools array");
6943 assert_eq!(tools.len(), 1);
6944 assert_eq!(tools[0]["name"], "search");
6945 assert_eq!(tools[0]["description"], "web search");
6946 assert_eq!(tools[0]["input_schema"]["type"], "object");
6947 }
6948
6949 #[test]
6950 fn parse_openai_request_raw_carry_preserves_full_body() {
6951 let body =
6953 br#"{"model":"gpt-4o","max_tokens":100,"messages":[{"role":"user","content":"hi"}]}"#;
6954 let req = parse_openai_request(body, true).expect("must parse");
6955 assert!(req.raw.is_object(), "raw must be the full JSON object");
6956 assert_eq!(req.raw["model"], "gpt-4o");
6957 assert_eq!(req.raw["max_tokens"], 100);
6958 assert!(req.tools.is_null(), "no tools in this request");
6960 }
6961
6962 #[test]
6963 fn parse_openai_request_http_image_returns_none() {
6964 let body = br#"{"model":"gpt-4o","messages":[{"role":"user","content":[
6966 {"type":"text","text":"describe this"},
6967 {"type":"image_url","image_url":{"url":"https://example.com/cat.png"}}
6968 ]}]}"#;
6969 let result = parse_openai_request(body, false);
6970 assert!(result.is_none(), "http image URL must fail translation");
6971 }
6972
6973 #[test]
6974 fn parse_openai_request_data_url_image_translates_to_anthropic_base64() {
6975 let body = br#"{"model":"gpt-4o","messages":[{"role":"user","content":[
6976 {"type":"text","text":"describe"},
6977 {"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgo="}}
6978 ]}]}"#;
6979 let req = parse_openai_request(body, false).expect("data URL must parse");
6980 let blocks = req.messages[0].content.as_array().expect("blocks");
6981 let img = blocks
6982 .iter()
6983 .find(|b| b["type"] == "image")
6984 .expect("image block");
6985 assert_eq!(img["source"]["type"], "base64");
6986 assert_eq!(img["source"]["media_type"], "image/png");
6987 assert_eq!(img["source"]["data"], "iVBORw0KGgo=");
6988 }
6989
6990 #[test]
6993 fn openai_response_json_renders_text_response() {
6994 let resp = ModelResponse {
6995 model: "gpt-4o".to_owned(),
6996 text: "Hello!".to_owned(),
6997 in_tokens: 10,
6998 cache_write_tokens: 0,
6999 cache_read_tokens: 0,
7000 out_tokens: 5,
7001 raw: serde_json::json!({
7002 "content": [{ "type": "text", "text": "Hello!" }]
7003 }),
7004 };
7005 let json = openai_response_json(&resp);
7006 assert_eq!(json["object"], "chat.completion");
7007 assert_eq!(json["model"], "gpt-4o");
7008 assert_eq!(json["choices"][0]["message"]["role"], "assistant");
7009 assert_eq!(json["choices"][0]["message"]["content"], "Hello!");
7010 assert_eq!(json["choices"][0]["finish_reason"], "stop");
7011 assert_eq!(json["usage"]["prompt_tokens"], 10);
7012 assert_eq!(json["usage"]["completion_tokens"], 5);
7013 }
7014
7015 #[test]
7016 fn openai_response_json_renders_tool_call() {
7017 let resp = ModelResponse {
7018 model: "gpt-4o".to_owned(),
7019 text: String::new(),
7020 in_tokens: 20,
7021 cache_write_tokens: 0,
7022 cache_read_tokens: 0,
7023 out_tokens: 15,
7024 raw: serde_json::json!({
7025 "content": [{
7026 "type": "tool_use",
7027 "id": "call_abc",
7028 "name": "search",
7029 "input": {"q": "Rust async"}
7030 }]
7031 }),
7032 };
7033 let json = openai_response_json(&resp);
7034 assert_eq!(json["choices"][0]["finish_reason"], "tool_calls");
7035 let tc = &json["choices"][0]["message"]["tool_calls"][0];
7036 assert_eq!(tc["id"], "call_abc");
7037 assert_eq!(tc["type"], "function");
7038 assert_eq!(tc["function"]["name"], "search");
7039 assert_eq!(json["choices"][0]["message"]["content"], Value::Null);
7041 }
7042
7043 #[test]
7044 fn openai_sse_from_message_plain_text_has_role_then_content_then_stop() {
7045 let resp = ModelResponse {
7046 model: "gpt-4o".to_owned(),
7047 text: "Hi there!".to_owned(),
7048 in_tokens: 5,
7049 cache_write_tokens: 0,
7050 cache_read_tokens: 0,
7051 out_tokens: 3,
7052 raw: serde_json::json!({
7053 "content": [{ "type": "text", "text": "Hi there!" }]
7054 }),
7055 };
7056 let sse = openai_sse_from_message(&openai_response_json(&resp));
7057 for line in sse.lines() {
7059 assert!(
7060 line.is_empty() || line.starts_with("data: "),
7061 "bad SSE line: {line:?}"
7062 );
7063 }
7064 let frames: Vec<&str> = sse
7065 .lines()
7066 .filter_map(|l| l.strip_prefix("data: "))
7067 .collect();
7068 assert_eq!(*frames.last().unwrap(), "[DONE]");
7070 let role_frame: Value = serde_json::from_str(frames[0]).unwrap();
7072 assert_eq!(role_frame["choices"][0]["delta"]["role"], "assistant");
7073 assert!(frames.iter().any(|f| {
7075 if *f == "[DONE]" {
7076 return false;
7077 }
7078 serde_json::from_str::<Value>(f)
7079 .ok()
7080 .is_some_and(|v| v["choices"][0]["delta"]["content"] == "Hi there!")
7081 }));
7082 assert!(frames.iter().any(|f| {
7084 if *f == "[DONE]" {
7085 return false;
7086 }
7087 serde_json::from_str::<Value>(f)
7088 .ok()
7089 .is_some_and(|v| v["choices"][0]["finish_reason"] == "stop")
7090 }));
7091 }
7092
7093 #[test]
7096 fn detects_openai_tool_calls() {
7097 let with_tool_calls = Bytes::from_static(br#"{"messages":[
7098 {"role":"user","content":"hi"},
7099 {"role":"assistant","content":null,"tool_calls":[{"id":"c1","type":"function","function":{"name":"f","arguments":"{}"}}]}
7100 ]}"#);
7101 let without = Bytes::from_static(br#"{"messages":[{"role":"user","content":"hi"}]}"#);
7102 let with_tool_msg = Bytes::from_static(
7103 br#"{"messages":[
7104 {"role":"tool","tool_call_id":"c1","content":"result"}
7105 ]}"#,
7106 );
7107 assert!(openai_messages_have_tool_calls(&with_tool_calls));
7108 assert!(!openai_messages_have_tool_calls(&without));
7109 assert!(openai_messages_have_tool_calls(&with_tool_msg));
7110 }
7111
7112 #[test]
7113 fn detects_openai_http_images() {
7114 let http_img = Bytes::from_static(
7115 br#"{"messages":[{"role":"user","content":[
7116 {"type":"image_url","image_url":{"url":"https://example.com/img.png"}}
7117 ]}]}"#,
7118 );
7119 let data_img = Bytes::from_static(
7120 br#"{"messages":[{"role":"user","content":[
7121 {"type":"image_url","image_url":{"url":"data:image/png;base64,abc"}}
7122 ]}]}"#,
7123 );
7124 let no_img = Bytes::from_static(br#"{"messages":[{"role":"user","content":"hi"}]}"#);
7125 assert!(openai_has_http_images(&http_img));
7126 assert!(!openai_has_http_images(&data_img));
7127 assert!(!openai_has_http_images(&no_img));
7128 }
7129
7130 #[test]
7131 fn enforce_can_handle_openai_inbound_all_openai_ladder() {
7132 let tools_body = Bytes::from_static(br#"{"model":"gpt-4o","messages":[{"role":"assistant","content":null,"tool_calls":[{"id":"c","type":"function","function":{"name":"f","arguments":"{}"}}]}]}"#);
7134 let f = extract_openai_features(&HeaderMap::new(), &tools_body);
7135 let ladder = vec!["openai/gpt-4o-mini".to_owned(), "openai/gpt-4o".to_owned()];
7136 let providers = crate::provider::ProviderRegistry::new("http://x", "http://x");
7137 assert!(enforce_can_handle(
7138 &f,
7139 &tools_body,
7140 true,
7141 &ladder,
7142 &providers,
7143 Dialect::Openai
7144 ));
7145 }
7146
7147 #[test]
7148 fn enforce_can_handle_openai_inbound_all_anthropic_ladder_no_http_image() {
7149 let tools_body = Bytes::from_static(br#"{"model":"gpt-4o","messages":[{"role":"assistant","content":null,"tool_calls":[{"id":"c","type":"function","function":{"name":"f","arguments":"{}"}}]}]}"#);
7151 let f = extract_openai_features(&HeaderMap::new(), &tools_body);
7152 let ladder = vec!["anthropic/claude-haiku-4-5".to_owned()];
7153 let providers = crate::provider::ProviderRegistry::new("http://x", "http://x");
7154 assert!(enforce_can_handle(
7155 &f,
7156 &tools_body,
7157 true,
7158 &ladder,
7159 &providers,
7160 Dialect::Openai,
7161 ));
7162 }
7163
7164 #[test]
7165 fn enforce_can_handle_openai_inbound_http_image_falls_back() {
7166 let img_body = Bytes::from_static(
7168 br#"{"model":"gpt-4o","messages":[{"role":"user","content":[
7169 {"type":"image_url","image_url":{"url":"https://example.com/img.png"}}
7170 ]}]}"#,
7171 );
7172 let f = extract_openai_features(&HeaderMap::new(), &img_body);
7173 let ladder = vec!["anthropic/claude-haiku-4-5".to_owned()];
7174 let providers = crate::provider::ProviderRegistry::new("http://x", "http://x");
7175 assert!(!enforce_can_handle(
7176 &f,
7177 &img_body,
7178 true,
7179 &ladder,
7180 &providers,
7181 Dialect::Openai,
7182 ));
7183 }
7184
7185 fn openai_enforce_state(mock_resp: ModelResponse) -> AppState {
7189 let toml = "[[price]]\nmodel = \"mock/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"mock/m\"]\ngates = [\"non-empty\"]\n";
7190 let config = ProxyConfig::from_lookup(|k| match k {
7191 "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
7192 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
7193 _ => None,
7194 })
7195 .unwrap();
7196 let mut outs = HashMap::new();
7197 outs.insert("mock/m".to_owned(), Ok(mock_resp));
7198 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
7199 map.insert("mock".to_owned(), Arc::new(MockProvider::new("mock", outs)));
7200 let (traces, _rx) = mpsc::channel(64);
7201 std::mem::forget(_rx);
7202 let tenant_rate_limiter = build_tenant_rate_limiter(&config);
7203 AppState {
7204 config: Arc::new(config),
7205 http: reqwest::Client::new(),
7206 providers: ProviderRegistry::from_map(map),
7207 gate_health: Arc::new(GateHealthRegistry::new()),
7208 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
7209 paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
7210 experiments: Arc::default(),
7211 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
7212 traces,
7213 adaptive: None,
7214 eprocess: None,
7215 bandit: None,
7216 promoter: None,
7217 verified_cache: None,
7218 predictor: None,
7219 prior: None,
7220 tenant_rate_limiter,
7221 spill: None,
7222 }
7223 }
7224
7225 #[tokio::test]
7226 async fn chat_completions_plain_text_enforce_returns_openai_shape() {
7227 let mock = model_resp("mock/m", "gated answer");
7228 let state = openai_enforce_state(mock);
7229 let body = Bytes::from_static(
7230 br#"{"model":"gpt-4o","messages":[{"role":"user","content":"hello"}]}"#,
7231 );
7232 let resp = chat_completions(
7233 State(state),
7234 Extension(TenantId("default".to_owned())),
7235 HeaderMap::new(),
7236 body,
7237 )
7238 .await;
7239 assert_eq!(resp.status(), axum::http::StatusCode::OK);
7240 let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
7241 .await
7242 .unwrap();
7243 let json: Value = serde_json::from_slice(&bytes).expect("must be JSON");
7244 assert_eq!(json["object"], "chat.completion");
7245 assert_eq!(json["choices"][0]["message"]["role"], "assistant");
7246 assert_eq!(json["choices"][0]["message"]["content"], "gated answer");
7247 assert_eq!(json["choices"][0]["finish_reason"], "stop");
7248 assert!(
7249 json["id"]
7250 .as_str()
7251 .is_some_and(|id| id.starts_with("chatcmpl-"))
7252 );
7253 }
7254
7255 #[tokio::test]
7256 async fn chat_completions_stream_true_returns_sse_with_openai_chunks() {
7257 let mock = model_resp("mock/m", "gated answer");
7258 let state = openai_enforce_state(mock);
7259 let body = Bytes::from_static(
7260 br#"{"model":"gpt-4o","stream":true,"messages":[{"role":"user","content":"hello"}]}"#,
7261 );
7262 let resp = chat_completions(
7263 State(state),
7264 Extension(TenantId("default".to_owned())),
7265 HeaderMap::new(),
7266 body,
7267 )
7268 .await;
7269 assert_eq!(resp.status(), axum::http::StatusCode::OK);
7270 assert!(
7271 resp.headers()
7272 .get(axum::http::header::CONTENT_TYPE)
7273 .and_then(|v| v.to_str().ok())
7274 .is_some_and(|ct| ct.starts_with("text/event-stream")),
7275 "stream:true must return SSE"
7276 );
7277 let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
7278 .await
7279 .unwrap();
7280 let text = String::from_utf8(bytes.to_vec()).unwrap();
7281 assert!(
7283 text.contains("chat.completion.chunk"),
7284 "must have OpenAI chunk frames"
7285 );
7286 assert!(text.contains("[DONE]"), "must end with [DONE]");
7287 assert!(
7288 text.contains("gated answer"),
7289 "content must be in the stream"
7290 );
7291 assert!(
7293 !text.contains("message_start"),
7294 "must not have Anthropic event types"
7295 );
7296 }
7297
7298 fn probe_state(
7302 sample_rate: f64,
7303 k: u32,
7304 outcomes: Vec<(&str, Result<ModelResponse, ProviderError>)>,
7305 ) -> (AppState, mpsc::Receiver<Trace>) {
7306 let toml = format!(
7307 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\nladder = [\"anthropic/claude-haiku-4-5\"]\ngates = [\"non-empty\"]\n\
7308 [escalation.probe]\nk = {k}\nsample_rate = {sample_rate}\n"
7309 );
7310 let config = ProxyConfig::from_lookup(|k_| match k_ {
7311 "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
7312 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
7313 _ => None,
7314 })
7315 .unwrap();
7316 let mut outs = HashMap::new();
7317 for (model, out) in outcomes {
7318 outs.insert(model.to_owned(), out);
7319 }
7320 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
7321 map.insert(
7322 "anthropic".to_owned(),
7323 Arc::new(MockProvider::new("anthropic", outs)),
7324 );
7325 let (traces, rx) = mpsc::channel(64);
7326 let state = AppState {
7327 config: Arc::new(config),
7328 http: reqwest::Client::new(),
7329 providers: ProviderRegistry::from_map(map),
7330 gate_health: Arc::new(GateHealthRegistry::new()),
7331 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
7332 paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
7333 experiments: Arc::default(),
7334 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
7335 traces,
7336 adaptive: None,
7337 eprocess: None,
7338 bandit: None,
7339 promoter: None,
7340 verified_cache: None,
7341 predictor: None,
7342 prior: None,
7343 tenant_rate_limiter: None,
7344 spill: None,
7345 };
7346 (state, rx)
7347 }
7348
7349 async fn run_enforce_get_trace(state: AppState, mut rx: mpsc::Receiver<Trace>) -> Trace {
7351 let resp = messages(
7352 State(state),
7353 Extension(TenantId("default".to_owned())),
7354 HeaderMap::new(),
7355 user_body(),
7356 )
7357 .await;
7358 assert_eq!(resp.status(), axum::http::StatusCode::OK);
7359 rx.try_recv().expect("trace must be enqueued")
7360 }
7361
7362 #[tokio::test]
7364 async fn probe_off_trace_has_no_probe_field() {
7365 let (state, rx) = enforce_state(
7366 &["anthropic/claude-haiku-4-5"],
7367 &["non-empty"],
7368 vec![(
7369 "anthropic/claude-haiku-4-5",
7370 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
7371 )],
7372 );
7373 assert!(
7374 state
7375 .config
7376 .routing
7377 .as_ref()
7378 .unwrap()
7379 .escalation
7380 .probe
7381 .is_none(),
7382 "probe must default to None"
7383 );
7384 let trace = run_enforce_get_trace(state, rx).await;
7385 assert!(
7386 trace.probe.is_none(),
7387 "probe=None config must not set trace.probe"
7388 );
7389 }
7390
7391 #[tokio::test]
7393 async fn probe_sample_rate_zero_never_fires() {
7394 let (state, rx) = probe_state(
7396 0.0,
7397 5,
7398 vec![(
7399 "anthropic/claude-haiku-4-5",
7400 Ok(model_resp("anthropic/claude-haiku-4-5", "hi")),
7401 )],
7402 );
7403 let trace = run_enforce_get_trace(state, rx).await;
7404 assert!(
7405 trace.probe.is_none(),
7406 "sample_rate=0.0 must never set trace.probe"
7407 );
7408 }
7409
7410 #[tokio::test]
7413 async fn probe_on_all_pass_sets_confident_pass() {
7414 let model = "anthropic/claude-haiku-4-5";
7416 let (state, mut rx) = probe_state(1.0, 3, vec![(model, Ok(model_resp(model, "hello")))]);
7417 let resp = messages(
7418 State(state),
7419 Extension(TenantId("default".to_owned())),
7420 HeaderMap::new(),
7421 user_body(),
7422 )
7423 .await;
7424 assert_eq!(resp.status(), axum::http::StatusCode::OK);
7425 let json = body_json(resp).await;
7427 assert_eq!(
7428 json["content"][0]["text"], "hello",
7429 "served content unchanged"
7430 );
7431
7432 let trace = rx.try_recv().expect("trace enqueued");
7433 let sig = trace.probe.expect("probe must be set when sample_rate=1.0");
7434 assert_eq!(sig.k, 3);
7435 assert_eq!(sig.gate_pass_count, 3, "all 3 samples must pass non-empty");
7436 assert_eq!(
7437 sig.regime,
7438 firstpass_core::ProbeRegime::ConfidentPass,
7439 "all-pass → ConfidentPass"
7440 );
7441 assert!(
7442 sig.probe_cost_usd > 0.0,
7443 "k model calls must cost something"
7444 );
7445 }
7446
7447 #[tokio::test]
7450 async fn probe_on_all_fail_sets_confident_fail() {
7451 let model = "anthropic/claude-haiku-4-5";
7452 let (state, mut rx) = probe_state(1.0, 3, vec![(model, Ok(model_resp(model, "")))]);
7454 let resp = messages(
7456 State(state),
7457 Extension(TenantId("default".to_owned())),
7458 HeaderMap::new(),
7459 user_body(),
7460 )
7461 .await;
7462 assert_eq!(resp.status(), axum::http::StatusCode::OK);
7463
7464 let trace = rx.try_recv().expect("trace enqueued");
7465 let sig = trace.probe.expect("probe must be set when sample_rate=1.0");
7466 assert_eq!(
7467 sig.gate_pass_count, 0,
7468 "empty response fails non-empty: all 0 pass"
7469 );
7470 assert_eq!(
7471 sig.regime,
7472 firstpass_core::ProbeRegime::ConfidentFail,
7473 "0 passes → ConfidentFail"
7474 );
7475 }
7476
7477 #[tokio::test]
7480 async fn probe_on_served_output_identical_to_probe_off() {
7481 let model = "anthropic/claude-haiku-4-5";
7482 let mk = |sample_rate: f64| {
7483 probe_state(
7484 sample_rate,
7485 2,
7486 vec![(model, Ok(model_resp(model, "gated answer")))],
7487 )
7488 };
7489
7490 let (state_off, mut rx_off) = mk(0.0);
7492 let resp_off = messages(
7493 State(state_off),
7494 Extension(TenantId("default".to_owned())),
7495 HeaderMap::new(),
7496 user_body(),
7497 )
7498 .await;
7499 let json_off = body_json(resp_off).await;
7500 let trace_off = rx_off.try_recv().unwrap();
7501
7502 let (state_on, mut rx_on) = mk(1.0);
7504 let resp_on = messages(
7505 State(state_on),
7506 Extension(TenantId("default".to_owned())),
7507 HeaderMap::new(),
7508 user_body(),
7509 )
7510 .await;
7511 let json_on = body_json(resp_on).await;
7512 let trace_on = rx_on.try_recv().unwrap();
7513
7514 assert_eq!(
7516 json_off["content"][0]["text"], json_on["content"][0]["text"],
7517 "served text must be identical regardless of probe"
7518 );
7519 assert!(
7521 (trace_off.final_.total_cost_usd - trace_on.final_.total_cost_usd).abs() < 1e-12,
7522 "total_cost_usd must not include probe cost: off={} on={}",
7523 trace_off.final_.total_cost_usd,
7524 trace_on.final_.total_cost_usd
7525 );
7526 assert!(trace_off.probe.is_none());
7528 assert!(trace_on.probe.is_some());
7529 assert!(
7531 trace_on.probe.as_ref().unwrap().probe_cost_usd > 0.0,
7532 "probe cost must be positive"
7533 );
7534 }
7535
7536 #[tokio::test]
7546 async fn probe_does_not_mutate_gate_health() {
7547 let model = "anthropic/claude-haiku-4-5";
7548 let (mut state, rx) = probe_state(1.0, 2, vec![(model, Ok(model_resp(model, "answer")))]);
7549
7550 let registry = GateHealthRegistry::new().with_budget("non-empty", 2, 0.4);
7553 registry.record("default", "non-empty", true);
7556 assert!(
7557 registry.enabled("default", "non-empty"),
7558 "gate must start enabled (window not full yet)"
7559 );
7560 state.gate_health = Arc::new(registry);
7561
7562 let trace = run_enforce_get_trace(state, rx).await;
7574 let sig = trace.probe.expect("probe must fire with sample_rate=1.0");
7575 assert_eq!(sig.k, 2);
7576 assert!(
7577 sig.gate_pass_count <= 2,
7578 "gate_pass_count must be in [0, k]"
7579 );
7580 }
7581
7582 fn predictor_state(
7584 enabled: bool,
7585 outcome: Result<ModelResponse, ProviderError>,
7586 ) -> (AppState, mpsc::Receiver<Trace>) {
7587 let toml = "[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/claude-haiku-4-5\"]\ngates = [\"non-empty\"]\n";
7588 let config = ProxyConfig::from_lookup(|k_| match k_ {
7589 "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
7590 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
7591 _ => None,
7592 })
7593 .unwrap();
7594 let mut outs = HashMap::new();
7595 outs.insert("anthropic/claude-haiku-4-5".to_owned(), outcome);
7596 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
7597 map.insert(
7598 "anthropic".to_owned(),
7599 Arc::new(MockProvider::new("anthropic", outs)),
7600 );
7601 let (traces, rx) = mpsc::channel(64);
7602 let predictor = enabled.then(|| {
7603 Arc::new(std::sync::Mutex::new(firstpass_core::PassPredictor::new(
7604 0.05, 1e-4,
7605 )))
7606 });
7607 let state = AppState {
7608 config: Arc::new(config),
7609 http: reqwest::Client::new(),
7610 providers: ProviderRegistry::from_map(map),
7611 gate_health: Arc::new(GateHealthRegistry::new()),
7612 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
7613 paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
7614 experiments: Arc::default(),
7615 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
7616 traces,
7617 adaptive: None,
7618 eprocess: None,
7619 bandit: None,
7620 promoter: None,
7621 verified_cache: None,
7622 predictor,
7623 prior: None,
7624 tenant_rate_limiter: None,
7625 spill: None,
7626 };
7627 (state, rx)
7628 }
7629
7630 #[tokio::test]
7631 async fn predictor_off_leaves_predicted_pass_none() {
7632 let (state, rx) =
7633 predictor_state(false, Ok(model_resp("anthropic/claude-haiku-4-5", "ok")));
7634 let trace = run_enforce_get_trace(state, rx).await;
7635 assert!(
7636 trace.predicted_pass.is_none(),
7637 "predictor off => no field (byte-identical)"
7638 );
7639 let j = serde_json::to_string(&trace).unwrap();
7641 assert!(!j.contains("predicted_pass"), "None must be omitted: {j}");
7642 }
7643
7644 #[tokio::test]
7645 async fn predictor_on_records_shadow_prediction_and_serves_identically() {
7646 let (state_off, rx_off) = predictor_state(
7649 false,
7650 Ok(model_resp("anthropic/claude-haiku-4-5", "served answer")),
7651 );
7652 let off = run_enforce_get_trace(state_off, rx_off).await;
7653
7654 let (state_on, rx_on) = predictor_state(
7655 true,
7656 Ok(model_resp("anthropic/claude-haiku-4-5", "served answer")),
7657 );
7658 let on = run_enforce_get_trace(state_on, rx_on).await;
7659
7660 assert_eq!(
7661 on.final_.served_rung, off.final_.served_rung,
7662 "served rung identical"
7663 );
7664 assert_eq!(on.attempts.len(), off.attempts.len(), "same attempts");
7665 assert_eq!(
7666 on.final_.total_cost_usd, off.final_.total_cost_usd,
7667 "predictor never adds served cost"
7668 );
7669 let p = on
7670 .predicted_pass
7671 .expect("predictor on => predicted_pass recorded");
7672 assert!(p > 0.0 && p < 1.0, "shadow prediction in (0,1): {p}");
7673 }
7674
7675 fn rollout_state(
7677 percent: f64,
7678 key: &str,
7679 outcomes: Vec<(&str, Result<ModelResponse, ProviderError>)>,
7680 ) -> (AppState, mpsc::Receiver<Trace>) {
7681 let toml = format!(
7682 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
7683 ladder = [\"anthropic/claude-haiku-4-5\"]\ngates = [\"non-empty\"]\n\
7684 [route.rollout]\npercent = {percent}\nkey = \"{key}\"\n"
7685 );
7686 let config = ProxyConfig::from_lookup(|k| match k {
7687 "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
7688 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
7689 _ => None,
7690 })
7691 .unwrap();
7692 let mut outs = HashMap::new();
7693 for (model, out) in outcomes {
7694 outs.insert(model.to_owned(), out);
7695 }
7696 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
7697 map.insert(
7698 "anthropic".to_owned(),
7699 Arc::new(MockProvider::new("anthropic", outs)),
7700 );
7701 let (traces, rx) = mpsc::channel(64);
7702 let state = AppState {
7703 config: Arc::new(config),
7704 http: reqwest::Client::new(),
7705 providers: ProviderRegistry::from_map(map),
7706 gate_health: Arc::new(GateHealthRegistry::new()),
7707 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
7708 paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
7709 experiments: Arc::default(),
7710 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
7711 traces,
7712 adaptive: None,
7713 eprocess: None,
7714 bandit: None,
7715 promoter: None,
7716 verified_cache: None,
7717 predictor: None,
7718 prior: None,
7719 tenant_rate_limiter: None,
7720 spill: None,
7721 };
7722 (state, rx)
7723 }
7724
7725 #[tokio::test]
7729 async fn rollout_at_zero_percent_never_enforces() {
7730 for i in 0..25 {
7731 let (state, mut rx) = rollout_state(
7732 0.0,
7733 "session",
7734 vec![(
7735 "anthropic/claude-haiku-4-5",
7736 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
7737 )],
7738 );
7739 let mut headers = HeaderMap::new();
7740 headers.insert("x-firstpass-session", format!("s{i}").parse().unwrap());
7741 let resp = messages(
7742 State(state),
7743 Extension(TenantId("default".to_owned())),
7744 headers,
7745 user_body(),
7746 )
7747 .await;
7748 let _ = resp;
7751 if let Ok(t) = rx.try_recv() {
7752 assert!(
7753 t.attempts.is_empty(),
7754 "0% rollout produced an enforced decision for session s{i}"
7755 );
7756 }
7757 }
7758 }
7759
7760 #[tokio::test]
7763 async fn rollout_at_hundred_percent_always_enforces() {
7764 for i in 0..25 {
7765 let (state, mut rx) = rollout_state(
7766 100.0,
7767 "session",
7768 vec![(
7769 "anthropic/claude-haiku-4-5",
7770 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
7771 )],
7772 );
7773 let mut headers = HeaderMap::new();
7774 headers.insert("x-firstpass-session", format!("s{i}").parse().unwrap());
7775 let resp = messages(
7776 State(state),
7777 Extension(TenantId("default".to_owned())),
7778 headers,
7779 user_body(),
7780 )
7781 .await;
7782 assert_eq!(resp.status(), axum::http::StatusCode::OK);
7783 let t = rx.try_recv().expect("trace enqueued");
7784 assert!(
7785 !t.attempts.is_empty(),
7786 "100% rollout skipped enforcement for session s{i}"
7787 );
7788 }
7789 }
7790
7791 #[tokio::test]
7799 async fn dispatch_arm_matches_the_predicted_arm_per_session() {
7800 let mut checked_enforced = 0;
7801 let mut checked_control = 0;
7802 for i in 0..24 {
7803 let session = format!("session-{i}");
7804 let (state, mut rx) = rollout_state(
7805 50.0,
7806 "session",
7807 vec![(
7808 "anthropic/claude-haiku-4-5",
7809 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
7810 )],
7811 );
7812 let expected = firstpass_core::rollout::decide(
7815 &state.config.prompt_salt,
7816 &firstpass_core::Rollout {
7817 percent: 50.0,
7818 key: firstpass_core::RolloutKey::Session,
7819 },
7820 &session,
7821 )
7822 .enforced;
7823
7824 let mut headers = HeaderMap::new();
7825 headers.insert("x-firstpass-session", session.parse().unwrap());
7826 let _ = messages(
7827 State(state),
7828 Extension(TenantId("default".to_owned())),
7829 headers,
7830 user_body(),
7831 )
7832 .await;
7833 let observed = rx
7834 .try_recv()
7835 .map(|t| !t.attempts.is_empty())
7836 .unwrap_or(false);
7837
7838 assert_eq!(
7839 observed,
7840 expected,
7841 "{session}: dispatch put it in the {} arm, bucketing predicted the {} arm",
7842 if observed { "enforced" } else { "control" },
7843 if expected { "enforced" } else { "control" }
7844 );
7845 if expected {
7846 checked_enforced += 1;
7847 } else {
7848 checked_control += 1;
7849 }
7850 }
7851 assert!(
7853 checked_enforced > 0 && checked_control > 0,
7854 "test saw only one arm ({checked_enforced} enforced, {checked_control} control)"
7855 );
7856 }
7857
7858 const INCUMBENT: &str = "anthropic/claude-sonnet-5";
7861 const CHALLENGER: &str = "anthropic/claude-haiku-4-5";
7862
7863 fn challenger_state(percent: f64, db: &str) -> (AppState, mpsc::Receiver<Trace>) {
7866 let toml = format!(
7867 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
7868 ladder = [\"{INCUMBENT}\"]\ngates = [\"non-empty\"]\n\
7869 [route.challenger]\nladder = [\"{CHALLENGER}\"]\npercent = {percent}\n"
7870 );
7871 let db = db.to_owned();
7872 let config = ProxyConfig::from_lookup(|k| match k {
7873 "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
7874 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
7875 "FIRSTPASS_DB" => Some(db.clone()),
7876 _ => None,
7877 })
7878 .unwrap();
7879 let mut outs = HashMap::new();
7880 for m in [INCUMBENT, CHALLENGER] {
7881 outs.insert(m.to_owned(), Ok(model_resp(m, "answer")));
7882 }
7883 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
7884 map.insert(
7885 "anthropic".to_owned(),
7886 Arc::new(MockProvider::new("anthropic", outs)),
7887 );
7888 let experiments = Arc::new(crate::experiment::ExperimentMonitor::from_config(
7889 config.routing.as_ref().unwrap(),
7890 ));
7891 let (traces, rx) = mpsc::channel(64);
7892 let state = AppState {
7893 config: Arc::new(config),
7894 http: reqwest::Client::new(),
7895 providers: ProviderRegistry::from_map(map),
7896 gate_health: Arc::new(GateHealthRegistry::new()),
7897 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
7898 paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
7899 experiments,
7900 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
7901 traces,
7902 adaptive: None,
7903 eprocess: None,
7904 bandit: None,
7905 promoter: None,
7906 verified_cache: None,
7907 predictor: None,
7908 prior: None,
7909 tenant_rate_limiter: None,
7910 spill: None,
7911 };
7912 (state, rx)
7913 }
7914
7915 async fn serve_session(
7916 state: &AppState,
7917 rx: &mut mpsc::Receiver<Trace>,
7918 session: &str,
7919 ) -> Trace {
7920 let mut headers = HeaderMap::new();
7921 headers.insert("x-firstpass-session", session.parse().unwrap());
7922 let resp = messages(
7923 State(state.clone()),
7924 Extension(TenantId("default".to_owned())),
7925 headers,
7926 user_body(),
7927 )
7928 .await;
7929 assert_eq!(resp.status(), axum::http::StatusCode::OK);
7930 rx.try_recv().expect("trace enqueued")
7931 }
7932
7933 fn session_in_arm(state: &AppState, want: firstpass_core::Arm, nth: usize) -> String {
7935 let ch = state.config.routing.as_ref().unwrap().routes[0]
7936 .challenger
7937 .as_ref()
7938 .unwrap();
7939 (0..)
7940 .map(|i| format!("sess-{i}"))
7941 .filter(|k| ch.arm(&state.config.prompt_salt, k) == want)
7942 .nth(nth)
7943 .unwrap()
7944 }
7945
7946 #[tokio::test]
7949 async fn challenger_arm_serves_its_ladder_and_both_arms_are_recorded() {
7950 use firstpass_core::Arm;
7951 let (state, mut rx) = challenger_state(50.0, "unused.db");
7952 for want in [Arm::Challenger, Arm::Incumbent] {
7953 for n in 0..5 {
7954 let session = session_in_arm(&state, want, n);
7955 let t = serve_session(&state, &mut rx, &session).await;
7956 let rec = t.experiment.as_ref().expect("experiment recorded");
7957 assert_eq!(rec.arm, want, "{session}");
7958 assert!((rec.percent - 50.0).abs() < f64::EPSILON);
7959 let expect_model = if want == Arm::Challenger {
7960 CHALLENGER
7961 } else {
7962 INCUMBENT
7963 };
7964 assert_eq!(t.attempts[0].model, expect_model, "{session}");
7965 assert_eq!(t.route_ix, Some(0));
7966 assert!(
7967 t.policy.id == "challenger-ladder@v0" || want == Arm::Incumbent,
7968 "challenger arm must not claim the incumbent's start-rung policy: {}",
7969 t.policy.id
7970 );
7971 }
7972 }
7973 let reports = state.experiments.reports();
7974 assert_eq!(reports[0].report.challenger.requests, 5);
7975 assert_eq!(reports[0].report.incumbent.requests, 5);
7976 }
7977
7978 #[tokio::test]
7981 async fn halted_experiment_stops_serving_the_challenger() {
7982 use firstpass_core::{Arm, ExperimentStatus, Verdict};
7983 let (state, mut rx) = challenger_state(50.0, "unused.db");
7984 for i in 0..500 {
7985 state.experiments.observe_outcome(
7986 "default",
7987 0,
7988 state.experiments.fingerprint(0).unwrap(),
7989 Arm::Incumbent,
7990 &format!("i{i}"),
7991 Verdict::Pass,
7992 );
7993 state.experiments.observe_outcome(
7994 "default",
7995 0,
7996 state.experiments.fingerprint(0).unwrap(),
7997 Arm::Challenger,
7998 &format!("c{i}"),
7999 Verdict::Fail,
8000 );
8001 }
8002 assert_eq!(
8003 state.experiments.status("default", 0),
8004 ExperimentStatus::Halted
8005 );
8006 let session = session_in_arm(&state, Arm::Challenger, 0);
8007 let t = serve_session(&state, &mut rx, &session).await;
8008 assert_eq!(
8009 t.attempts[0].model, INCUMBENT,
8010 "halted challenger was still served"
8011 );
8012 assert_eq!(t.experiment.unwrap().arm, Arm::Incumbent);
8013 }
8014
8015 #[tokio::test]
8025 async fn shared_log_wiring_halt_propagates_to_replica_b() {
8026 use firstpass_core::{Arm, ExperimentStatus, Verdict};
8027
8028 let shared_log = Arc::new(crate::experiment::InMemoryExperimentLog::new());
8029
8030 let (mut state_a, _rx_a) = challenger_state(50.0, "unused-a.db");
8033 {
8034 let monitor = crate::experiment::ExperimentMonitor::from_config(
8035 state_a.config.routing.as_ref().unwrap(),
8036 )
8037 .with_log(Arc::clone(&shared_log) as Arc<dyn crate::experiment::ExperimentLogBackend>);
8038 state_a.experiments = Arc::new(monitor);
8039 }
8040
8041 let (mut state_b, mut rx_b) = challenger_state(50.0, "unused-b.db");
8044 {
8045 let monitor = crate::experiment::ExperimentMonitor::from_config(
8046 state_b.config.routing.as_ref().unwrap(),
8047 )
8048 .with_log(Arc::clone(&shared_log) as Arc<dyn crate::experiment::ExperimentLogBackend>);
8049 state_b.experiments = Arc::new(monitor);
8050 }
8051
8052 let fp = state_a.experiments.fingerprint(0).unwrap().to_owned();
8053
8054 let mut halted = false;
8057 for i in 0..500 {
8058 state_a
8059 .experiments
8060 .record_outcome(
8061 "default",
8062 0,
8063 &fp,
8064 Arm::Incumbent,
8065 &format!("i{i}"),
8066 Verdict::Pass,
8067 0.10,
8068 )
8069 .await;
8070 state_a
8071 .experiments
8072 .record_outcome(
8073 "default",
8074 0,
8075 &fp,
8076 Arm::Challenger,
8077 &format!("c{i}"),
8078 Verdict::Fail,
8079 0.02,
8080 )
8081 .await;
8082 if state_a.experiments.status("default", 0) == ExperimentStatus::Halted {
8083 halted = true;
8084 break;
8085 }
8086 }
8087 assert!(halted, "experiment never halted on replica A");
8088
8089 assert!(
8091 !state_b.experiments.challenger_blocked("default", 0),
8092 "replica B must not block before syncing"
8093 );
8094
8095 state_b
8097 .experiments
8098 .sync_from_backend("default", 0, &fp)
8099 .await;
8100 assert!(
8101 state_b.experiments.challenger_blocked("default", 0),
8102 "replica B must block the challenger after syncing the halt from the shared log"
8103 );
8104
8105 let ch_session = session_in_arm(&state_b, Arm::Challenger, 999);
8108 let t = serve_session(&state_b, &mut rx_b, &ch_session).await;
8109 assert_eq!(
8110 t.experiment.as_ref().unwrap().arm,
8111 Arm::Incumbent,
8112 "a halted challenger-arm session must be rerouted to the incumbent on replica B"
8113 );
8114 assert_eq!(
8115 t.attempts[0].model, INCUMBENT,
8116 "the incumbent model must have been called, not the challenger"
8117 );
8118 }
8119
8120 #[tokio::test]
8123 async fn unavailable_experiment_history_serves_the_incumbent_only() {
8124 use firstpass_core::Arm;
8125 let (mut state, mut rx) = challenger_state(50.0, "unused.db");
8126 state.experiments = Arc::new(crate::experiment::ExperimentMonitor::unavailable(
8127 state.config.routing.as_ref().unwrap(),
8128 ));
8129 for n in 0..3 {
8130 let session = session_in_arm(&state, Arm::Challenger, n);
8131 let t = serve_session(&state, &mut rx, &session).await;
8132 assert_eq!(t.attempts[0].model, INCUMBENT);
8133 assert_eq!(t.experiment.unwrap().arm, Arm::Incumbent);
8134 }
8135 }
8136
8137 #[tokio::test]
8140 async fn challenger_arm_cannot_share_the_incumbents_cache_entry() {
8141 use firstpass_core::Arm;
8142 let (mut state, mut rx) = challenger_state(50.0, "unused.db");
8143 state.verified_cache = Some(Arc::new(crate::verified_cache::VerifiedCache::new(
8144 Duration::from_secs(60),
8145 100,
8146 )));
8147 let a = serve_session(&state, &mut rx, &session_in_arm(&state, Arm::Incumbent, 0)).await;
8149 assert!(!a.attempts.is_empty());
8150 let b = serve_session(&state, &mut rx, &session_in_arm(&state, Arm::Challenger, 0)).await;
8152 assert!(
8153 !b.attempts.is_empty(),
8154 "challenger replayed the incumbent's cached answer"
8155 );
8156 assert_eq!(b.attempts[0].model, CHALLENGER);
8157 let c = serve_session(&state, &mut rx, &session_in_arm(&state, Arm::Incumbent, 1)).await;
8160 assert!(
8161 c.attempts.is_empty(),
8162 "incumbent should have been a cache hit"
8163 );
8164 assert_eq!(c.experiment.as_ref().unwrap().arm, Arm::Incumbent);
8165 assert_eq!(c.route_ix, Some(0));
8166 let d = serve_session(&state, &mut rx, &session_in_arm(&state, Arm::Challenger, 1)).await;
8168 assert!(d.attempts.is_empty(), "challenger should hit its own entry");
8169 assert_eq!(d.experiment.as_ref().unwrap().arm, Arm::Challenger);
8170 }
8171
8172 #[tokio::test]
8175 async fn feedback_feeds_the_experiment_once_per_session() {
8176 use firstpass_core::Arm;
8177 let db = std::env::temp_dir().join(format!("firstpass-exp-{}.db", Uuid::now_v7()));
8178 let (state, mut rx) = challenger_state(50.0, &db.to_string_lossy());
8179 let session = session_in_arm(&state, Arm::Challenger, 0);
8180 let t = serve_session(&state, &mut rx, &session).await;
8181 let trace_id = t.trace_id.to_string();
8182
8183 let (tx, handle) = crate::store::open(&db).unwrap();
8185 tx.try_send(t).unwrap();
8186 drop(tx);
8187 handle.await.unwrap();
8188
8189 let send = |verdict: &'static str| {
8190 let state = state.clone();
8191 let body = Bytes::from(
8192 serde_json::json!({
8193 "trace_id": trace_id, "gate_id": "tests", "verdict": verdict, "reporter": "ci"
8194 })
8195 .to_string(),
8196 );
8197 async move {
8198 feedback(
8199 State(state),
8200 Extension(TenantId("default".to_owned())),
8201 body,
8202 )
8203 .await
8204 }
8205 };
8206 assert_eq!(
8207 send("pass").await.status(),
8208 axum::http::StatusCode::ACCEPTED
8209 );
8210 assert_eq!(
8212 send("fail").await.status(),
8213 axum::http::StatusCode::ACCEPTED
8214 );
8215 assert_eq!(
8217 send("abstain").await.status(),
8218 axum::http::StatusCode::ACCEPTED
8219 );
8220
8221 let r = &state.experiments.reports()[0].report;
8222 assert_eq!(r.challenger.outcomes, 1);
8223 assert!(
8224 (r.challenger.outcome_sum - 1.0).abs() < f64::EPSILON,
8225 "first outcome (pass) counts"
8226 );
8227 assert_eq!(r.incumbent.outcomes, 0);
8228 let _ = std::fs::remove_file(&db);
8229 }
8230
8231 async fn spawn_fake_prior_server(body: serde_json::Value) -> String {
8237 use axum::routing::post;
8238 let app = axum::Router::new().route(
8239 "/v1/systemone",
8240 post(move || {
8241 let body = body.clone();
8242 async move { axum::Json(body) }
8243 }),
8244 );
8245 let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
8246 .await
8247 .expect("bind");
8248 let addr = listener.local_addr().expect("addr");
8249 tokio::spawn(async move {
8250 let _ = axum::serve(listener, app).await;
8251 });
8252 format!("http://{addr}")
8253 }
8254
8255 fn prior_client(base_url: String, rungs: &[&str]) -> crate::prior::PriorClient {
8256 let cfg = firstpass_core::PriorConfig {
8257 provider: "typesafe".to_owned(),
8258 model: "jev-latest".to_owned(),
8259 api_key_env: "TYPESAFE_API_KEY".to_owned(),
8260 base_url,
8261 timeout_ms: 2000,
8262 strength: 10.0,
8263 rungs: rungs.iter().map(|s| (*s).to_owned()).collect(),
8264 };
8265 crate::prior::PriorClient::new(reqwest::Client::new(), cfg, "test-key".to_owned())
8266 }
8267
8268 const OPUS: &str = "anthropic/claude-opus-4-8";
8269
8270 #[tokio::test]
8271 async fn confident_prior_starts_above_zero_and_records_decision_prior() {
8272 let base_url = spawn_fake_prior_server(serde_json::json!({
8273 "answers": { "rung": { "choice": "r2", "confidence": 0.9,
8274 "probabilities": { "r0": 0.05, "r1": 0.05, "r2": 0.9 } } }
8275 }))
8276 .await;
8277 let (mut state, rx) = enforce_state(
8278 &[
8279 "anthropic/claude-haiku-4-5",
8280 "anthropic/claude-sonnet-5",
8281 OPUS,
8282 ],
8283 &["non-empty"],
8284 vec![
8285 (
8286 "anthropic/claude-haiku-4-5",
8287 Ok(model_resp("anthropic/claude-haiku-4-5", "ok")),
8288 ),
8289 (
8290 "anthropic/claude-sonnet-5",
8291 Ok(model_resp("anthropic/claude-sonnet-5", "ok")),
8292 ),
8293 (OPUS, Ok(model_resp(OPUS, "ok"))),
8294 ],
8295 );
8296 state.prior = Some(Arc::new(prior_client(
8297 base_url,
8298 &[
8299 "anthropic/claude-haiku-4-5",
8300 "anthropic/claude-sonnet-5",
8301 OPUS,
8302 ],
8303 )));
8304 let trace = run_enforce_get_trace(state, rx).await;
8305
8306 assert!(
8307 !trace.attempts.is_empty(),
8308 "at least one attempt must be recorded"
8309 );
8310 assert!(
8311 trace.attempts[0].rung >= 1,
8312 "a confident hard prior (rung 0/1 nearly hopeless) must start above rung 0, started \
8313 at {}",
8314 trace.attempts[0].rung
8315 );
8316 let decision_prior = trace
8317 .decision_prior
8318 .as_ref()
8319 .expect("a successful prior fetch must record decision_prior");
8320 assert_eq!(decision_prior.len(), 3);
8321 assert!(
8322 decision_prior[2] > decision_prior[0],
8323 "cumulative prior must be monotone non-decreasing: {decision_prior:?}"
8324 );
8325 }
8326
8327 #[tokio::test]
8328 async fn prior_server_error_fails_open_start_rung_zero_no_decision_prior() {
8329 let unreachable = "http://127.0.0.1:1".to_owned();
8332 let (mut state, rx) = enforce_state(
8333 &["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"],
8334 &["non-empty"],
8335 vec![(
8336 "anthropic/claude-haiku-4-5",
8337 Ok(model_resp("anthropic/claude-haiku-4-5", "ok")),
8338 )],
8339 );
8340 state.prior = Some(Arc::new(prior_client(
8341 unreachable,
8342 &["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"],
8343 )));
8344 let trace = run_enforce_get_trace(state, rx).await;
8345
8346 assert_eq!(
8347 trace.attempts[0].rung, 0,
8348 "a failed prior fetch must fail open to the static start rung 0"
8349 );
8350 assert!(
8351 trace.decision_prior.is_none(),
8352 "a failed prior fetch must leave decision_prior unset"
8353 );
8354 }
8355
8356 #[tokio::test]
8359 async fn gate_failed_start_rung_is_never_served_with_prior_on() {
8360 let base_url = spawn_fake_prior_server(serde_json::json!({
8361 "answers": { "rung": { "choice": "r2", "confidence": 0.9,
8362 "probabilities": { "r0": 0.05, "r1": 0.05, "r2": 0.9 } } }
8363 }))
8364 .await;
8365 let (mut state, rx) = enforce_state(
8369 &[
8370 "anthropic/claude-haiku-4-5",
8371 "anthropic/claude-sonnet-5",
8372 OPUS,
8373 ],
8374 &["non-empty"],
8375 vec![
8376 (
8377 "anthropic/claude-haiku-4-5",
8378 Ok(model_resp("anthropic/claude-haiku-4-5", "")),
8379 ),
8380 (
8381 "anthropic/claude-sonnet-5",
8382 Ok(model_resp("anthropic/claude-sonnet-5", "")),
8383 ),
8384 (OPUS, Ok(model_resp(OPUS, "ok"))),
8385 ],
8386 );
8387 state.prior = Some(Arc::new(prior_client(
8388 base_url,
8389 &[
8390 "anthropic/claude-haiku-4-5",
8391 "anthropic/claude-sonnet-5",
8392 OPUS,
8393 ],
8394 )));
8395 let trace = run_enforce_get_trace(state, rx).await;
8396
8397 assert_eq!(
8398 trace.final_.served_rung,
8399 Some(2),
8400 "only the top rung passes the gate, so it must be what's served, regardless of \
8401 which rung the prior started at"
8402 );
8403 for attempt in &trace.attempts[..trace.attempts.len() - 1] {
8404 assert_eq!(
8405 attempt.verdict,
8406 firstpass_core::Verdict::Fail,
8407 "every attempt before the served one must be a recorded gate failure, not a \
8408 silently-served output"
8409 );
8410 }
8411 assert_eq!(
8412 trace.attempts.last().unwrap().verdict,
8413 firstpass_core::Verdict::Pass
8414 );
8415 }
8416
8417 fn ux_state(db: &str) -> AppState {
8421 let (mut state, _rx) = challenger_state(50.0, db);
8422 let hash_a = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
8425 let hash_b = crate::tenant_auth::TenantKeys::hash_key("key-b").unwrap();
8426 let keys = format!("{{\"tenant-a\": {hash_a:?}, \"tenant-b\": {hash_b:?}}}");
8427 let toml = format!(
8428 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
8429 ladder = [\"{INCUMBENT}\"]\ngates = [\"non-empty\"]\n\
8430 [route.challenger]\nladder = [\"{CHALLENGER}\"]\npercent = 50\n"
8431 );
8432 let db = db.to_owned();
8433 state.config = Arc::new(
8434 ProxyConfig::from_lookup(|k| match k {
8435 "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
8436 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
8437 "FIRSTPASS_DB" => Some(db.clone()),
8438 "FIRSTPASS_REQUIRE_AUTH" => Some("true".to_owned()),
8439 "FIRSTPASS_TENANT_KEYS_JSON" => Some(keys.clone()),
8440 _ => None,
8441 })
8442 .unwrap(),
8443 );
8444 let (traces, rx) = mpsc::channel(256);
8446 std::mem::forget(rx);
8447 state.traces = traces;
8448 state
8449 }
8450
8451 #[tokio::test]
8454 async fn paired_ledger_is_seeded_from_todays_pairs_at_startup() {
8455 let db = std::env::temp_dir().join(format!("firstpass-ledger-{}.db", Uuid::now_v7()));
8456 let (tx, handle) = crate::store::open(&db).unwrap();
8457 drop(tx);
8458 handle.await.unwrap();
8459 let now = jiff::Timestamp::now();
8460 let row = |tenant: &str, usd: f64, ts: jiff::Timestamp| crate::store::ExperimentPairRow {
8461 tenant: tenant.to_owned(),
8462 route_ix: 0,
8463 fingerprint: "fp".to_owned(),
8464 trace_id: Uuid::now_v7(),
8465 x_inc: 1.0,
8466 x_ch: 1.0,
8467 challenger_cost_usd: usd,
8468 ts,
8469 };
8470 let two_days_ago = now - jiff::SignedDuration::from_hours(48);
8471 for r in [
8472 row("tenant-a", 0.40, now),
8473 row("tenant-a", 0.70, now),
8474 row("tenant-a", 9.00, two_days_ago),
8475 row("tenant-b", 0.10, now),
8476 ] {
8477 crate::store::append_experiment_pair(&db, &r).unwrap();
8478 }
8479 let toml = format!(
8480 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
8481 ladder = [\"{INCUMBENT}\"]\ngates = [\"json-valid\"]\n\
8482 [route.challenger]\nladder = [\"{CHALLENGER}\"]\nmode = \"paired\"\n\
8483 max_usd_per_day = 1.0\n"
8484 );
8485 let db_s = db.to_string_lossy().into_owned();
8486 let cfg = ProxyConfig::from_lookup(|k| match k {
8487 "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
8488 "FIRSTPASS_DB" => Some(db_s.clone()),
8489 _ => None,
8490 })
8491 .unwrap();
8492 let ledger = build_paired_ledger(&cfg);
8493 assert!((ledger.spent_today("tenant-a", 0, now) - 1.10).abs() < 1e-9);
8494 assert!((ledger.spent_today("tenant-b", 0, now) - 0.10).abs() < 1e-9);
8495 assert!(
8496 !ledger.may_spend("tenant-a", 0, 1.0, now),
8497 "cap survives restart"
8498 );
8499 assert!(ledger.may_spend("tenant-b", 0, 1.0, now));
8500 let _ = std::fs::remove_file(&db);
8501 }
8502
8503 #[tokio::test]
8506 async fn experiments_endpoint_reports_paired_mode_and_tenant_spend() {
8507 use tower::ServiceExt;
8508 let db = std::env::temp_dir().join(format!("firstpass-ux-paired-{}.db", Uuid::now_v7()));
8509 let mut state = ux_state(&db.to_string_lossy());
8510 let toml = format!(
8511 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
8512 ladder = [\"{INCUMBENT}\"]\ngates = [\"json-valid\"]\n\
8513 [route.challenger]\nladder = [\"{CHALLENGER}\"]\nmode = \"paired\"\n\
8514 sample_rate = 0.25\nmax_usd_per_day = 2.0\n"
8515 );
8516 let mut cfg = (*state.config).clone();
8517 cfg.routing = Some(firstpass_core::Config::parse(&toml).expect("paired config parses"));
8518 state.config = Arc::new(cfg);
8519 let now = jiff::Timestamp::now();
8520 state.paired_ledger.debit("tenant-a", 0, 0.5, now);
8521 state.paired_ledger.debit("tenant-b", 0, 1.25, now);
8522 let router = app(state).expect("router");
8523
8524 let resp = router
8525 .clone()
8526 .oneshot(ux_get("/v1/experiments", Some("tenant-a.key-a")))
8527 .await
8528 .unwrap();
8529 assert_eq!(resp.status(), axum::http::StatusCode::OK);
8530 let a = body_json(resp).await;
8531 let c = &a["configured"][0];
8532 assert_eq!(c["mode"], "paired", "{a}");
8533 assert_eq!(c["paired"]["sample_rate"], 0.25);
8534 assert_eq!(c["paired"]["max_usd_per_day"], 2.0);
8535 assert_eq!(c["paired"]["spent_today_usd"], 0.5, "tenant-a's spend only");
8536
8537 let resp = router
8538 .oneshot(ux_get("/v1/experiments", Some("tenant-b.key-b")))
8539 .await
8540 .unwrap();
8541 let b = body_json(resp).await;
8542 assert_eq!(b["configured"][0]["paired"]["spent_today_usd"], 1.25);
8543 let _ = std::fs::remove_file(&db);
8544 }
8545
8546 #[tokio::test]
8550 async fn paired_challenger_that_fails_its_gate_is_recorded_as_a_failure() {
8551 use tower::ServiceExt;
8552 let db = std::env::temp_dir().join(format!("firstpass-paired-fail-{}.db", Uuid::now_v7()));
8553 let (tx, handle) = crate::store::open(&db).unwrap();
8555 drop(tx);
8556 handle.await.unwrap();
8557 let (mut state, _rx) = challenger_state(50.0, &db.to_string_lossy());
8558 let toml = format!(
8559 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
8560 ladder = [\"{INCUMBENT}\"]\ngates = [\"json-valid\"]\n\
8561 [route.challenger]\nladder = [\"{CHALLENGER}\"]\nmode = \"paired\"\n\
8562 max_usd_per_day = 5.0\n"
8563 );
8564 let mut cfg = (*state.config).clone();
8565 cfg.routing = Some(firstpass_core::Config::parse(&toml).expect("paired config parses"));
8566 state.experiments = Arc::new(crate::experiment::ExperimentMonitor::from_config(
8567 cfg.routing.as_ref().unwrap(),
8568 ));
8569 state.config = Arc::new(cfg);
8570 let mut outs = HashMap::new();
8571 outs.insert(
8572 INCUMBENT.to_owned(),
8573 Ok(model_resp(INCUMBENT, "{\"ok\": 1}")),
8574 );
8575 outs.insert(
8576 CHALLENGER.to_owned(),
8577 Ok(model_resp(CHALLENGER, "not json")),
8578 );
8579 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
8580 map.insert(
8581 "anthropic".to_owned(),
8582 Arc::new(MockProvider::new("anthropic", outs)),
8583 );
8584 state.providers = ProviderRegistry::from_map(map);
8585 let router = app(state).expect("router");
8586
8587 let req = axum::http::Request::builder()
8588 .method("POST")
8589 .uri("/v1/messages")
8590 .body(Body::from(user_body()))
8591 .unwrap();
8592 let resp = router.oneshot(req).await.unwrap();
8593 assert_eq!(resp.status(), axum::http::StatusCode::OK);
8594
8595 let mut pairs = Vec::new();
8597 for _ in 0..200 {
8598 pairs = crate::store::load_all_experiment_pairs(&db).unwrap_or_default();
8599 if !pairs.is_empty() {
8600 break;
8601 }
8602 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
8603 }
8604 assert_eq!(pairs.len(), 1, "one pair recorded");
8605 assert_eq!(pairs[0].x_inc, 1.0, "incumbent passed json-valid");
8606 assert_eq!(
8607 pairs[0].x_ch, 0.0,
8608 "challenger failed json-valid on every rung"
8609 );
8610 let _ = std::fs::remove_file(&db);
8611 }
8612
8613 #[tokio::test]
8616 async fn paired_challenger_error_is_a_failure_and_arm_health_is_reported() {
8617 use tower::ServiceExt;
8618 let db = std::env::temp_dir().join(format!("firstpass-paired-err-{}.db", Uuid::now_v7()));
8619 let (tx, handle) = crate::store::open(&db).unwrap();
8620 drop(tx);
8621 handle.await.unwrap();
8622 let (mut state, _rx) = challenger_state(50.0, &db.to_string_lossy());
8623 let toml = format!(
8624 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
8625 ladder = [\"{INCUMBENT}\"]\ngates = [\"json-valid\"]\n\
8626 [route.challenger]\nladder = [\"{CHALLENGER}\"]\nmode = \"paired\"\n\
8627 max_usd_per_day = 5.0\n"
8628 );
8629 let mut cfg = (*state.config).clone();
8630 cfg.routing = Some(firstpass_core::Config::parse(&toml).expect("paired config parses"));
8631 state.experiments = Arc::new(crate::experiment::ExperimentMonitor::from_config(
8632 cfg.routing.as_ref().unwrap(),
8633 ));
8634 state.config = Arc::new(cfg);
8635 let mut cut = model_resp(INCUMBENT, "{\"ok\": 1}");
8637 cut.raw = serde_json::json!({"stop_reason": "max_tokens"});
8638 let mut outs = HashMap::new();
8639 outs.insert(INCUMBENT.to_owned(), Ok(cut));
8640 outs.insert(
8641 CHALLENGER.to_owned(),
8642 Err(ProviderError::Transport("timed out".into())),
8643 );
8644 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
8645 map.insert(
8646 "anthropic".to_owned(),
8647 Arc::new(MockProvider::new("anthropic", outs)),
8648 );
8649 state.providers = ProviderRegistry::from_map(map);
8650 let monitor = state.experiments.clone();
8651 let router = app(state).expect("router");
8652
8653 let req = axum::http::Request::builder()
8654 .method("POST")
8655 .uri("/v1/messages")
8656 .body(Body::from(user_body()))
8657 .unwrap();
8658 let resp = router.oneshot(req).await.unwrap();
8659 assert_eq!(resp.status(), axum::http::StatusCode::OK);
8660
8661 let mut pairs = Vec::new();
8662 for _ in 0..200 {
8663 pairs = crate::store::load_all_experiment_pairs(&db).unwrap_or_default();
8664 if !pairs.is_empty() {
8665 break;
8666 }
8667 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
8668 }
8669 assert_eq!(
8670 pairs.len(),
8671 1,
8672 "the challenger's error still records a pair"
8673 );
8674 assert_eq!(pairs[0].x_inc, 1.0);
8675 assert_eq!(pairs[0].x_ch, 0.0, "an error is a failure");
8676 let health = monitor.reports()[0].health;
8677 assert_eq!(health.incumbent.truncated, 1, "{health:?}");
8678 assert_eq!(health.incumbent.errors, 0, "{health:?}");
8679 assert_eq!(health.challenger.errors, 1, "{health:?}");
8680 let _ = std::fs::remove_file(&db);
8681 }
8682
8683 #[tokio::test]
8685 async fn halted_paired_challenger_stops_spending() {
8686 use tower::ServiceExt;
8687 let db = std::env::temp_dir().join(format!("firstpass-paired-halt-{}.db", Uuid::now_v7()));
8688 let (tx, handle) = crate::store::open(&db).unwrap();
8689 drop(tx);
8690 handle.await.unwrap();
8691 let (mut state, _rx) = challenger_state(50.0, &db.to_string_lossy());
8692 let toml = format!(
8693 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
8694 ladder = [\"{INCUMBENT}\"]\ngates = [\"json-valid\"]\n\
8695 [route.challenger]\nladder = [\"{CHALLENGER}\"]\nmode = \"paired\"\n\
8696 max_usd_per_day = 5.0\n"
8697 );
8698 let mut cfg = (*state.config).clone();
8699 cfg.routing = Some(firstpass_core::Config::parse(&toml).expect("paired config parses"));
8700 state.experiments = Arc::new(crate::experiment::ExperimentMonitor::from_config(
8701 cfg.routing.as_ref().unwrap(),
8702 ));
8703 state.config = Arc::new(cfg);
8704 let fingerprint = state.experiments.fingerprint(0).unwrap().to_owned();
8705 for _ in 0..500 {
8706 if state.experiments.challenger_blocked("default", 0) {
8707 break;
8708 }
8709 state
8710 .experiments
8711 .record_pair("default", 0, &fingerprint, 1.0, 0.0, 0.01, 0.001)
8712 .await;
8713 }
8714 assert!(
8715 state.experiments.challenger_blocked("default", 0),
8716 "a challenger that always loses halts"
8717 );
8718 let mut outs = HashMap::new();
8719 outs.insert(
8720 INCUMBENT.to_owned(),
8721 Ok(model_resp(INCUMBENT, "{\"ok\": 1}")),
8722 );
8723 outs.insert(
8724 CHALLENGER.to_owned(),
8725 Ok(model_resp(CHALLENGER, "{\"ok\": 1}")),
8726 );
8727 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
8728 map.insert(
8729 "anthropic".to_owned(),
8730 Arc::new(MockProvider::new("anthropic", outs)),
8731 );
8732 state.providers = ProviderRegistry::from_map(map);
8733 let ledger = state.paired_ledger.clone();
8734 let router = app(state).expect("router");
8735
8736 let req = axum::http::Request::builder()
8737 .method("POST")
8738 .uri("/v1/messages")
8739 .body(Body::from(user_body()))
8740 .unwrap();
8741 let resp = router.oneshot(req).await.unwrap();
8742 assert_eq!(
8743 resp.status(),
8744 axum::http::StatusCode::OK,
8745 "incumbent still serves"
8746 );
8747 tokio::time::sleep(std::time::Duration::from_millis(300)).await;
8748 assert!(
8749 crate::store::load_all_experiment_pairs(&db)
8750 .unwrap()
8751 .is_empty(),
8752 "no pair after the halt"
8753 );
8754 assert!(
8755 ledger.may_spend("default", 0, 1e-9, jiff::Timestamp::now()),
8756 "nothing debited after the halt"
8757 );
8758 let _ = std::fs::remove_file(&db);
8759 }
8760
8761 #[tokio::test]
8764 async fn shadow_reports_no_pass_when_every_rung_fails_its_gate() {
8765 let (mut state, _rx) = challenger_state(50.0, ":memory:");
8766 let mut outs = HashMap::new();
8767 outs.insert(INCUMBENT.to_owned(), Ok(model_resp(INCUMBENT, "not json")));
8768 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
8769 map.insert(
8770 "anthropic".to_owned(),
8771 Arc::new(MockProvider::new("anthropic", outs)),
8772 );
8773 state.providers = ProviderRegistry::from_map(map);
8774 let cfg = firstpass_core::Config::parse(&format!(
8775 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
8776 ladder = [\"{INCUMBENT}\"]\ngates = [\"json-valid\"]\n"
8777 ))
8778 .unwrap();
8779 let shadow = firstpass_core::rollout::Shadow {
8780 sample_rate: 1.0,
8781 max_usd_per_day: 5.0,
8782 };
8783 let signal = evaluate_shadow(
8784 &state,
8785 &cfg.routes[0],
8786 shadow,
8787 0,
8788 &user_body(),
8789 Auth::from_headers(&HeaderMap::new()),
8790 &Features::new(TaskKind::Other),
8791 "t".to_owned(),
8792 "s".to_owned(),
8793 "anthropic.messages",
8794 0.0,
8795 )
8796 .await;
8797 assert_eq!(signal.skipped, None);
8798 assert!(!signal.would_pass, "a best attempt is not a pass");
8799 assert_eq!(signal.would_serve_rung, None);
8800 }
8801
8802 fn ux_get(uri: &str, bearer: Option<&str>) -> axum::http::Request<Body> {
8803 let mut b = axum::http::Request::builder().method("GET").uri(uri);
8804 if let Some(k) = bearer {
8805 b = b.header("authorization", format!("Bearer {k}"));
8806 }
8807 b.body(Body::empty()).unwrap()
8808 }
8809
8810 #[tokio::test]
8814 async fn experiments_endpoint_reports_per_arm_counts_scoped_to_the_tenant() {
8815 use firstpass_core::Arm;
8816 use tower::ServiceExt;
8817 let db = std::env::temp_dir().join(format!("firstpass-ux-{}.db", Uuid::now_v7()));
8818 let mut state = ux_state(&db.to_string_lossy());
8819 let (traces_tx, mut traces_rx) = mpsc::channel(256);
8821 state.traces = traces_tx;
8822 let router = app(state.clone()).expect("router");
8823
8824 let mut sessions = Vec::new();
8826 for arm in [Arm::Incumbent, Arm::Challenger] {
8827 for n in 0..6 {
8828 sessions.push((arm, session_in_arm(&state, arm, n)));
8829 }
8830 }
8831 for (_, session) in &sessions {
8832 let req = axum::http::Request::builder()
8833 .method("POST")
8834 .uri("/v1/messages")
8835 .header("authorization", "Bearer tenant-a.key-a")
8836 .header("x-firstpass-session", session.as_str())
8837 .body(Body::from(user_body()))
8838 .unwrap();
8839 let resp = router.clone().oneshot(req).await.unwrap();
8840 assert_eq!(resp.status(), axum::http::StatusCode::OK);
8841 }
8842 let mut traces = Vec::new();
8843 while let Ok(t) = traces_rx.try_recv() {
8844 traces.push(t);
8845 }
8846 assert_eq!(traces.len(), 12);
8847
8848 let (tx, handle) = crate::store::open(&db).unwrap();
8850 for t in &traces {
8851 tx.try_send(t.clone()).unwrap();
8852 }
8853 drop(tx);
8854 handle.await.unwrap();
8855
8856 let mut challenger_seen = 0;
8858 for t in &traces {
8859 let arm = t.experiment.as_ref().unwrap().arm;
8860 let verdict = if arm == Arm::Challenger {
8861 challenger_seen += 1;
8862 if challenger_seen <= 2 { "fail" } else { "pass" }
8863 } else {
8864 "pass"
8865 };
8866 let req = axum::http::Request::builder()
8867 .method("POST")
8868 .uri("/v1/feedback")
8869 .header("authorization", "Bearer tenant-a.key-a")
8870 .body(Body::from(
8871 serde_json::json!({
8872 "trace_id": t.trace_id, "gate_id": "tests",
8873 "verdict": verdict, "reporter": "ci"
8874 })
8875 .to_string(),
8876 ))
8877 .unwrap();
8878 let resp = router.clone().oneshot(req).await.unwrap();
8879 assert_eq!(resp.status(), axum::http::StatusCode::ACCEPTED);
8880 }
8881
8882 let resp = router
8884 .clone()
8885 .oneshot(ux_get("/v1/experiments", Some("tenant-a.key-a")))
8886 .await
8887 .unwrap();
8888 assert_eq!(resp.status(), axum::http::StatusCode::OK);
8889 let a = body_json(resp).await;
8890 let exps = a["experiments"].as_array().unwrap();
8891 assert_eq!(exps.len(), 1, "{a}");
8892 let r = &exps[0]["report"];
8893 assert_eq!(r["status"], "undecided");
8894 assert_eq!(r["pairs"], 6);
8895 assert_eq!(r["incumbent"]["requests"], 6);
8896 assert_eq!(r["incumbent"]["outcomes"], 6);
8897 assert_eq!(r["incumbent"]["outcome_sum"], 6.0);
8898 assert_eq!(r["challenger"]["requests"], 6);
8899 assert_eq!(r["challenger"]["outcomes"], 6);
8900 assert_eq!(r["challenger"]["outcome_sum"], 4.0);
8901 assert_eq!(exps[0]["challenger_ladder"][0], CHALLENGER);
8902 assert_eq!(exps[0]["recommendation"], "keep_running");
8903 assert_eq!(a["configured"][0]["unavailable"], false);
8904 assert_eq!(a["configured"][0]["mode"], "live");
8905 assert!(a["configured"][0]["paired"].is_null());
8906
8907 let resp = router
8909 .clone()
8910 .oneshot(ux_get("/v1/experiments", Some("tenant-b.key-b")))
8911 .await
8912 .unwrap();
8913 assert_eq!(resp.status(), axum::http::StatusCode::OK);
8914 let b = body_json(resp).await;
8915 assert_eq!(b["experiments"].as_array().unwrap().len(), 0, "{b}");
8916 assert!(!b.to_string().contains("tenant-a"));
8917
8918 let resp = router
8920 .clone()
8921 .oneshot(ux_get("/v1/receipts?limit=3", Some("tenant-a.key-a")))
8922 .await
8923 .unwrap();
8924 let ra = body_json(resp).await;
8925 assert_eq!(ra["receipts"].as_array().unwrap().len(), 3);
8926 assert_eq!(ra["summary"]["requests"], 12, "summary spans all receipts");
8927 let first = &ra["receipts"][0];
8928 assert!(first["experiment"]["arm"].is_string());
8929 assert_eq!(first["hash"].as_str().unwrap().len(), 64);
8930 assert!(first["prev_hash"].is_string());
8931 assert!(first["attempts"][0]["gates"][0]["gate_id"].is_string());
8932 let resp = router
8933 .clone()
8934 .oneshot(ux_get("/v1/receipts", Some("tenant-b.key-b")))
8935 .await
8936 .unwrap();
8937 let rb = body_json(resp).await;
8938 assert_eq!(rb["receipts"].as_array().unwrap().len(), 0);
8939 assert_eq!(rb["summary"]["requests"], 0);
8940
8941 for bearer in [None, Some("tenant-a.wrong")] {
8943 let resp = router
8944 .clone()
8945 .oneshot(ux_get("/v1/experiments", bearer))
8946 .await
8947 .unwrap();
8948 assert_eq!(resp.status(), axum::http::StatusCode::UNAUTHORIZED);
8949 }
8950 let _ = std::fs::remove_file(&db);
8951 }
8952
8953 #[test]
8956 fn receipts_summary_json_keeps_losses_signed() {
8957 let s = summary_json(&crate::store::ReceiptsSummary {
8958 requests: 2,
8959 spent_usd: 6.0,
8960 baseline_usd: 6.0,
8961 first_pass: 1,
8962 escalated: 1,
8963 gates: [3, 1, 2],
8964 truncated: 4,
8965 });
8966 assert_eq!(s["requests"], 2);
8967 assert_eq!(s["truncated"], 4);
8968 assert_eq!(s["first_pass"], 1);
8969 assert_eq!(s["escalated"], 1);
8970 assert_eq!(s["gates"]["pass"], 3);
8971 assert_eq!(s["gates"]["fail"], 1);
8972 assert_eq!(s["gates"]["abstain"], 2);
8973 assert!((s["saved_usd"].as_f64().unwrap()).abs() < 1e-9);
8974 let loss = summary_json(&crate::store::ReceiptsSummary {
8975 spent_usd: 5.0,
8976 baseline_usd: 3.0,
8977 ..Default::default()
8978 });
8979 assert!(
8980 loss["saved_usd"].as_f64().unwrap() < 0.0,
8981 "loss stays negative"
8982 );
8983 }
8984
8985 #[tokio::test]
8988 async fn panel_is_served_with_a_csp_pinning_its_own_script() {
8989 use sha2::Digest as _;
8990 let resp = panel().await.into_response();
8991 let csp = resp
8992 .headers()
8993 .get(axum::http::header::CONTENT_SECURITY_POLICY)
8994 .expect("panel sets a CSP")
8995 .to_str()
8996 .unwrap()
8997 .to_owned();
8998 let body = PANEL_HTML.split_once("<script>").unwrap().1;
8999 let script = body.split_once("</script>").unwrap().0;
9000 assert!(script.len() > 1000, "extracted the panel script");
9003 let want = format!(
9004 "script-src 'sha256-{}'",
9005 base64_std(&sha2::Sha256::digest(script.as_bytes()))
9006 );
9007 assert!(csp.contains(&want), "{csp}");
9008 for directive in [
9009 "default-src 'none'",
9010 "connect-src 'self'",
9011 "frame-ancestors 'none'",
9012 ] {
9013 assert!(csp.contains(directive), "{directive} missing: {csp}");
9014 }
9015 assert!(!csp.contains("unsafe-eval") && !csp.contains("script-src 'unsafe-inline'"));
9016 assert_eq!(
9017 PANEL_HTML.matches("<script").count(),
9018 1,
9019 "one inline script"
9020 );
9021 assert_eq!(
9022 resp.headers()[axum::http::header::X_CONTENT_TYPE_OPTIONS],
9023 "nosniff"
9024 );
9025 }
9026
9027 #[test]
9028 fn base64_std_matches_rfc4648_vectors() {
9029 for (input, want) in [
9030 ("", ""),
9031 ("f", "Zg=="),
9032 ("fo", "Zm8="),
9033 ("foo", "Zm9v"),
9034 ("foob", "Zm9vYg=="),
9035 ("fooba", "Zm9vYmE="),
9036 ("foobar", "Zm9vYmFy"),
9037 ] {
9038 assert_eq!(base64_std(input.as_bytes()), want, "{input:?}");
9039 }
9040 assert_eq!(base64_std(&[0xfb, 0xff]), "+/8=");
9041 }
9042
9043 #[test]
9046 fn panel_wires_the_experiments_view_and_has_no_external_references() {
9047 assert!(
9048 PANEL_HTML.contains("/v1/experiments"),
9049 "fetches the endpoint"
9050 );
9051 assert!(PANEL_HTML.contains("id=\"view-experiments\""));
9052 assert!(PANEL_HTML.contains("id=\"exp-list\""));
9053 assert!(
9054 PANEL_HTML.contains("[route.challenger]"),
9055 "empty state teaches the config"
9056 );
9057 assert!(
9058 PANEL_HTML.contains("data-copy"),
9059 "adopt snippet has a copy button"
9060 );
9061 assert!(
9062 PANEL_HTML.contains("document.hidden"),
9063 "polling pauses in a hidden tab"
9064 );
9065 assert!(PANEL_HTML.contains("prefers-reduced-motion"));
9066 assert!(PANEL_HTML.contains("prefers-color-scheme: dark"));
9067 for needle in [
9070 "http://",
9071 "https://",
9072 "src=\"//",
9073 "href=\"//",
9074 "@import",
9075 "<link",
9076 ] {
9077 assert!(!PANEL_HTML.contains(needle), "external reference: {needle}");
9078 }
9079 for tag in ["<script", "<img", "<iframe", "<source"] {
9080 for (i, _) in PANEL_HTML.match_indices(tag) {
9081 let end = PANEL_HTML[i..]
9082 .find('>')
9083 .map_or(PANEL_HTML.len(), |e| i + e);
9084 assert!(
9085 !PANEL_HTML[i..end].contains("src="),
9086 "{tag} loads an external source: {}",
9087 &PANEL_HTML[i..end]
9088 );
9089 }
9090 }
9091 }
9092
9093 #[test]
9097 fn panel_interpolates_only_escaped_or_numeric_values() {
9098 let start = PANEL_HTML.find("<script>").expect("script") + "<script>".len();
9099 let js = &PANEL_HTML[start..PANEL_HTML.find("</script>").expect("end")];
9100 let bytes = js.as_bytes();
9101 let mut i = 0;
9102 let mut count = 0;
9103 while let Some(off) = js[i..].find("${") {
9104 let begin = i + off + 2;
9105 let (mut depth, mut j) = (1, begin);
9106 while depth > 0 && j < bytes.len() {
9107 match bytes[j] {
9108 b'{' => depth += 1,
9109 b'}' => depth -= 1,
9110 _ => {}
9111 }
9112 j += 1;
9113 }
9114 let expr = js[begin..j - 1].trim();
9115 assert!(
9116 ["esc(", "usd(", "num(", "pct(", "cls(", "h_"]
9117 .iter()
9118 .any(|p| expr.starts_with(p)),
9119 "unescaped interpolation in the panel: ${{{expr}}}"
9120 );
9121 count += 1;
9122 i = begin;
9123 }
9124 assert!(
9125 count > 40,
9126 "the scan must actually see the templates ({count})"
9127 );
9128 assert!(js.contains("const esc ="), "the escaper must exist");
9129 }
9130}