1use std::sync::Arc;
5use std::time::Instant;
6
7use axum::body::Body;
8use axum::extract::{Request, State};
9use axum::http::HeaderMap;
10use axum::middleware::Next;
11use axum::response::{IntoResponse, Response};
12use axum::routing::{get, post};
13use axum::{Extension, Json, Router};
14use bytes::Bytes;
15use firstpass_core::features::{hour_bucket, token_bucket};
16use firstpass_core::hashchain::sha256_hex;
17use firstpass_core::{
18 Attempt, DeferredVerdict, Dialect, FEATURE_VERSION, Features, FinalOutcome, GENESIS_HASH, Mode,
19 ModelRef, PolicyRef, ProbeRegime, ProbeSignal, RequestInfo, RoutingMode, Score, ServedFrom,
20 TaskKind, Trace, Verdict,
21};
22use serde::Deserialize;
23use serde_json::Value;
24use std::future::Future;
25use std::time::Duration;
26use tokio::sync::mpsc::error::TrySendError;
27use uuid::Uuid;
28
29use crate::config::ProxyConfig;
30use crate::error::ProxyError;
31use crate::gate::{GateHealthRegistry, aggregate_with_policy, resolve_gates};
32use crate::provider::{Auth, ChatMessage, ModelRequest, ModelResponse, ProviderRegistry};
33use crate::router::{EnforceCtx, EngineOutcome, route_enforce};
34use crate::store;
35use crate::tenant_auth::{TenantId, auth_middleware};
36use crate::upstream::{forward_anthropic, forward_anthropic_streaming};
37use firstpass_core::Route;
38use firstpass_core::trace::ShadowSignal;
39
40#[derive(Clone)]
43pub struct AppState {
44 pub config: Arc<ProxyConfig>,
46 pub http: reqwest::Client,
48 pub providers: ProviderRegistry,
50 pub gate_health: Arc<GateHealthRegistry>,
52 pub shadow_ledger: Arc<crate::shadow::ShadowLedger>,
55 pub guardrails: Arc<crate::guard::GuardrailRegistry>,
58 pub traces: store::TraceSender,
60 pub adaptive: Option<Arc<std::sync::Mutex<firstpass_core::conformal::AdaptiveConformal>>>,
64 pub bandit: Option<Arc<std::sync::Mutex<crate::bandit::StartRungBandit>>>,
69 pub promoter: Option<Arc<crate::affinity::SessionPromoter>>,
74 pub verified_cache: Option<Arc<dyn crate::verified_cache::CacheStore>>,
78 pub predictor: Option<Arc<std::sync::Mutex<firstpass_core::PassPredictor>>>,
84 pub tenant_rate_limiter: Option<Arc<governor::DefaultKeyedRateLimiter<String>>>,
88 pub spill: Option<store::SpillHandle>,
92}
93
94#[must_use]
110fn routing_cfg_condense(state: &AppState) -> Option<&firstpass_core::config::CondenseConfig> {
111 state
112 .config
113 .routing
114 .as_ref()
115 .and_then(|r| r.escalation.condense.as_ref())
116}
117
118#[must_use]
123fn mode_now() -> jiff::Timestamp {
125 jiff::Timestamp::now()
126}
127
128fn cache_hit_trace(
136 ctx: &crate::router::EnforceCtx<'_>,
137 entry: &crate::verified_cache::Entry,
138 ts: jiff::Timestamp,
139) -> Trace {
140 Trace {
141 trace_id: uuid::Uuid::now_v7(),
142 prev_hash: firstpass_core::GENESIS_HASH.to_owned(),
143 tenant_id: ctx.tenant_id.clone(),
144 session_id: ctx.session_id.clone(),
145 ts,
146 mode: Mode::Enforce,
147 policy: PolicyRef {
148 id: format!("{}+cache", ctx.policy_id),
149 explore: false,
150 propensity: None,
151 mode_profile: None,
152 },
153 request: RequestInfo {
154 api: ctx.api.clone(),
155 prompt_hash: ctx.prompt_hash.clone(),
156 features: ctx.features.clone(),
157 },
158 attempts: Vec::new(),
159 final_: FinalOutcome {
160 served_rung: None,
161 served_from: ServedFrom::Cache,
162 total_cost_usd: 0.0,
166 gate_cost_usd: 0.0,
167 total_latency_ms: 0,
168 escalations: 0,
169 counterfactual_baseline_usd: entry.original_cost_usd,
170 savings_usd: entry.original_cost_usd,
171 cache_source: Some(entry.source),
172 },
173 deferred: Vec::new(),
174 predicted_pass: None,
175 probe: None,
176 elastic: None,
177 rollout: None,
178 shadow: None,
179 route_ix: None,
180 }
181}
182
183pub async fn build_verified_cache(
184 config: &ProxyConfig,
185) -> Result<Option<Arc<dyn crate::verified_cache::CacheStore>>, String> {
186 let Some(c) = config
187 .routing
188 .as_ref()
189 .and_then(|r| r.escalation.verified_cache.as_ref())
190 else {
191 return Ok(None);
192 };
193 if let Some(url) = c.redis_url.as_deref() {
194 #[cfg(feature = "redis-cache")]
195 {
196 let store = crate::verified_cache::RedisStore::connect(url, c.ttl_secs).await?;
197 tracing::info!("verified cache: shared via redis");
198 return Ok(Some(Arc::new(store)));
199 }
200 #[cfg(not(feature = "redis-cache"))]
201 {
202 let _ = url;
203 return Err(
204 "[escalation.verified_cache] redis_url is set, but this binary was built without \
205 the `redis-cache` feature. Rebuild with `--features redis-cache`, or remove \
206 redis_url to use the in-process cache."
207 .to_owned(),
208 );
209 }
210 }
211 Ok(Some(Arc::new(crate::verified_cache::VerifiedCache::new(
212 std::time::Duration::from_secs(c.ttl_secs),
213 c.max_entries,
214 ))))
215}
216
217pub async fn build_promoter_async(
218 config: &ProxyConfig,
219) -> Result<Option<Arc<crate::affinity::SessionPromoter>>, String> {
220 let Some(sp) = config
221 .routing
222 .as_ref()
223 .and_then(|r| r.escalation.session_promotion.as_ref())
224 else {
225 return Ok(None);
226 };
227 let promoter = crate::affinity::SessionPromoter::new(sp.clone()).map_err(|e| e.to_string())?;
228 if let Some(url) = sp.redis_url.as_deref() {
229 #[cfg(feature = "redis-cache")]
230 {
231 let ttl = sp.window_duration().map_err(|e| e.to_string())?.as_secs();
232 let store = crate::affinity::RedisPromotionStore::connect(url, ttl).await?;
233 tracing::info!("session promotion: shared via redis");
234 return Ok(Some(Arc::new(promoter.with_shared(Arc::new(store)))));
235 }
236 #[cfg(not(feature = "redis-cache"))]
237 {
238 let _ = url;
239 return Err(
240 "[escalation.session_promotion] redis_url is set, but this binary was \
241 built without the `redis-cache` feature. Rebuild with \
242 `--features redis-cache`, or remove redis_url to keep promotion \
243 in-process."
244 .to_owned(),
245 );
246 }
247 }
248 Ok(Some(Arc::new(promoter)))
249}
250
251#[must_use]
252pub fn build_tenant_rate_limiter(
253 config: &ProxyConfig,
254) -> Option<Arc<governor::DefaultKeyedRateLimiter<String>>> {
255 let per_sec = config.tenant_rate_per_sec?;
256 Some(Arc::new(governor::RateLimiter::keyed(
257 governor::Quota::per_second(per_sec),
258 )))
259}
260
261pub async fn tenant_rate_limit_middleware(
265 State(state): State<AppState>,
266 Extension(tenant): Extension<TenantId>,
267 req: Request,
268 next: Next,
269) -> Response {
270 if let Some(limiter) = &state.tenant_rate_limiter
271 && limiter.check_key(&tenant.0).is_err()
272 {
273 return ProxyError::RateLimited.into_response();
274 }
275 next.run(req).await
276}
277
278impl std::fmt::Debug for AppState {
279 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
280 f.debug_struct("AppState")
281 .field("config", &self.config)
282 .finish_non_exhaustive()
283 }
284}
285
286fn offer_trace(traces: &store::TraceSender, spill: Option<&store::SpillHandle>, trace: Trace) {
303 record_trace_metrics(&trace);
304 match traces.try_send(trace) {
305 Ok(()) => {}
306 Err(TrySendError::Full(t)) => {
307 if let Some(handle) = spill {
308 match store::append_to_spill(handle, &t) {
309 Ok(()) => {
310 metrics::counter!("firstpass_receipts_spilled_total").increment(1);
311 }
312 Err(e) => {
313 tracing::error!(%e, "durable mode: spill write failed; trace lost");
314 metrics::counter!("firstpass_traces_dropped_total").increment(1);
315 }
316 }
317 } else {
318 tracing::warn!("trace channel full; dropping trace (writer behind under load)");
319 metrics::counter!("firstpass_traces_dropped_total").increment(1);
320 }
321 }
322 Err(TrySendError::Closed(_)) => {
323 tracing::warn!("trace writer is gone; dropping trace");
324 }
325 }
326}
327
328fn record_trace_metrics(trace: &Trace) {
332 if trace.mode == Mode::Enforce {
333 metrics::histogram!("firstpass_enforce_latency_ms")
334 .record(trace.final_.total_latency_ms as f64);
335 if trace.final_.escalations > 0 {
336 metrics::counter!("firstpass_escalations_total")
337 .increment(u64::from(trace.final_.escalations));
338 }
339 }
340 let served_from = match trace.final_.served_from {
341 ServedFrom::Attempt => "attempt",
342 ServedFrom::BestAttempt => "best_attempt",
343 ServedFrom::Error => "error",
344 ServedFrom::Cache => "cache",
348 };
349 metrics::counter!("firstpass_served_total", "served_from" => served_from).increment(1);
350 if trace.final_.served_from == ServedFrom::Error {
351 let provider = trace
356 .attempts
357 .last()
358 .map_or_else(|| "unknown".to_owned(), |a| a.provider.clone());
359 metrics::counter!("firstpass_upstream_failures_total", "provider" => provider).increment(1);
360 }
361
362 for a in &trace.attempts {
368 metrics::histogram!(
369 "firstpass_attempt_latency_ms",
370 "provider" => a.provider.clone(),
371 "rung" => a.rung.to_string()
372 )
373 .record(a.latency_ms as f64);
374 metrics::counter!(
375 "firstpass_attempt_total",
376 "provider" => a.provider.clone(),
377 "rung" => a.rung.to_string()
378 )
379 .increment(1);
380 metrics::gauge!(
381 "firstpass_attempt_cost_usd_total",
382 "provider" => a.provider.clone(),
383 "rung" => a.rung.to_string()
384 )
385 .increment(a.cost_usd);
386
387 for g in &a.gates {
391 let verdict = match g.verdict {
392 Verdict::Pass => "pass",
393 Verdict::Fail => "fail",
394 Verdict::Abstain => "abstain",
395 };
396 metrics::counter!(
397 "firstpass_gate_verdict_total",
398 "gate_id" => g.gate_id.clone(),
399 "verdict" => verdict
400 )
401 .increment(1);
402 metrics::histogram!("firstpass_gate_latency_ms", "gate_id" => g.gate_id.clone())
403 .record(g.ms as f64);
404 metrics::gauge!("firstpass_gate_cost_by_id_usd_total", "gate_id" => g.gate_id.clone())
405 .increment(g.cost_usd);
406 }
407 }
408 metrics::gauge!("firstpass_cost_usd_total").increment(trace.final_.total_cost_usd);
412 metrics::gauge!("firstpass_gate_cost_usd_total").increment(trace.final_.gate_cost_usd);
413 metrics::gauge!("firstpass_baseline_usd_total")
414 .increment(trace.final_.counterfactual_baseline_usd);
415 metrics::gauge!("firstpass_savings_usd_total").increment(trace.final_.savings_usd);
416 if let Some(rung) = trace.final_.served_rung {
418 let model = trace
419 .attempts
420 .iter()
421 .find(|a| a.rung == rung)
422 .map(|a| a.model.clone())
423 .unwrap_or_else(|| "unknown".to_owned());
424 metrics::counter!(
425 "firstpass_served_rung_total",
426 "rung" => rung.to_string(),
427 "model" => model
428 )
429 .increment(1);
430 }
431}
432
433const MAX_BODY_BYTES: usize = 16 * 1024 * 1024;
437
438pub(crate) fn u01(seed: u128) -> f64 {
449 let lo = splitmix64_finalise(seed as u64);
450 let hi = splitmix64_finalise((seed >> 64) as u64);
451 ((lo ^ hi) >> 11) as f64 * (1.0_f64 / (1u64 << 53) as f64)
453}
454
455fn splitmix64_finalise(mut z: u64) -> u64 {
456 z = z.wrapping_add(0x9E37_79B9_7F4A_7C15);
457 z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
458 z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
459 z ^ (z >> 31)
460}
461
462#[must_use]
469pub(crate) fn epsilon_propensity(chosen: u32, greedy: u32, epsilon: f64, k: usize) -> f64 {
470 let greedy_term = if chosen == greedy { 1.0 - epsilon } else { 0.0 };
471 greedy_term + epsilon / k as f64
472}
473
474pub fn app(state: AppState) -> Result<Router, ProxyError> {
481 crate::metrics::install()?;
482 let max_concurrency = state.config.max_concurrency;
483
484 let business = Router::new()
493 .route("/v1/messages", post(messages))
494 .route("/v1/chat/completions", post(chat_completions))
495 .route("/v1/responses", post(responses))
496 .route("/v1/feedback", post(feedback))
497 .route("/v1/capabilities", get(capabilities))
498 .route("/v1/models", get(models))
499 .route("/v1/receipts", get(receipts))
502 .layer(axum::middleware::from_fn_with_state(
503 state.clone(),
504 tenant_rate_limit_middleware,
505 ))
506 .layer(axum::middleware::from_fn_with_state(
507 state.clone(),
508 auth_middleware,
509 ));
510
511 Ok(Router::new()
512 .merge(business)
513 .route("/healthz", get(healthz))
514 .route("/metrics", get(crate::metrics::handler))
515 .route("/panel", get(panel))
518 .layer(axum::extract::DefaultBodyLimit::max(MAX_BODY_BYTES))
520 .layer(tower::limit::GlobalConcurrencyLimitLayer::new(
523 max_concurrency,
524 ))
525 .with_state(state))
526}
527
528async fn healthz() -> impl IntoResponse {
530 Json(serde_json::json!({
536 "status": "ok",
537 "service": "firstpass",
538 "version": env!("CARGO_PKG_VERSION"),
539 }))
540}
541
542const PANEL_HTML: &str = include_str!("panel.html");
546
547async fn receipts(
553 State(state): State<AppState>,
554 Extension(TenantId(tenant)): Extension<TenantId>,
555 axum::extract::Query(q): axum::extract::Query<ReceiptsQuery>,
556) -> Response {
557 let limit = q.limit.unwrap_or(20).clamp(1, 200);
559 let db = state.config.db_path.clone();
560 let traces =
561 match tokio::task::spawn_blocking(move || crate::store::load_tenant_traces(&db, &tenant))
562 .await
563 {
564 Ok(Ok(t)) => t,
565 _ => Vec::new(),
568 };
569 let items: Vec<serde_json::Value> = traces
570 .iter()
571 .rev()
572 .take(limit)
573 .map(|t| {
574 let f = &t.final_;
575 serde_json::json!({
576 "trace_id": t.trace_id,
577 "served_rung": f.served_rung,
578 "served_model": f.served_rung
583 .and_then(|r| t.attempts.iter().find(|a| a.rung == r))
584 .map(|a| a.model.clone()),
585 "attempts": t.attempts.iter().map(|a| serde_json::json!({
586 "rung": a.rung,
587 "model": a.model,
588 "verdict": a.verdict,
589 "cost_usd": a.cost_usd,
590 "latency_ms": a.latency_ms,
591 })).collect::<Vec<_>>(),
592 "total_cost_usd": f.total_cost_usd,
593 "baseline_usd": f.counterfactual_baseline_usd,
594 "savings_usd": f.savings_usd,
595 })
596 })
597 .collect();
598 axum::Json(serde_json::json!({ "receipts": items })).into_response()
599}
600
601#[derive(Debug, serde::Deserialize)]
603struct ReceiptsQuery {
604 limit: Option<usize>,
606}
607
608async fn panel() -> impl IntoResponse {
614 (
615 [(axum::http::header::CONTENT_TYPE, "text/html; charset=utf-8")],
616 PANEL_HTML,
617 )
618}
619
620async fn capabilities(State(state): State<AppState>) -> impl IntoResponse {
621 let (ladder, gates) = state
624 .config
625 .routing
626 .as_ref()
627 .and_then(|c| c.routes.iter().find(|r| r.mode == Mode::Enforce))
628 .map(|r| (r.ladder.clone(), r.gates.clone()))
629 .unwrap_or_default();
630 let routing_modes: Vec<serde_json::Value> = RoutingMode::ALL
631 .iter()
632 .map(|m| {
633 let p = m.preset();
634 serde_json::json!({
635 "name": m.as_str(),
636 "description": p.description,
637 "tradeoff": p.tradeoff,
638 })
639 })
640 .collect();
641 Json(serde_json::json!({
642 "service": "firstpass",
643 "version": env!("CARGO_PKG_VERSION"),
644 "feature_version": FEATURE_VERSION,
645 "modes": ["observe", "enforce"],
646 "routing_modes": routing_modes,
647 "wire_apis": ["anthropic.messages", "openai.chat_completions", "openai.responses"],
648 "ladder": ladder,
649 "gates": gates,
650 "feedback_api": "POST /v1/feedback",
651 "offboarding": "unset ANTHROPIC_BASE_URL (or OPENAI_BASE_URL for OpenAI clients)",
652 }))
653}
654
655async fn models(State(state): State<AppState>) -> impl IntoResponse {
667 Json(model_list(state.config.routing.as_ref()))
668}
669
670fn model_list(routing: Option<&firstpass_core::config::Config>) -> serde_json::Value {
673 let prices = routing.map(|c| c.price_table()).unwrap_or_default();
674 let mut seen = std::collections::HashSet::new();
677 let data: Vec<serde_json::Value> = routing
678 .map(|c| c.routes.as_slice())
679 .unwrap_or_default()
680 .iter()
681 .flat_map(|r| r.ladder.iter())
682 .filter(|id| seen.insert(id.as_str()))
683 .map(|id| {
684 let price = prices.get(id);
685 serde_json::json!({
686 "id": id,
687 "object": "model",
688 "created": 0,
691 "owned_by": id.split_once('/').map_or("firstpass", |(p, _)| p),
692 "firstpass": {
693 "input_per_mtok": price.map(|p| p.input_per_mtok),
694 "output_per_mtok": price.map(|p| p.output_per_mtok),
695 },
696 })
697 })
698 .collect();
699 serde_json::json!({ "object": "list", "data": data })
700}
701
702#[derive(Debug, Deserialize)]
704struct FeedbackRequest {
705 trace_id: String,
707 gate_id: String,
709 verdict: String,
711 #[serde(default)]
713 score: Option<f64>,
714 reporter: String,
716}
717
718async fn feedback(
723 State(state): State<AppState>,
724 Extension(TenantId(tenant)): Extension<TenantId>,
725 body: Bytes,
726) -> Response {
727 let req: FeedbackRequest = match serde_json::from_slice(&body) {
728 Ok(r) => r,
729 Err(e) => {
730 return ProxyError::BadRequest(format!("invalid feedback body: {e}")).into_response();
731 }
732 };
733 let verdict = match req.verdict.as_str() {
734 "pass" => Verdict::Pass,
735 "fail" => Verdict::Fail,
736 "abstain" => Verdict::Abstain,
737 other => {
738 return ProxyError::BadRequest(format!("unknown verdict {other:?}")).into_response();
739 }
740 };
741 let score = match req.score {
742 Some(s) => match Score::new(s) {
743 Ok(sc) => Some(sc),
744 Err(_) => {
745 return ProxyError::BadRequest(format!("score {s} out of range [0,1]"))
746 .into_response();
747 }
748 },
749 None => None,
750 };
751
752 let db = state.config.db_path.clone();
753
754 let (db_check, tenant_check, tid_check) = (db.clone(), tenant.clone(), req.trace_id.clone());
759 match tokio::task::spawn_blocking(move || {
760 store::trace_exists(&db_check, &tenant_check, &tid_check)
761 })
762 .await
763 {
764 Ok(Ok(true)) => {}
765 Ok(Ok(false)) => {
766 return ProxyError::NotFound(format!("unknown trace_id {:?}", req.trace_id))
767 .into_response();
768 }
769 Ok(Err(e)) => {
770 tracing::error!(%e, "feedback: trace_exists check failed");
771 return ProxyError::Internal(e.to_string()).into_response();
772 }
773 Err(e) => {
774 tracing::error!(%e, "feedback: trace_exists task panicked");
775 return ProxyError::Internal(e.to_string()).into_response();
776 }
777 }
778
779 if verdict == Verdict::Fail
784 && let Some(cache) = state.verified_cache.as_ref()
785 {
786 let reported = req.trace_id.parse::<uuid::Uuid>().ok();
791 let origin = {
792 let (d, t, i) = (db.clone(), tenant.clone(), req.trace_id.clone());
793 tokio::task::spawn_blocking(move || store::trace_cache_source(&d, &t, &i))
794 .await
795 .ok()
796 .and_then(Result::ok)
797 .flatten()
798 .flatten()
799 };
800 let mut dropped = 0usize;
801 for id in reported.into_iter().chain(origin) {
802 dropped += cache.retract(id).await;
803 }
804 if dropped > 0 {
805 tracing::info!(
806 trace = %req.trace_id,
807 ?origin,
808 dropped,
809 "verified cache: retracted a disproven answer"
810 );
811 metrics::counter!("firstpass_cache_retractions_total").increment(dropped as u64);
812 }
813 }
814
815 let feedback_signal = match verdict {
817 Verdict::Pass => Some(true),
818 Verdict::Fail => Some(false),
819 Verdict::Abstain => None,
820 };
821 let dv = DeferredVerdict {
822 gate_id: req.gate_id,
823 verdict,
824 score,
825 reported_at: jiff::Timestamp::now(),
826 reporter: req.reporter,
827 };
828 let trace_id = req.trace_id.clone();
829 match tokio::task::spawn_blocking(move || store::append_deferred(&db, &req.trace_id, &dv)).await
830 {
831 Ok(Ok(())) => {
832 if let (Some(a), Some(correct)) = (state.adaptive.as_ref(), feedback_signal)
834 && let Ok(mut g) = a.lock()
835 {
836 g.observe_served(correct);
837 metrics::gauge!("firstpass_serve_threshold").set(g.threshold());
838 metrics::gauge!("firstpass_realized_served_failure")
839 .set(g.realized_served_failure());
840 }
841
842 let attributed_route = tokio::task::spawn_blocking({
847 let (db, ten, tid) = (
848 state.config.db_path.clone(),
849 tenant.clone(),
850 trace_id.clone(),
851 );
852 move || store::trace_route(&db, &ten, &tid)
853 })
854 .await
855 .ok()
856 .and_then(Result::ok)
857 .flatten()
858 .flatten()
859 .unwrap_or(0) as usize;
860
861 if let (Some(cfg), Some(correct)) = (
866 state.config.routing.as_ref().and_then(|r| r.guardrail),
867 feedback_signal,
868 ) {
869 let cooldown = state
870 .config
871 .routing
872 .as_ref()
873 .map_or(3_600, |r| r.guardrail_cooldown_secs);
874 let reaction = state.guardrails.record(
875 &tenant,
876 attributed_route,
877 &cfg,
878 correct,
879 jiff::Timestamp::now().as_second(),
880 cooldown,
881 );
882 match &reaction {
883 crate::guard::Reaction::Demoted(v) => {
884 metrics::counter!("firstpass_guardrail_demotions_total").increment(1);
885 tracing::error!(
886 tenant = %tenant,
887 n = v.n,
888 rate = v.rate,
889 bound = v.bound,
890 alpha = cfg.alpha,
891 "GUARDRAIL: served-failure bound exceeded target — route demoted to observe; traffic now serves as it would without Firstpass"
892 );
893 }
894 crate::guard::Reaction::Alarmed(v) => {
895 metrics::counter!("firstpass_guardrail_breaches_total").increment(1);
896 tracing::error!(
897 tenant = %tenant,
898 n = v.n,
899 rate = v.rate,
900 bound = v.bound,
901 alpha = cfg.alpha,
902 "GUARDRAIL: served-failure bound exceeded target (alarm only — routing unchanged)"
903 );
904 }
905 crate::guard::Reaction::None => {}
906 }
907 }
908 (
909 axum::http::StatusCode::ACCEPTED,
910 Json(serde_json::json!({ "status": "recorded", "trace_id": trace_id })),
911 )
912 .into_response()
913 }
914 Ok(Err(e)) => {
915 tracing::error!(%e, "feedback: append_deferred failed");
916 ProxyError::Internal(e.to_string()).into_response()
917 }
918 Err(e) => {
919 tracing::error!(%e, "feedback: append_deferred task panicked");
920 ProxyError::Internal(e.to_string()).into_response()
921 }
922 }
923}
924
925const SESSION_HEADER: &str = "x-firstpass-session";
928
929const AGENT_HEADER: &str = "x-firstpass-agent";
931const SUBAGENT_HEADER: &str = "x-firstpass-subagent";
933#[expect(
939 clippy::too_many_arguments,
940 reason = "mirrors the enforce context it evaluates; a wrapper struct would only move the list"
941)]
942fn spawn_shadow(
943 state: &AppState,
944 route: &firstpass_core::Route,
945 route_ix: usize,
946 body: Bytes,
947 auth: Auth,
948 features: Features,
949 tenant: String,
950 session_id: String,
951 api: &str,
952) {
953 let Some(shadow) = route.shadow else {
954 return;
955 };
956 if !shadow.sampled(&state.config.prompt_salt, &session_id) {
959 return;
960 }
961 let (state, route, api) = (state.clone(), route.clone(), api.to_owned());
962 tokio::spawn(async move {
963 let signal = evaluate_shadow(
964 &state, &route, shadow, route_ix, &body, auth, &features, tenant, session_id, &api, 0.0,
965 )
966 .await;
967 tracing::debug!(
969 would_pass = signal.would_pass,
970 projected_usd = signal.projected_cost_usd,
971 skipped = ?signal.skipped,
972 "shadow evaluation complete"
973 );
974 });
975}
976
977#[expect(
984 clippy::too_many_arguments,
985 reason = "mirrors the enforce context; grouping them into a struct would only move the list"
986)]
987async fn evaluate_shadow(
988 state: &AppState,
989 route: &firstpass_core::Route,
990 shadow: firstpass_core::rollout::Shadow,
991 route_ix: usize,
992 body: &Bytes,
993 auth: Auth,
994 features: &Features,
995 tenant: String,
996 session_id: String,
997 api: &str,
998 actual_cost_usd: f64,
999) -> ShadowSignal {
1000 let skipped = |why: &str| ShadowSignal {
1001 would_serve_rung: None,
1002 would_pass: false,
1003 projected_cost_usd: 0.0,
1004 actual_cost_usd,
1005 skipped: Some(why.to_owned()),
1006 };
1007
1008 let now = jiff::Timestamp::now();
1009 if !state
1010 .shadow_ledger
1011 .may_spend(&tenant, route_ix, shadow.max_usd_per_day, now)
1012 {
1013 return skipped("budget_exhausted");
1016 }
1017
1018 let Some(base_request) = parse_model_request(body) else {
1019 return skipped("unparseable_request");
1020 };
1021 let gate_defs = state
1022 .config
1023 .routing
1024 .as_ref()
1025 .map_or(&[][..], |cfg| &cfg.gate_defs);
1026 let gates = resolve_gates(
1027 &route.gates,
1028 gate_defs,
1029 &state.providers,
1030 &auth,
1031 &state.config.prices,
1032 );
1033 let (budget, max_rungs) = state
1034 .config
1035 .routing
1036 .as_ref()
1037 .map_or((None, u32::MAX), |cfg| {
1038 (
1039 cfg.budget.per_request_usd,
1040 cfg.escalation.max_rungs_per_request,
1041 )
1042 });
1043
1044 let mut base_request = base_request;
1045 base_request.cache_prefix = state
1049 .config
1050 .routing
1051 .as_ref()
1052 .is_some_and(|r| r.escalation.prompt_cache);
1053 let ctx = EnforceCtx {
1054 condense: routing_cfg_condense(state),
1055 ladder: &route.ladder,
1056 gates: &gates,
1057 health: &state.gate_health,
1058 base_request: &base_request,
1059 providers: &state.providers,
1060 auth: &auth,
1061 prices: &state.config.prices,
1062 budget_per_request_usd: budget,
1063 max_rungs,
1064 speculation: 0,
1067 serve_threshold: None,
1068 elastic: None,
1069 features: features.clone(),
1070 start_rung: 0,
1071 tenant_id: tenant.clone(),
1072 session_id,
1073 prompt_hash: prompt_hash(&state.config.prompt_salt, body),
1074 api: api.to_owned(),
1075 policy_id: "shadow".to_owned(),
1076 };
1077
1078 let (outcome, trace) = route_enforce(ctx).await;
1079 let spent = trace.final_.total_cost_usd;
1080 state.shadow_ledger.debit(&tenant, route_ix, spent, now);
1081
1082 let would_pass = matches!(outcome, EngineOutcome::Served(_));
1085 let would_serve_rung = if would_pass {
1086 trace.final_.served_rung
1087 } else {
1088 None
1089 };
1090 ShadowSignal {
1091 would_serve_rung,
1092 would_pass,
1093 projected_cost_usd: spent,
1094 actual_cost_usd,
1095 skipped: None,
1096 }
1097}
1098
1099const MODE_PROFILE_HEADER: &str = "x-firstpass-mode";
1102
1103fn resolve_mode(headers: &HeaderMap, route: &Route, config: &ProxyConfig) -> RoutingMode {
1112 if let Some(val) = header_str(headers, MODE_PROFILE_HEADER) {
1114 match val.trim().to_ascii_lowercase().as_str() {
1115 "observe" => return RoutingMode::Observe,
1116 "cost" => return RoutingMode::Cost,
1117 "balanced" => return RoutingMode::Balanced,
1118 "quality" => return RoutingMode::Quality,
1119 "latency" => return RoutingMode::Latency,
1120 "max" => return RoutingMode::Max,
1121 other => {
1122 tracing::warn!(
1123 value = other,
1124 "unknown x-firstpass-mode value; ignoring \
1125 (valid: observe|cost|balanced|quality|latency|max)"
1126 );
1127 }
1128 }
1129 }
1130 if let Some(m) = route.routing_mode {
1132 return m;
1133 }
1134 config.default_routing_mode
1136}
1137
1138async fn messages(
1143 State(state): State<AppState>,
1144 Extension(TenantId(tenant)): Extension<TenantId>,
1145 headers: HeaderMap,
1146 body: Bytes,
1147) -> Response {
1148 let session_header = header_str(&headers, SESSION_HEADER);
1149
1150 if let Some(routing) = state.config.routing.as_ref() {
1153 let features = extract_features(&headers, &body);
1154 if let Some(route) = routing
1155 .route_for(&features)
1156 .filter(|r| r.mode == Mode::Enforce && !r.ladder.is_empty())
1157 {
1158 let route_ix = routing
1161 .routes
1162 .iter()
1163 .position(|r| std::ptr::eq(r, route))
1164 .unwrap_or(0);
1165 let route = route.clone();
1168 let routing_mode = resolve_mode(&headers, &route, &state.config);
1170 if routing_mode == RoutingMode::Observe {
1172 return observe_passthrough(state, headers, body, session_header, tenant).await;
1173 }
1174 if state
1178 .guardrails
1179 .is_demoted(&tenant, route_ix, jiff::Timestamp::now().as_second())
1180 {
1181 return observe_passthrough(state, headers, body, session_header, tenant).await;
1182 }
1183 if let Some(rollout) = route.rollout.as_ref() {
1188 let key_value = match rollout.key {
1189 firstpass_core::RolloutKey::Session => session_header
1190 .clone()
1191 .unwrap_or_else(|| firstpass_core::rollout::request_identity(&body)),
1192 firstpass_core::RolloutKey::Request => {
1198 firstpass_core::rollout::request_identity(&body)
1199 }
1200 firstpass_core::RolloutKey::Tenant => tenant.to_string(),
1201 };
1202 let decision =
1203 firstpass_core::rollout::decide(&state.config.prompt_salt, rollout, &key_value);
1204 if !decision.enforced {
1205 let shadow_ctx = route.shadow.map(|_| {
1207 (
1208 body.clone(),
1209 Auth::from_headers(&headers),
1210 features.clone(),
1211 tenant.clone(),
1212 session_header
1213 .clone()
1214 .unwrap_or_else(|| Uuid::now_v7().to_string()),
1215 )
1216 });
1217 let resp =
1218 observe_passthrough(state.clone(), headers, body, session_header, tenant)
1219 .await;
1220 if let Some((b, auth, feats, ten, sess)) = shadow_ctx {
1223 spawn_shadow(
1224 &state,
1225 &route,
1226 route_ix,
1227 b,
1228 auth,
1229 feats,
1230 ten,
1231 sess,
1232 "anthropic",
1233 );
1234 }
1235 return resp;
1236 }
1237 }
1238 if enforce_can_handle(
1239 &features,
1240 &body,
1241 routing.escalation.enforce_structured,
1242 &route.ladder,
1243 &state.providers,
1244 Dialect::Anthropic,
1245 ) {
1246 return handle_enforce(
1247 &state,
1248 &headers,
1249 &body,
1250 features,
1251 &route,
1252 route_ix,
1253 session_header,
1254 tenant,
1255 routing_mode,
1256 )
1257 .await;
1258 }
1259 tracing::info!(
1263 "enforce route matched but structured request can't be routed faithfully (flag/ladder); serving via observe passthrough"
1264 );
1265 }
1266 }
1267 observe_passthrough(state, headers, body, session_header, tenant).await
1268}
1269
1270fn enforce_can_handle(
1284 features: &Features,
1285 body: &[u8],
1286 enforce_structured: bool,
1287 ladder: &[String],
1288 providers: &crate::provider::ProviderRegistry,
1289 inbound: Dialect,
1290) -> bool {
1291 let structured = features.tool_count > 0
1292 || features.has_images
1293 || match inbound {
1294 Dialect::Anthropic => messages_have_tool_blocks(body),
1295 Dialect::Openai => openai_messages_have_tool_calls(body),
1296 Dialect::Gemini => false,
1297 };
1298 if !structured {
1299 return true;
1300 }
1301 if !enforce_structured {
1302 return false;
1303 }
1304 let all_verbatim = ladder.iter().all(|rung| {
1306 let provider_id = rung.split('/').next().unwrap_or_default();
1307 providers
1308 .get(provider_id)
1309 .is_some_and(|p| p.carries_structured_verbatim(inbound))
1310 });
1311 if all_verbatim {
1312 return true;
1313 }
1314 if inbound == Dialect::Openai && !openai_has_http_images(body) {
1318 let all_anthropic = ladder.iter().all(|rung| {
1319 let pid = rung.split('/').next().unwrap_or_default();
1320 providers
1321 .get(pid)
1322 .is_some_and(|p| p.carries_structured_verbatim(Dialect::Anthropic))
1323 });
1324 if all_anthropic {
1325 return true;
1326 }
1327 }
1328 false
1329}
1330
1331fn messages_have_tool_blocks(body: &[u8]) -> bool {
1334 serde_json::from_slice::<Value>(body)
1335 .ok()
1336 .and_then(|json| {
1337 json.get("messages")
1338 .and_then(Value::as_array)
1339 .map(|messages| messages.iter().any(message_has_tool_block))
1340 })
1341 .unwrap_or(false)
1342}
1343
1344fn message_has_tool_block(message: &Value) -> bool {
1346 message
1347 .get("content")
1348 .and_then(Value::as_array)
1349 .is_some_and(|blocks| {
1350 blocks.iter().any(|block| {
1351 matches!(
1352 block.get("type").and_then(Value::as_str),
1353 Some("tool_use" | "tool_result")
1354 )
1355 })
1356 })
1357}
1358
1359fn is_stream_request(body: &[u8]) -> bool {
1361 serde_json::from_slice::<Value>(body)
1362 .ok()
1363 .and_then(|json| json.get("stream").and_then(Value::as_bool))
1364 .unwrap_or(false)
1365}
1366
1367fn header_str(headers: &HeaderMap, name: &str) -> Option<String> {
1369 headers
1370 .get(name)
1371 .and_then(|v| v.to_str().ok())
1372 .map(str::to_owned)
1373}
1374
1375fn extract_features(headers: &HeaderMap, body: &[u8]) -> Features {
1378 let (_model, tool_count, has_images) = request_features(body);
1379 let mut f = Features::new(TaskKind::Other);
1380 f.agent = header_str(headers, AGENT_HEADER);
1381 f.subagent = header_str(headers, SUBAGENT_HEADER);
1382 f.tool_count = tool_count;
1383 f.has_images = has_images;
1384 f.prompt_token_bucket = token_bucket(body.len() as u64);
1387 f.hour_bucket = hour_bucket(jiff::Timestamp::now());
1388 f
1389}
1390
1391#[allow(clippy::too_many_arguments)]
1394async fn handle_enforce(
1395 state: &AppState,
1396 headers: &HeaderMap,
1397 body: &Bytes,
1398 features: Features,
1399 route: &Route,
1400 route_ix: usize,
1403 session_header: Option<String>,
1404 tenant: String,
1405 routing_mode: RoutingMode,
1406) -> Response {
1407 if is_stream_request(body) {
1415 let (tx, rx) = tokio::sync::oneshot::channel::<Result<Value, ProxyError>>();
1416 let (state_c, headers_c, body_c, route_c) =
1417 (state.clone(), headers.clone(), body.clone(), route.clone());
1418 tokio::spawn(async move {
1419 let out = enforce_pipeline(
1420 &state_c,
1421 &headers_c,
1422 &body_c,
1423 features,
1424 &route_c,
1425 route_ix,
1426 session_header,
1427 tenant,
1428 routing_mode,
1429 )
1430 .await;
1431 let _ = tx.send(out);
1432 });
1433 return sse_keepalive_response(rx, anthropic_sse_from_message);
1434 }
1435 match enforce_pipeline(
1436 state,
1437 headers,
1438 body,
1439 features,
1440 route,
1441 route_ix,
1442 session_header,
1443 tenant,
1444 routing_mode,
1445 )
1446 .await
1447 {
1448 Ok(message) => (axum::http::StatusCode::OK, Json(message)).into_response(),
1449 Err(e) => e.into_response(),
1450 }
1451}
1452
1453#[allow(clippy::too_many_arguments)] async fn enforce_pipeline_inner(
1458 state: &AppState,
1459 body: &Bytes,
1460 base_request: ModelRequest,
1461 auth: Auth,
1462 features: Features,
1463 route: &Route,
1464 session_header: Option<String>,
1465 tenant: String,
1466 api: &str,
1467 routing_mode: RoutingMode,
1468 route_ix: usize,
1469) -> Result<ModelResponse, ProxyError> {
1470 let gate_defs = state
1471 .config
1472 .routing
1473 .as_ref()
1474 .map_or(&[][..], |cfg| &cfg.gate_defs);
1475 let gates = resolve_gates(
1476 &route.gates,
1477 gate_defs,
1478 &state.providers,
1479 &auth,
1480 &state.config.prices,
1481 );
1482 let session_id = session_header.unwrap_or_else(|| Uuid::now_v7().to_string());
1483 let (budget, max_rungs, speculation, serve_threshold, elastic) =
1484 match state.config.routing.as_ref() {
1485 Some(cfg) => (
1486 cfg.budget.per_request_usd,
1487 cfg.escalation.max_rungs_per_request,
1488 cfg.escalation.speculation,
1489 cfg.escalation.serve_threshold,
1490 cfg.escalation.elastic.as_ref(),
1491 ),
1492 None => (None, 3, 0, None, None),
1493 };
1494 let serve_threshold = state
1497 .adaptive
1498 .as_ref()
1499 .and_then(|a| a.lock().ok().map(|g| g.threshold()))
1500 .or(serve_threshold);
1501
1502 let preset = routing_mode.preset();
1505 let max_rungs = if let Some(delta) = preset.max_rungs_delta {
1506 (max_rungs as i32 + delta).max(1) as u32
1507 } else {
1508 max_rungs
1509 };
1510 let speculation = preset.speculation.unwrap_or(speculation);
1511
1512 let bandit_ctx = crate::bandit::ContextBucket::from_features(&features);
1516
1517 let (greedy_rung, base_policy_id, ts_propensity) = {
1521 let (chosen, ts_p) = state
1522 .bandit
1523 .as_ref()
1524 .and_then(|b| b.lock().ok())
1525 .map(|mut b| {
1526 b.choose_start_with_propensity(&bandit_ctx, &route.ladder, &state.config.prices)
1527 })
1528 .unwrap_or((0, None));
1529 let policy = if ts_p.is_some() {
1530 "bandit@v2-ts".to_owned()
1531 } else if chosen > 0 {
1532 "bandit@v1".to_owned()
1533 } else {
1534 "static-ladder@v0".to_owned()
1535 };
1536 (chosen, policy, ts_p)
1537 };
1538
1539 let exploration_epsilon = state
1544 .config
1545 .routing
1546 .as_ref()
1547 .and_then(|cfg| cfg.escalation.exploration.as_ref())
1548 .map(|e| e.epsilon);
1549
1550 let (start_rung, policy_id, explore_flag, propensity) = if ts_propensity.is_some() {
1551 if exploration_epsilon.is_some() {
1554 tracing::warn!(
1555 "bandit.algorithm = thompson already logs propensities; \
1556 [escalation.exploration] epsilon is ignored"
1557 );
1558 }
1559 (greedy_rung, base_policy_id, false, ts_propensity)
1560 } else if let Some(epsilon) = exploration_epsilon {
1561 let k = route.ladder.len().max(1);
1562 let u = u01(Uuid::now_v7().as_u128());
1564 let (chosen, eps_branch) = if u < epsilon {
1565 let idx = ((u / epsilon) * k as f64) as u32;
1567 (idx.min(k as u32 - 1), true)
1568 } else {
1569 (greedy_rung, false)
1570 };
1571 let p = epsilon_propensity(chosen, greedy_rung, epsilon, k);
1572 (chosen, format!("{base_policy_id}+eps"), eps_branch, Some(p))
1573 } else {
1574 (greedy_rung, base_policy_id, false, None)
1575 };
1576
1577 let promotion = state.promoter.as_ref().cloned();
1582 let promotion = match promotion {
1585 Some(p) => {
1586 p.decide_async(&tenant, &session_id, std::time::Instant::now())
1587 .await
1588 }
1589 None => crate::affinity::Decision::Cold,
1590 };
1591 let start_rung = start_rung.max(promotion.start_rung());
1592 let promo_key = state
1595 .promoter
1596 .as_ref()
1597 .map(|_| (tenant.clone(), session_id.clone()));
1598
1599 let start_rung = if preset.start_at_top {
1602 route.ladder.len().saturating_sub(1) as u32
1603 } else {
1604 start_rung
1605 };
1606
1607 let speculation = match state
1613 .config
1614 .routing
1615 .as_ref()
1616 .and_then(|cfg| cfg.escalation.speculation_band)
1617 {
1618 Some([lo, hi]) if speculation > 0 => {
1619 let estimate = state
1620 .bandit
1621 .as_ref()
1622 .and_then(|b| b.lock().ok())
1623 .and_then(|b| b.pass_estimate(&bandit_ctx, start_rung));
1624 match estimate {
1625 Some(p) if p < lo || p > hi => {
1626 metrics::counter!("firstpass_speculation_skipped_total").increment(1);
1627 0
1628 }
1629 _ => speculation,
1630 }
1631 }
1632 _ => speculation,
1633 };
1634
1635 if state.bandit.is_some() {
1637 metrics::counter!(
1638 "firstpass_bandit_start_rung",
1639 "rung" => start_rung.to_string()
1640 )
1641 .increment(1);
1642 }
1643
1644 let mut base_request = base_request;
1645 base_request.cache_prefix = state
1649 .config
1650 .routing
1651 .as_ref()
1652 .is_some_and(|r| r.escalation.prompt_cache);
1653 let ctx = EnforceCtx {
1654 condense: routing_cfg_condense(state),
1655 ladder: &route.ladder,
1656 gates: &gates,
1657 health: &state.gate_health,
1658 base_request: &base_request,
1659 providers: &state.providers,
1660 auth: &auth,
1661 prices: &state.config.prices,
1662 budget_per_request_usd: budget,
1663 max_rungs,
1664 speculation,
1665 serve_threshold,
1666 elastic,
1667 features,
1668 start_rung,
1669 tenant_id: tenant,
1672 session_id,
1673 prompt_hash: prompt_hash(&state.config.prompt_salt, body),
1674 api: api.to_owned(),
1675 policy_id,
1676 };
1677
1678 if let Some(cache) = state.verified_cache.as_ref()
1683 && let Some(entry) = cache
1684 .get(
1685 &crate::verified_cache::VerifiedCache::compose_key(
1686 &ctx.tenant_id,
1687 &ctx.prompt_hash,
1688 &route.ladder,
1689 ),
1690 std::time::Instant::now(),
1691 )
1692 .await
1693 {
1694 metrics::counter!("firstpass_cache_hits_total").increment(1);
1695 let trace = cache_hit_trace(&ctx, &entry, mode_now());
1696 offer_trace(&state.traces, state.spill.as_ref(), trace);
1697 return Ok(entry.response);
1698 }
1699
1700 let (outcome, mut trace) = route_enforce(ctx).await;
1701
1702 trace.route_ix = Some(u32::try_from(route_ix).unwrap_or(0));
1706 trace.policy.explore = explore_flag;
1709 trace.policy.propensity = propensity;
1710 if routing_mode != RoutingMode::Balanced {
1713 trace.policy.mode_profile = Some(routing_mode.as_str().to_owned());
1714 }
1715
1716 if let Some(bandit) = state.bandit.as_ref()
1720 && let Ok(mut b) = bandit.lock()
1721 {
1722 for attempt in &trace.attempts {
1723 b.observe(&bandit_ctx, attempt.rung, attempt.verdict);
1724 }
1725 }
1726
1727 if let Some(promoter) = state.promoter.as_ref()
1732 && let Some((tenant_key, session_key)) = promo_key.as_ref()
1733 && let Some(served) = trace.final_.served_rung
1734 {
1735 promoter
1736 .record_async(
1737 tenant_key,
1738 session_key,
1739 served,
1740 served > start_rung,
1741 std::time::Instant::now(),
1742 )
1743 .await;
1744 }
1745
1746 if let Some(predictor) = state.predictor.as_ref()
1752 && let Ok(mut p) = predictor.lock()
1753 {
1754 let predicted = p.predict(&trace.request.features, start_rung);
1756 for attempt in &trace.attempts {
1757 match attempt.verdict {
1758 Verdict::Pass => p.update(&trace.request.features, attempt.rung, true),
1759 Verdict::Fail => p.update(&trace.request.features, attempt.rung, false),
1760 Verdict::Abstain => {} }
1762 }
1763 trace.predicted_pass = Some(predicted);
1764 metrics::histogram!("firstpass_predictor_pass_prob").record(predicted);
1765 }
1766
1767 if let Some(probe_cfg) = state
1774 .config
1775 .routing
1776 .as_ref()
1777 .and_then(|c| c.escalation.probe)
1778 && u01(Uuid::now_v7().as_u128()) < probe_cfg.sample_rate
1779 {
1780 let probe_rung = (start_rung as usize).min(route.ladder.len().saturating_sub(1));
1782 if let Some(probe_model_str) = route.ladder.get(probe_rung).cloned() {
1783 let probe_provider = ModelRef::parse(&probe_model_str)
1784 .ok()
1785 .and_then(|m| state.providers.get(&m.provider));
1786
1787 if let Some(probe_provider) = probe_provider {
1788 let mut join_set = tokio::task::JoinSet::new();
1791 for _ in 0..probe_cfg.k {
1792 let mut probe_req = base_request.clone();
1793 probe_req.model = probe_model_str.clone();
1794 let probe_auth = auth.clone();
1795 let prov = probe_provider.clone();
1796 join_set.spawn(async move { prov.complete(&probe_req, &probe_auth).await });
1797 }
1798
1799 let fail_closed_owned: std::collections::HashSet<String> = gates
1802 .iter()
1803 .filter(|g| g.abstain_fails_closed())
1804 .map(|g| g.id().to_owned())
1805 .collect();
1806
1807 let mut gate_pass_count = 0u32;
1808 let mut probe_cost_usd = 0.0f64;
1809
1810 while let Some(task_result) = join_set.join_next().await {
1811 let Ok(Ok(probe_resp)) = task_result else {
1812 continue; };
1814 probe_cost_usd += state
1817 .config
1818 .prices
1819 .cost_usd_with_cache(
1820 &probe_model_str,
1821 probe_resp.in_tokens,
1822 probe_resp.cache_write_tokens,
1823 probe_resp.cache_read_tokens,
1824 probe_resp.out_tokens,
1825 )
1826 .unwrap_or(0.0);
1827
1828 let mut probe_gate_req = base_request.clone();
1829 probe_gate_req.model = probe_model_str.clone();
1830
1831 let mut probe_gate_results = Vec::with_capacity(gates.len());
1834 for g in &gates {
1835 if !state.gate_health.enabled(&trace.tenant_id, g.id()) {
1837 continue;
1838 }
1839 let r = g.evaluate(&probe_gate_req, &probe_resp).await;
1840 probe_gate_results.push(r);
1842 }
1843
1844 let fail_closed_refs: std::collections::HashSet<&str> =
1845 fail_closed_owned.iter().map(|s| s.as_str()).collect();
1846 let verdict = aggregate_with_policy(&probe_gate_results, &fail_closed_refs);
1847 let passes = match serve_threshold {
1850 None => verdict == Verdict::Pass,
1851 Some(t) => crate::calibrate::gate_score(&probe_gate_results, verdict) >= t,
1852 };
1853 if passes {
1854 gate_pass_count += 1;
1855 }
1856 }
1857
1858 let regime = ProbeRegime::classify(gate_pass_count, probe_cfg.k);
1859 let regime_label = match regime {
1860 ProbeRegime::ConfidentPass => "confident_pass",
1861 ProbeRegime::ConfidentFail => "confident_fail",
1862 ProbeRegime::Ambiguous => "ambiguous",
1863 };
1864 metrics::counter!(
1865 "firstpass_probe_regime_total",
1866 "regime" => regime_label
1867 )
1868 .increment(1);
1869 metrics::gauge!("firstpass_probe_cost_usd_total").increment(probe_cost_usd);
1870 trace.probe = Some(ProbeSignal {
1871 k: probe_cfg.k,
1872 gate_pass_count,
1873 regime,
1874 probe_cost_usd,
1875 });
1876 }
1877 }
1878 }
1879 if let Some(cache) = state.verified_cache.as_ref()
1892 && let EngineOutcome::Served(resp) = &outcome
1893 && crate::verified_cache::is_cacheable(&trace)
1894 {
1895 let now = std::time::Instant::now();
1896 cache
1897 .put(
1898 &crate::verified_cache::VerifiedCache::compose_key(
1899 &trace.tenant_id,
1900 &trace.request.prompt_hash,
1901 &route.ladder,
1902 ),
1903 trace.trace_id,
1904 crate::verified_cache::Entry::new(resp.clone(), &trace, now),
1905 now,
1906 )
1907 .await;
1908 metrics::counter!("firstpass_cache_stores_total").increment(1);
1909 }
1910
1911 offer_trace(&state.traces, state.spill.as_ref(), trace);
1913
1914 match outcome {
1915 EngineOutcome::Served(resp) => Ok(resp),
1916 EngineOutcome::Failed(msg) => Err(ProxyError::Engine(msg)),
1917 }
1918}
1919
1920#[allow(clippy::too_many_arguments)]
1923async fn enforce_pipeline(
1924 state: &AppState,
1925 headers: &HeaderMap,
1926 body: &Bytes,
1927 features: Features,
1928 route: &Route,
1929 route_ix: usize,
1931 session_header: Option<String>,
1932 tenant: String,
1933 routing_mode: RoutingMode,
1934) -> Result<Value, ProxyError> {
1935 let Some(base_request) = parse_model_request(body) else {
1936 return Err(ProxyError::BadRequest(
1937 "request body is not a valid Anthropic Messages request".to_owned(),
1938 ));
1939 };
1940 let auth = Auth::from_headers(headers);
1941 let resp = enforce_pipeline_inner(
1942 state,
1943 body,
1944 base_request,
1945 auth,
1946 features,
1947 route,
1948 session_header,
1949 tenant,
1950 "anthropic.messages",
1951 routing_mode,
1952 route_ix,
1953 )
1954 .await?;
1955 Ok(anthropic_response_json(&resp))
1956}
1957
1958#[allow(clippy::too_many_arguments)]
1961async fn enforce_pipeline_openai(
1962 state: &AppState,
1963 headers: &HeaderMap,
1964 body: &Bytes,
1965 features: Features,
1966 route: &Route,
1967 route_ix: usize,
1969 session_header: Option<String>,
1970 tenant: String,
1971 routing_mode: RoutingMode,
1972) -> Result<Value, ProxyError> {
1973 enforce_pipeline_openai_as(
1974 state,
1975 headers,
1976 body,
1977 body,
1978 "openai.chat_completions",
1979 features,
1980 route,
1981 route_ix,
1982 session_header,
1983 tenant,
1984 routing_mode,
1985 )
1986 .await
1987}
1988
1989#[allow(clippy::too_many_arguments)]
1998async fn enforce_pipeline_openai_as(
1999 state: &AppState,
2000 headers: &HeaderMap,
2001 body: &Bytes,
2002 receipt_body: &Bytes,
2003 api: &str,
2004 features: Features,
2005 route: &Route,
2006 route_ix: usize,
2007 session_header: Option<String>,
2008 tenant: String,
2009 routing_mode: RoutingMode,
2010) -> Result<Value, ProxyError> {
2011 let providers = &state.providers;
2013 let all_openai = route.ladder.iter().all(|rung| {
2014 let pid = rung.split('/').next().unwrap_or_default();
2015 providers
2016 .get(pid)
2017 .is_some_and(|p| p.carries_structured_verbatim(Dialect::Openai))
2018 });
2019 let Some(base_request) = parse_openai_request(body, all_openai) else {
2020 return Err(ProxyError::BadRequest(
2021 "request body is not a valid OpenAI Chat Completions request".to_owned(),
2022 ));
2023 };
2024 let auth = Auth::from_headers(headers);
2025 let resp = enforce_pipeline_inner(
2026 state,
2027 receipt_body,
2028 base_request,
2029 auth,
2030 features,
2031 route,
2032 session_header,
2033 tenant,
2034 api,
2035 routing_mode,
2036 route_ix,
2037 )
2038 .await?;
2039 Ok(openai_response_json(&resp))
2040}
2041
2042const SSE_KEEPALIVE_EVERY: Duration = Duration::from_secs(5);
2044
2045fn sse_keepalive_response(
2054 rx: tokio::sync::oneshot::Receiver<Result<Value, ProxyError>>,
2055 format_message: fn(&Value) -> String,
2056) -> Response {
2057 let mut ticks = tokio::time::interval(SSE_KEEPALIVE_EVERY);
2058 ticks.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
2059 ticks.reset(); let stream = KeepaliveStream {
2061 rx: Some(rx),
2062 ticks,
2063 format_message,
2064 };
2065 (
2066 axum::http::StatusCode::OK,
2067 [(
2068 axum::http::header::CONTENT_TYPE,
2069 "text/event-stream; charset=utf-8",
2070 )],
2071 axum::body::Body::from_stream(stream),
2072 )
2073 .into_response()
2074}
2075
2076struct KeepaliveStream {
2079 rx: Option<tokio::sync::oneshot::Receiver<Result<Value, ProxyError>>>,
2081 ticks: tokio::time::Interval,
2082 format_message: fn(&Value) -> String,
2084}
2085
2086impl futures_core::Stream for KeepaliveStream {
2087 type Item = Result<Bytes, std::convert::Infallible>;
2088
2089 fn poll_next(
2090 mut self: std::pin::Pin<&mut Self>,
2091 cx: &mut std::task::Context<'_>,
2092 ) -> std::task::Poll<Option<Self::Item>> {
2093 use std::task::Poll;
2094 let Some(rx) = self.rx.as_mut() else {
2095 return Poll::Ready(None); };
2097 if let Poll::Ready(out) = std::pin::Pin::new(rx).poll(cx) {
2098 let fmt = self.format_message;
2099 let frame = match out {
2100 Ok(Ok(message)) => fmt(&message),
2101 Ok(Err(e)) => sse_error_event(&e),
2102 Err(_) => sse_error_event(&ProxyError::Internal(
2103 "enforce pipeline task dropped".to_owned(),
2104 )),
2105 };
2106 self.rx = None;
2107 return Poll::Ready(Some(Ok(Bytes::from(frame))));
2108 }
2109 if self.ticks.poll_tick(cx).is_ready() {
2110 return Poll::Ready(Some(Ok(Bytes::from_static(b": firstpass routing\n\n"))));
2111 }
2112 Poll::Pending
2113 }
2114}
2115
2116fn sse_error_event(e: &ProxyError) -> String {
2119 let mut out = String::new();
2120 sse_event(
2121 &mut out,
2122 "error",
2123 &serde_json::json!({
2124 "type": "error",
2125 "error": { "type": "api_error", "message": e.client_message() }
2126 }),
2127 );
2128 out
2129}
2130
2131fn parse_model_request(body: &[u8]) -> Option<ModelRequest> {
2140 let json: Value = serde_json::from_slice(body).ok()?;
2141 let raw = json.clone();
2142 let messages_json = json.get("messages")?.as_array()?;
2143 let messages = messages_json
2144 .iter()
2145 .map(|m| ChatMessage {
2146 role: m
2147 .get("role")
2148 .and_then(Value::as_str)
2149 .unwrap_or("user")
2150 .to_owned(),
2151 content: m
2152 .get("content")
2153 .cloned()
2154 .unwrap_or_else(|| Value::String(String::new())),
2155 })
2156 .collect();
2157 let system = json
2158 .get("system")
2159 .and_then(Value::as_str)
2160 .map(str::to_owned);
2161 let max_tokens = json
2162 .get("max_tokens")
2163 .and_then(Value::as_u64)
2164 .and_then(|n| u32::try_from(n).ok())
2165 .unwrap_or(1024);
2166 let tools = json.get("tools").cloned().unwrap_or(Value::Null);
2167 Some(ModelRequest {
2168 model: json
2169 .get("model")
2170 .and_then(Value::as_str)
2171 .unwrap_or_default()
2172 .to_owned(),
2173 system,
2174 messages,
2175 max_tokens,
2176 tools,
2177 raw,
2178 cache_prefix: false,
2179 })
2180}
2181
2182fn anthropic_response_json(resp: &ModelResponse) -> Value {
2192 let content = resp
2193 .raw
2194 .get("content")
2195 .filter(|c| c.is_array())
2196 .cloned()
2197 .unwrap_or_else(|| serde_json::json!([{ "type": "text", "text": resp.text }]));
2198 serde_json::json!({
2199 "id": format!("msg_{}", Uuid::now_v7()),
2200 "type": "message",
2201 "role": "assistant",
2202 "model": resp.model,
2203 "content": content,
2204 "usage": { "input_tokens": resp.in_tokens, "output_tokens": resp.out_tokens },
2205 })
2206}
2207
2208fn sse_event(out: &mut String, event: &str, data: &Value) {
2210 out.push_str("event: ");
2211 out.push_str(event);
2212 out.push_str("\ndata: ");
2213 out.push_str(&data.to_string());
2214 out.push_str("\n\n");
2215}
2216
2217fn anthropic_sse_from_message(message: &Value) -> String {
2224 let mut out = String::new();
2225
2226 let mut start_msg = message.clone();
2228 start_msg["content"] = Value::Array(Vec::new());
2229 sse_event(
2230 &mut out,
2231 "message_start",
2232 &serde_json::json!({ "type": "message_start", "message": start_msg }),
2233 );
2234
2235 let empty = Vec::new();
2236 let blocks = message
2237 .get("content")
2238 .and_then(Value::as_array)
2239 .unwrap_or(&empty);
2240 for (i, block) in blocks.iter().enumerate() {
2241 match block.get("type").and_then(Value::as_str) {
2242 Some("tool_use") => {
2243 let mut shell = block.clone();
2245 shell["input"] = serde_json::json!({});
2246 sse_event(
2247 &mut out,
2248 "content_block_start",
2249 &serde_json::json!({ "type": "content_block_start", "index": i, "content_block": shell }),
2250 );
2251 let input_json = block
2252 .get("input")
2253 .map_or_else(|| "{}".to_owned(), std::string::ToString::to_string);
2254 sse_event(
2255 &mut out,
2256 "content_block_delta",
2257 &serde_json::json!({ "type": "content_block_delta", "index": i,
2258 "delta": { "type": "input_json_delta", "partial_json": input_json } }),
2259 );
2260 }
2261 _ => {
2262 let text = block.get("text").and_then(Value::as_str).unwrap_or("");
2264 sse_event(
2265 &mut out,
2266 "content_block_start",
2267 &serde_json::json!({ "type": "content_block_start", "index": i,
2268 "content_block": { "type": "text", "text": "" } }),
2269 );
2270 sse_event(
2271 &mut out,
2272 "content_block_delta",
2273 &serde_json::json!({ "type": "content_block_delta", "index": i,
2274 "delta": { "type": "text_delta", "text": text } }),
2275 );
2276 }
2277 }
2278 sse_event(
2279 &mut out,
2280 "content_block_stop",
2281 &serde_json::json!({ "type": "content_block_stop", "index": i }),
2282 );
2283 }
2284
2285 let out_tokens = message
2286 .pointer("/usage/output_tokens")
2287 .cloned()
2288 .unwrap_or_else(|| Value::from(0));
2289 sse_event(
2290 &mut out,
2291 "message_delta",
2292 &serde_json::json!({ "type": "message_delta", "delta": { "stop_reason": "end_turn" },
2293 "usage": { "output_tokens": out_tokens } }),
2294 );
2295 sse_event(
2296 &mut out,
2297 "message_stop",
2298 &serde_json::json!({ "type": "message_stop" }),
2299 );
2300 out
2301}
2302
2303async fn observe_passthrough(
2305 state: AppState,
2306 headers: HeaderMap,
2307 body: Bytes,
2308 session_header: Option<String>,
2309 tenant: String,
2310) -> Response {
2311 if is_stream_request(&body) {
2313 return observe_stream(state, headers, body, session_header, tenant).await;
2314 }
2315 let start = Instant::now();
2316 let result = forward_anthropic(
2317 &state.http,
2318 &state.config.upstream_anthropic,
2319 &headers,
2320 body.clone(),
2321 )
2322 .await;
2323 let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
2324
2325 match result {
2326 Ok((status, resp_headers, resp_body)) => {
2327 spawn_trace(
2331 &state,
2332 body,
2333 Some(resp_body.clone()),
2334 latency_ms,
2335 session_header,
2336 tenant,
2337 );
2338 (status, resp_headers, resp_body).into_response()
2339 }
2340 Err(err) => {
2341 spawn_trace(&state, body, None, latency_ms, session_header, tenant);
2342 err.into_response()
2343 }
2344 }
2345}
2346
2347async fn observe_stream(
2356 state: AppState,
2357 headers: HeaderMap,
2358 body: Bytes,
2359 session_header: Option<String>,
2360 tenant: String,
2361) -> Response {
2362 let start = Instant::now();
2363 let result = forward_anthropic_streaming(
2364 &state.http,
2365 &state.config.upstream_anthropic,
2366 &headers,
2367 body.clone(),
2368 )
2369 .await;
2370 let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
2371
2372 match result {
2373 Ok((status, resp_headers, response)) => {
2374 spawn_stream_trace(&state, body, latency_ms, session_header, tenant);
2375 let stream_body = Body::from_stream(response.bytes_stream());
2376 (status, resp_headers, stream_body).into_response()
2377 }
2378 Err(err) => {
2379 spawn_trace(&state, body, None, latency_ms, session_header, tenant);
2380 err.into_response()
2381 }
2382 }
2383}
2384
2385fn spawn_stream_trace(
2387 state: &AppState,
2388 req_body: Bytes,
2389 latency_ms: u64,
2390 session_header: Option<String>,
2391 tenant: String,
2392) {
2393 let config = state.config.clone();
2394 let traces = state.traces.clone();
2395 let spill = state.spill.clone();
2396 tokio::spawn(async move {
2397 let mut trace =
2398 build_stream_trace(&config, &req_body, latency_ms, session_header.as_deref());
2399 trace.tenant_id = tenant;
2401 offer_trace(&traces, spill.as_ref(), trace);
2402 });
2403}
2404
2405fn spawn_trace(
2410 state: &AppState,
2411 req_body: Bytes,
2412 resp_body: Option<Bytes>,
2413 latency_ms: u64,
2414 session_header: Option<String>,
2415 tenant: String,
2416) {
2417 let config = state.config.clone();
2418 let traces = state.traces.clone();
2419 let spill = state.spill.clone();
2420 tokio::spawn(async move {
2421 let mut trace = match resp_body {
2422 Some(resp) => build_trace(
2423 &config,
2424 &req_body,
2425 &resp,
2426 latency_ms,
2427 session_header.as_deref(),
2428 ),
2429 None => build_error_trace(&config, &req_body, latency_ms, session_header.as_deref()),
2430 };
2431 trace.tenant_id = tenant;
2433 offer_trace(&traces, spill.as_ref(), trace);
2434 });
2435}
2436
2437fn session_id(session_header: Option<&str>, trace_id: Uuid) -> String {
2439 session_header
2440 .map(str::to_owned)
2441 .unwrap_or_else(|| trace_id.to_string())
2442}
2443
2444fn prompt_hash(salt: &str, body: &[u8]) -> String {
2447 let mut salted = Vec::with_capacity(salt.len() + body.len());
2448 salted.extend_from_slice(salt.as_bytes());
2449 salted.extend_from_slice(body);
2450 sha256_hex(&salted)
2451}
2452
2453fn request_features(body: &[u8]) -> (Option<String>, u32, bool) {
2457 let Ok(json) = serde_json::from_slice::<Value>(body) else {
2458 return (None, 0, false);
2459 };
2460 let model = json.get("model").and_then(Value::as_str).map(str::to_owned);
2461 let tool_count = json
2462 .get("tools")
2463 .and_then(Value::as_array)
2464 .map_or(0, |tools| u32::try_from(tools.len()).unwrap_or(u32::MAX));
2465 let has_images = json
2466 .get("messages")
2467 .and_then(Value::as_array)
2468 .is_some_and(|messages| messages.iter().any(message_has_image));
2469 (model, tool_count, has_images)
2470}
2471
2472fn message_has_image(message: &Value) -> bool {
2474 message
2475 .get("content")
2476 .and_then(Value::as_array)
2477 .is_some_and(|blocks| {
2478 blocks
2479 .iter()
2480 .any(|block| block.get("type").and_then(Value::as_str) == Some("image"))
2481 })
2482}
2483
2484#[derive(Debug, Default, Clone, PartialEq, Eq)]
2486struct ResponseUsage {
2487 model: Option<String>,
2488 in_tokens: u64,
2489 cache_write_tokens: u64,
2490 cache_read_tokens: u64,
2491 out_tokens: u64,
2492}
2493
2494fn response_usage(body: &[u8]) -> ResponseUsage {
2502 let Ok(json) = serde_json::from_slice::<Value>(body) else {
2503 return ResponseUsage::default();
2504 };
2505 let u = |ptr: &str| json.pointer(ptr).and_then(Value::as_u64).unwrap_or(0);
2506 ResponseUsage {
2507 model: json.get("model").and_then(Value::as_str).map(str::to_owned),
2508 in_tokens: u("/usage/input_tokens"),
2509 cache_write_tokens: u("/usage/cache_creation_input_tokens"),
2510 cache_read_tokens: u("/usage/cache_read_input_tokens"),
2511 out_tokens: u("/usage/output_tokens"),
2512 }
2513}
2514
2515fn build_trace(
2517 config: &ProxyConfig,
2518 req_body: &Bytes,
2519 resp_body: &Bytes,
2520 latency_ms: u64,
2521 session_header: Option<&str>,
2522) -> Trace {
2523 let (req_model, tool_count, has_images) = request_features(req_body);
2524 let usage = response_usage(resp_body);
2525 let (resp_model, in_tokens, out_tokens) = (usage.model, usage.in_tokens, usage.out_tokens);
2526 let (cache_write_tokens, cache_read_tokens) =
2527 (usage.cache_write_tokens, usage.cache_read_tokens);
2528 let model = resp_model
2529 .or(req_model)
2530 .unwrap_or_else(|| "unknown".to_owned());
2531
2532 let cost_usd = config
2533 .prices
2534 .cost_usd_with_cache(
2535 &format!("anthropic/{model}"),
2536 in_tokens,
2537 cache_write_tokens,
2538 cache_read_tokens,
2539 out_tokens,
2540 )
2541 .unwrap_or(0.0);
2542
2543 let attempt = Attempt {
2544 rung: 0,
2545 model,
2546 provider: "anthropic".to_owned(),
2547 in_tokens,
2548 cache_write_tokens,
2549 cache_read_tokens,
2550 out_tokens,
2551 cost_usd,
2552 latency_ms,
2553 gates: Vec::new(),
2554 verdict: Verdict::Pass,
2555 };
2556
2557 let mut trace = base_trace(config, req_body, latency_ms, session_header);
2558 trace.request.features.prompt_token_bucket = token_bucket(in_tokens);
2559 trace.request.features.tool_count = tool_count;
2560 trace.request.features.has_images = has_images;
2561 trace.attempts.push(attempt);
2562 trace.final_ = FinalOutcome {
2563 served_rung: Some(0),
2564 served_from: ServedFrom::Attempt,
2565 total_cost_usd: cost_usd,
2566 gate_cost_usd: 0.0,
2567 total_latency_ms: latency_ms,
2568 escalations: 0,
2569 counterfactual_baseline_usd: cost_usd,
2570 savings_usd: 0.0,
2571 cache_source: None,
2572 };
2573 trace.recompute_savings();
2574 trace
2575}
2576
2577fn build_stream_trace(
2581 config: &ProxyConfig,
2582 req_body: &Bytes,
2583 latency_ms: u64,
2584 session_header: Option<&str>,
2585) -> Trace {
2586 let (req_model, tool_count, has_images) = request_features(req_body);
2587 let model = req_model.unwrap_or_else(|| "unknown".to_owned());
2588
2589 let attempt = Attempt {
2590 rung: 0,
2591 model,
2592 provider: "anthropic".to_owned(),
2593 in_tokens: 0,
2594 cache_write_tokens: 0,
2596 cache_read_tokens: 0,
2597 out_tokens: 0,
2598 cost_usd: 0.0,
2599 latency_ms,
2600 gates: Vec::new(),
2601 verdict: Verdict::Pass,
2602 };
2603
2604 let mut trace = base_trace(config, req_body, latency_ms, session_header);
2605 trace.request.features.tool_count = tool_count;
2606 trace.request.features.has_images = has_images;
2607 trace.attempts.push(attempt);
2608 trace.final_ = FinalOutcome {
2609 served_rung: Some(0),
2610 served_from: ServedFrom::Attempt,
2611 total_cost_usd: 0.0,
2612 gate_cost_usd: 0.0,
2613 total_latency_ms: latency_ms,
2614 escalations: 0,
2615 counterfactual_baseline_usd: 0.0,
2616 savings_usd: 0.0,
2617 cache_source: None,
2618 };
2619 trace.recompute_savings();
2620 trace
2621}
2622
2623fn build_error_trace(
2627 config: &ProxyConfig,
2628 req_body: &Bytes,
2629 latency_ms: u64,
2630 session_header: Option<&str>,
2631) -> Trace {
2632 let (_, tool_count, has_images) = request_features(req_body);
2633 let mut trace = base_trace(config, req_body, latency_ms, session_header);
2634 trace.request.features.tool_count = tool_count;
2635 trace.request.features.has_images = has_images;
2636 trace.final_ = FinalOutcome {
2637 served_rung: None,
2638 served_from: ServedFrom::Error,
2639 total_cost_usd: 0.0,
2640 gate_cost_usd: 0.0,
2641 total_latency_ms: latency_ms,
2642 escalations: 0,
2643 counterfactual_baseline_usd: 0.0,
2644 savings_usd: 0.0,
2645 cache_source: None,
2646 };
2647 trace.recompute_savings();
2648 trace
2649}
2650
2651fn base_trace(
2654 config: &ProxyConfig,
2655 req_body: &Bytes,
2656 latency_ms: u64,
2657 session_header: Option<&str>,
2658) -> Trace {
2659 let trace_id = Uuid::now_v7();
2660 let mut features = Features::new(TaskKind::Other);
2661 features.hour_bucket = hour_bucket(jiff::Timestamp::now());
2662
2663 Trace {
2664 trace_id,
2665 prev_hash: GENESIS_HASH.to_owned(),
2666 tenant_id: config.tenant_id.clone(),
2667 session_id: session_id(session_header, trace_id),
2668 ts: jiff::Timestamp::now(),
2669 mode: Mode::Observe,
2670 policy: PolicyRef {
2671 id: "observe-passthrough@v0".to_owned(),
2672 explore: false,
2673 propensity: None,
2674 mode_profile: None,
2675 },
2676 request: RequestInfo {
2677 api: "anthropic.messages".to_owned(),
2678 prompt_hash: prompt_hash(&config.prompt_salt, req_body),
2679 features,
2680 },
2681 attempts: Vec::new(),
2682 deferred: Vec::new(),
2683 final_: FinalOutcome {
2684 served_rung: None,
2685 served_from: ServedFrom::Error,
2686 total_cost_usd: 0.0,
2687 gate_cost_usd: 0.0,
2688 total_latency_ms: latency_ms,
2689 escalations: 0,
2690 counterfactual_baseline_usd: 0.0,
2691 savings_usd: 0.0,
2692 cache_source: None,
2693 },
2694 probe: None,
2695 rollout: None,
2696 shadow: None,
2697 route_ix: None,
2698 predicted_pass: None,
2699 elastic: None,
2700 }
2701}
2702
2703fn openai_messages_have_tool_calls(body: &[u8]) -> bool {
2708 serde_json::from_slice::<Value>(body)
2709 .ok()
2710 .and_then(|json| {
2711 json.get("messages").and_then(Value::as_array).map(|msgs| {
2712 msgs.iter().any(|m| {
2713 m.get("tool_calls").is_some()
2714 || m.get("role").and_then(Value::as_str) == Some("tool")
2715 })
2716 })
2717 })
2718 .unwrap_or(false)
2719}
2720
2721fn openai_has_http_images(body: &[u8]) -> bool {
2725 serde_json::from_slice::<Value>(body)
2726 .ok()
2727 .and_then(|json| {
2728 json.get("messages").and_then(Value::as_array).map(|msgs| {
2729 msgs.iter().any(|m| {
2730 m.get("content")
2731 .and_then(Value::as_array)
2732 .is_some_and(|parts| {
2733 parts.iter().any(|p| {
2734 p.get("type").and_then(Value::as_str) == Some("image_url")
2735 && p.pointer("/image_url/url")
2736 .and_then(Value::as_str)
2737 .is_some_and(|u| {
2738 u.starts_with("http://") || u.starts_with("https://")
2739 })
2740 })
2741 })
2742 })
2743 })
2744 })
2745 .unwrap_or(false)
2746}
2747
2748fn openai_messages_have_images(body: &[u8]) -> bool {
2751 serde_json::from_slice::<Value>(body)
2752 .ok()
2753 .and_then(|json| {
2754 json.get("messages").and_then(Value::as_array).map(|msgs| {
2755 msgs.iter().any(|m| {
2756 m.get("content")
2757 .and_then(Value::as_array)
2758 .is_some_and(|parts| {
2759 parts
2760 .iter()
2761 .any(|p| p.get("type").and_then(Value::as_str) == Some("image_url"))
2762 })
2763 })
2764 })
2765 })
2766 .unwrap_or(false)
2767}
2768
2769fn extract_openai_features(headers: &HeaderMap, body: &[u8]) -> Features {
2772 let Ok(json) = serde_json::from_slice::<Value>(body) else {
2773 let mut f = Features::new(TaskKind::Other);
2774 f.hour_bucket = hour_bucket(jiff::Timestamp::now());
2775 return f;
2776 };
2777 let tool_count = json
2778 .get("tools")
2779 .and_then(Value::as_array)
2780 .map_or(0, |tools| u32::try_from(tools.len()).unwrap_or(u32::MAX));
2781 let has_images = openai_messages_have_images(body);
2782 let mut f = Features::new(TaskKind::Other);
2783 f.agent = header_str(headers, AGENT_HEADER);
2784 f.subagent = header_str(headers, SUBAGENT_HEADER);
2785 f.tool_count = tool_count;
2786 f.has_images = has_images;
2787 f.prompt_token_bucket = token_bucket(body.len() as u64);
2788 f.hour_bucket = hour_bucket(jiff::Timestamp::now());
2789 f
2790}
2791
2792fn parse_data_url(url: &str) -> Option<(&str, &str)> {
2796 let rest = url.strip_prefix("data:")?;
2797 let (meta, data) = rest.split_once(',')?;
2798 let media_type = meta.strip_suffix(";base64")?;
2799 Some((media_type, data))
2800}
2801
2802fn translate_openai_user_content(content: &Value) -> Option<Value> {
2805 match content {
2806 Value::String(_) => Some(content.clone()),
2808 Value::Array(parts) => {
2809 let mut blocks: Vec<Value> = Vec::with_capacity(parts.len());
2810 for part in parts {
2811 match part.get("type").and_then(Value::as_str) {
2812 Some("text") => {
2813 let text = part.get("text").and_then(Value::as_str).unwrap_or("");
2814 blocks.push(serde_json::json!({ "type": "text", "text": text }));
2815 }
2816 Some("image_url") => {
2817 let url = part.pointer("/image_url/url").and_then(Value::as_str)?;
2818 if url.starts_with("http://") || url.starts_with("https://") {
2819 return None; }
2821 let (media_type, data) = parse_data_url(url)?;
2823 blocks.push(serde_json::json!({
2824 "type": "image",
2825 "source": { "type": "base64", "media_type": media_type, "data": data }
2826 }));
2827 }
2828 _ => {} }
2830 }
2831 Some(Value::Array(blocks))
2832 }
2833 _ => Some(Value::String(String::new())),
2834 }
2835}
2836
2837fn translate_openai_tools(tools: &Value) -> Value {
2841 let Some(arr) = tools.as_array() else {
2842 return Value::Null;
2843 };
2844 let converted: Vec<Value> = arr
2845 .iter()
2846 .map(|tool| {
2847 let func = tool.get("function").unwrap_or(&Value::Null);
2848 let mut out = serde_json::json!({
2849 "name": func.get("name").cloned().unwrap_or(Value::String(String::new())),
2850 "input_schema": func.get("parameters").cloned()
2851 .unwrap_or_else(|| serde_json::json!({ "type": "object" })),
2852 });
2853 if let Some(desc) = func.get("description") {
2854 out["description"] = desc.clone();
2855 }
2856 out
2857 })
2858 .collect();
2859 Value::Array(converted)
2860}
2861
2862fn translate_openai_tool_choice(tc: &Value) -> Value {
2864 match tc {
2865 Value::String(s) => match s.as_str() {
2866 "auto" => serde_json::json!({ "type": "auto" }),
2867 "required" => serde_json::json!({ "type": "any" }),
2868 _ => serde_json::json!({ "type": "auto" }),
2870 },
2871 Value::Object(_) => {
2872 if tc.get("type").and_then(Value::as_str) == Some("function") {
2874 let name = tc.pointer("/function/name").cloned().unwrap_or(Value::Null);
2875 serde_json::json!({ "type": "tool", "name": name })
2876 } else {
2877 serde_json::json!({ "type": "auto" })
2878 }
2879 }
2880 _ => serde_json::json!({ "type": "auto" }),
2881 }
2882}
2883
2884pub fn parse_openai_request(body: &[u8], carry_raw: bool) -> Option<ModelRequest> {
2897 let json: Value = serde_json::from_slice(body).ok()?;
2898 let raw = if carry_raw { json.clone() } else { Value::Null };
2899
2900 let messages_json = json.get("messages")?.as_array()?;
2901
2902 let mut system: Option<String> = None;
2903 let mut messages: Vec<ChatMessage> = Vec::with_capacity(messages_json.len());
2904 let mut tools = Value::Null;
2905 let mut tool_choice_override: Option<Value> = None;
2906
2907 for msg in messages_json {
2908 let role = msg.get("role").and_then(Value::as_str).unwrap_or("user");
2909 match role {
2910 "system" => {
2911 if let Some(s) = msg.get("content").and_then(Value::as_str) {
2916 system = Some(s.to_owned());
2917 }
2918 }
2919 "user" => {
2920 let content_val = msg.get("content").unwrap_or(&Value::Null);
2921 let translated = translate_openai_user_content(content_val)?;
2922 messages.push(ChatMessage {
2923 role: "user".to_owned(),
2924 content: translated,
2925 });
2926 }
2927 "assistant" => {
2928 if let Some(tc_arr) = msg.get("tool_calls").and_then(Value::as_array) {
2929 let mut blocks: Vec<Value> = Vec::new();
2931 if let Some(text) = msg.get("content").and_then(Value::as_str)
2933 && !text.is_empty()
2934 {
2935 blocks.push(serde_json::json!({ "type": "text", "text": text }));
2936 }
2937 for tc in tc_arr {
2938 let id = tc.get("id").and_then(Value::as_str).unwrap_or("");
2939 let func = tc.get("function").unwrap_or(&Value::Null);
2940 let name = func.get("name").and_then(Value::as_str).unwrap_or("");
2941 let args_str = func
2942 .get("arguments")
2943 .and_then(Value::as_str)
2944 .unwrap_or("{}");
2945 let input: Value = serde_json::from_str(args_str)
2946 .unwrap_or_else(|_| serde_json::json!({}));
2947 blocks.push(serde_json::json!({
2948 "type": "tool_use",
2949 "id": id,
2950 "name": name,
2951 "input": input,
2952 }));
2953 }
2954 messages.push(ChatMessage {
2955 role: "assistant".to_owned(),
2956 content: Value::Array(blocks),
2957 });
2958 } else {
2959 let content = msg
2961 .get("content")
2962 .cloned()
2963 .unwrap_or_else(|| Value::String(String::new()));
2964 messages.push(ChatMessage {
2965 role: "assistant".to_owned(),
2966 content,
2967 });
2968 }
2969 }
2970 "tool" => {
2971 let tool_call_id = msg
2973 .get("tool_call_id")
2974 .and_then(Value::as_str)
2975 .unwrap_or("");
2976 let content = msg
2977 .get("content")
2978 .cloned()
2979 .unwrap_or_else(|| Value::String(String::new()));
2980 let result_block = serde_json::json!({
2981 "type": "tool_result",
2982 "tool_use_id": tool_call_id,
2983 "content": content,
2984 });
2985 messages.push(ChatMessage {
2986 role: "user".to_owned(),
2987 content: Value::Array(vec![result_block]),
2988 });
2989 }
2990 _ => {} }
2992 }
2993
2994 if !carry_raw {
2996 if let Some(t) = json.get("tools") {
2997 tools = translate_openai_tools(t);
2998 }
2999 if let Some(tc) = json.get("tool_choice") {
3000 tool_choice_override = Some(translate_openai_tool_choice(tc));
3001 }
3002 } else {
3003 tools = json.get("tools").cloned().unwrap_or(Value::Null);
3004 }
3005
3006 let max_tokens = json
3007 .get("max_tokens")
3008 .and_then(Value::as_u64)
3009 .and_then(|n| u32::try_from(n).ok())
3010 .unwrap_or(1024);
3011
3012 let model = json
3013 .get("model")
3014 .and_then(Value::as_str)
3015 .unwrap_or_default()
3016 .to_owned();
3017
3018 let _ = tool_choice_override; Some(ModelRequest {
3029 model,
3030 system,
3031 messages,
3032 max_tokens,
3033 tools,
3034 raw,
3035 cache_prefix: false,
3036 })
3037}
3038
3039fn extract_openai_content_and_tools(raw: &Value, text: &str) -> (Value, Option<Value>) {
3046 if let Some(blocks) = raw.get("content").and_then(Value::as_array) {
3048 let mut text_parts: Vec<&str> = Vec::new();
3049 let mut tool_calls: Vec<Value> = Vec::new();
3050 for block in blocks {
3051 match block.get("type").and_then(Value::as_str) {
3052 Some("text") => {
3053 if let Some(t) = block.get("text").and_then(Value::as_str) {
3054 text_parts.push(t);
3055 }
3056 }
3057 Some("tool_use") => {
3058 let id = block.get("id").and_then(Value::as_str).unwrap_or("");
3059 let name = block.get("name").and_then(Value::as_str).unwrap_or("");
3060 let input_str = block
3061 .get("input")
3062 .map_or_else(|| "{}".to_owned(), std::string::ToString::to_string);
3063 tool_calls.push(serde_json::json!({
3064 "id": id,
3065 "type": "function",
3066 "function": { "name": name, "arguments": input_str },
3067 }));
3068 }
3069 _ => {}
3070 }
3071 }
3072 let content_text = if tool_calls.is_empty() || !text_parts.is_empty() {
3073 Value::String(text_parts.join(""))
3074 } else {
3075 Value::Null };
3077 let tc = if tool_calls.is_empty() {
3078 None
3079 } else {
3080 Some(Value::Array(tool_calls))
3081 };
3082 return (content_text, tc);
3083 }
3084
3085 if let Some(msg) = raw.pointer("/choices/0/message") {
3087 let content = msg
3088 .get("content")
3089 .cloned()
3090 .unwrap_or(Value::String(text.to_owned()));
3091 let tc = msg.get("tool_calls").cloned();
3092 return (content, tc);
3093 }
3094
3095 (Value::String(text.to_owned()), None)
3097}
3098
3099fn openai_response_json(resp: &ModelResponse) -> Value {
3102 let (content_text, tool_calls) = extract_openai_content_and_tools(&resp.raw, &resp.text);
3103 let finish_reason = if tool_calls.is_some() {
3104 "tool_calls"
3105 } else {
3106 "stop"
3107 };
3108 let mut message = serde_json::json!({
3109 "role": "assistant",
3110 "content": content_text,
3111 });
3112 if let Some(tc) = tool_calls {
3113 message["tool_calls"] = tc;
3114 }
3115 serde_json::json!({
3116 "id": format!("chatcmpl-{}", Uuid::now_v7()),
3117 "object": "chat.completion",
3118 "created": jiff::Timestamp::now().as_second(),
3119 "model": resp.model,
3120 "choices": [{
3121 "index": 0,
3122 "message": message,
3123 "finish_reason": finish_reason,
3124 }],
3125 "usage": {
3126 "prompt_tokens": resp.in_tokens,
3127 "completion_tokens": resp.out_tokens,
3128 "total_tokens": resp.in_tokens + resp.out_tokens,
3129 }
3130 })
3131}
3132
3133fn openai_sse_from_message(message: &Value) -> String {
3137 let id = message
3138 .get("id")
3139 .and_then(Value::as_str)
3140 .unwrap_or("chatcmpl-unknown")
3141 .to_owned();
3142 let created = message.get("created").cloned().unwrap_or(Value::from(0));
3143 let model = message
3144 .get("model")
3145 .and_then(Value::as_str)
3146 .unwrap_or("unknown");
3147
3148 let choices = message.get("choices").and_then(Value::as_array);
3149 let msg = choices
3150 .and_then(|c| c.first())
3151 .and_then(|c| c.get("message"));
3152 let content = msg
3153 .and_then(|m| m.get("content"))
3154 .cloned()
3155 .unwrap_or(Value::Null);
3156 let tool_calls = msg.and_then(|m| m.get("tool_calls")).cloned();
3157 let finish_reason = choices
3158 .and_then(|c| c.first())
3159 .and_then(|c| c.get("finish_reason"))
3160 .cloned()
3161 .unwrap_or_else(|| Value::String("stop".to_owned()));
3162
3163 let mut out = String::new();
3164 let chunk = |delta: Value| {
3165 serde_json::json!({
3166 "id": id,
3167 "object": "chat.completion.chunk",
3168 "created": created,
3169 "model": model,
3170 "choices": [{ "index": 0, "delta": delta, "finish_reason": Value::Null }]
3171 })
3172 };
3173
3174 let role_chunk = chunk(serde_json::json!({ "role": "assistant", "content": "" }));
3176 out.push_str("data: ");
3177 out.push_str(&role_chunk.to_string());
3178 out.push_str("\n\n");
3179
3180 if let Value::String(text) = &content
3182 && !text.is_empty()
3183 {
3184 let content_chunk = chunk(serde_json::json!({ "content": text }));
3185 out.push_str("data: ");
3186 out.push_str(&content_chunk.to_string());
3187 out.push_str("\n\n");
3188 }
3189
3190 if let Some(tc) = tool_calls {
3192 let tc_chunk = chunk(serde_json::json!({ "tool_calls": tc }));
3193 out.push_str("data: ");
3194 out.push_str(&tc_chunk.to_string());
3195 out.push_str("\n\n");
3196 }
3197
3198 let finish_chunk = serde_json::json!({
3200 "id": id,
3201 "object": "chat.completion.chunk",
3202 "created": created,
3203 "model": model,
3204 "choices": [{ "index": 0, "delta": {}, "finish_reason": finish_reason }]
3205 });
3206 out.push_str("data: ");
3207 out.push_str(&finish_chunk.to_string());
3208 out.push_str("\n\n");
3209
3210 out.push_str("data: [DONE]\n\n");
3211 out
3212}
3213
3214#[allow(clippy::too_many_arguments)]
3219async fn handle_enforce_openai(
3220 state: &AppState,
3221 headers: &HeaderMap,
3222 body: &Bytes,
3223 features: Features,
3224 route: &Route,
3225 route_ix: usize,
3228 session_header: Option<String>,
3229 tenant: String,
3230 routing_mode: RoutingMode,
3231) -> Response {
3232 if is_stream_request(body) {
3233 let (tx, rx) = tokio::sync::oneshot::channel::<Result<Value, ProxyError>>();
3234 let (state_c, headers_c, body_c, route_c) =
3235 (state.clone(), headers.clone(), body.clone(), route.clone());
3236 tokio::spawn(async move {
3237 let out = enforce_pipeline_openai(
3238 &state_c,
3239 &headers_c,
3240 &body_c,
3241 features,
3242 &route_c,
3243 route_ix,
3244 session_header,
3245 tenant,
3246 routing_mode,
3247 )
3248 .await;
3249 let _ = tx.send(out);
3250 });
3251 return sse_keepalive_response(rx, openai_sse_from_message);
3252 }
3253 match enforce_pipeline_openai(
3254 state,
3255 headers,
3256 body,
3257 features,
3258 route,
3259 route_ix,
3260 session_header,
3261 tenant,
3262 routing_mode,
3263 )
3264 .await
3265 {
3266 Ok(message) => (axum::http::StatusCode::OK, Json(message)).into_response(),
3267 Err(e) => e.into_response(),
3268 }
3269}
3270
3271async fn observe_passthrough_openai(
3277 state: AppState,
3278 headers: HeaderMap,
3279 body: Bytes,
3280 session_header: Option<String>,
3281 tenant: String,
3282) -> Response {
3283 observe_passthrough_openai_path(
3284 state,
3285 headers,
3286 body,
3287 session_header,
3288 tenant,
3289 crate::upstream::OPENAI_CHAT_PATH,
3290 )
3291 .await
3292}
3293
3294async fn observe_passthrough_openai_path(
3300 state: AppState,
3301 headers: HeaderMap,
3302 body: Bytes,
3303 session_header: Option<String>,
3304 tenant: String,
3305 path: &'static str,
3306) -> Response {
3307 if is_stream_request(&body) {
3308 return observe_stream_openai_path(state, headers, body, session_header, tenant, path)
3309 .await;
3310 }
3311 let start = Instant::now();
3312 let result = crate::upstream::forward_openai_path(
3313 &state.http,
3314 &state.config.upstream_openai,
3315 path,
3316 &headers,
3317 body.clone(),
3318 )
3319 .await;
3320 let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
3321 match result {
3322 Ok((status, resp_headers, resp_body)) => {
3323 spawn_trace(
3324 &state,
3325 body,
3326 Some(resp_body.clone()),
3327 latency_ms,
3328 session_header,
3329 tenant,
3330 );
3331 (status, resp_headers, resp_body).into_response()
3332 }
3333 Err(err) => {
3334 spawn_trace(&state, body, None, latency_ms, session_header, tenant);
3335 err.into_response()
3336 }
3337 }
3338}
3339
3340async fn observe_stream_openai_path(
3344 state: AppState,
3345 headers: HeaderMap,
3346 body: Bytes,
3347 session_header: Option<String>,
3348 tenant: String,
3349 path: &'static str,
3350) -> Response {
3351 let start = Instant::now();
3352 let result = crate::upstream::forward_openai_streaming_path(
3353 &state.http,
3354 &state.config.upstream_openai,
3355 path,
3356 &headers,
3357 body.clone(),
3358 )
3359 .await;
3360 let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
3361 match result {
3362 Ok((status, resp_headers, response)) => {
3363 spawn_stream_trace(&state, body, latency_ms, session_header, tenant);
3364 let stream_body = Body::from_stream(response.bytes_stream());
3365 (status, resp_headers, stream_body).into_response()
3366 }
3367 Err(err) => {
3368 spawn_trace(&state, body, None, latency_ms, session_header, tenant);
3369 err.into_response()
3370 }
3371 }
3372}
3373
3374async fn chat_completions(
3379 State(state): State<AppState>,
3380 Extension(TenantId(tenant)): Extension<TenantId>,
3381 headers: HeaderMap,
3382 body: Bytes,
3383) -> Response {
3384 let session_header = header_str(&headers, SESSION_HEADER);
3385
3386 if let Some(routing) = state.config.routing.as_ref() {
3387 let features = extract_openai_features(&headers, &body);
3388 if let Some(route) = routing
3389 .route_for(&features)
3390 .filter(|r| r.mode == Mode::Enforce && !r.ladder.is_empty())
3391 {
3392 let route_ix = routing
3394 .routes
3395 .iter()
3396 .position(|r| std::ptr::eq(r, route))
3397 .unwrap_or(0);
3398 let route = route.clone();
3399 let routing_mode = resolve_mode(&headers, &route, &state.config);
3401 if routing_mode == RoutingMode::Observe {
3403 return observe_passthrough_openai(state, headers, body, session_header, tenant)
3404 .await;
3405 }
3406 if enforce_can_handle(
3407 &features,
3408 &body,
3409 routing.escalation.enforce_structured,
3410 &route.ladder,
3411 &state.providers,
3412 Dialect::Openai,
3413 ) {
3414 return handle_enforce_openai(
3415 &state,
3416 &headers,
3417 &body,
3418 features,
3419 &route,
3420 route_ix,
3421 session_header,
3422 tenant,
3423 routing_mode,
3424 )
3425 .await;
3426 }
3427 tracing::info!(
3428 "enforce route matched but OpenAI structured request can't be routed faithfully (flag/ladder); serving via observe passthrough"
3429 );
3430 }
3431 }
3432 observe_passthrough_openai(state, headers, body, session_header, tenant).await
3433}
3434
3435async fn responses(
3444 State(state): State<AppState>,
3445 Extension(TenantId(tenant)): Extension<TenantId>,
3446 headers: HeaderMap,
3447 body: Bytes,
3448) -> Response {
3449 let session_header = header_str(&headers, SESSION_HEADER);
3450
3451 if is_stream_request(&body) {
3455 return observe_passthrough_openai_path(
3456 state,
3457 headers,
3458 body,
3459 session_header,
3460 tenant,
3461 crate::upstream::OPENAI_RESPONSES_PATH,
3462 )
3463 .await;
3464 }
3465
3466 let Ok(parsed) = serde_json::from_slice::<Value>(&body) else {
3467 return observe_passthrough_openai_path(
3468 state,
3469 headers,
3470 body,
3471 session_header,
3472 tenant,
3473 crate::upstream::OPENAI_RESPONSES_PATH,
3474 )
3475 .await;
3476 };
3477 if crate::responses::has_untranslatable_content(&parsed) {
3481 return observe_passthrough_openai_path(
3482 state,
3483 headers,
3484 body,
3485 session_header,
3486 tenant,
3487 crate::upstream::OPENAI_RESPONSES_PATH,
3488 )
3489 .await;
3490 }
3491 let chat_body = Bytes::from(crate::responses::request_to_chat(&parsed).to_string());
3492
3493 if let Some(routing) = state.config.routing.as_ref() {
3494 let features = extract_openai_features(&headers, &chat_body);
3495 if let Some(route) = routing
3496 .route_for(&features)
3497 .filter(|r| r.mode == Mode::Enforce && !r.ladder.is_empty())
3498 {
3499 let route_ix = routing
3500 .routes
3501 .iter()
3502 .position(|r| std::ptr::eq(r, route))
3503 .unwrap_or(0);
3504 let route = route.clone();
3505 let routing_mode = resolve_mode(&headers, &route, &state.config);
3506 if routing_mode != RoutingMode::Observe
3507 && enforce_can_handle(
3508 &features,
3509 &chat_body,
3510 routing.escalation.enforce_structured,
3511 &route.ladder,
3512 &state.providers,
3513 Dialect::Openai,
3514 )
3515 {
3516 return match enforce_pipeline_openai_as(
3517 &state,
3518 &headers,
3519 &chat_body,
3520 &body,
3521 "openai.responses",
3522 features,
3523 &route,
3524 route_ix,
3525 session_header,
3526 tenant,
3527 routing_mode,
3528 )
3529 .await
3530 {
3531 Ok(chat) => Json(crate::responses::response_from_chat(&chat)).into_response(),
3532 Err(e) => e.into_response(),
3533 };
3534 }
3535 }
3536 }
3537 observe_passthrough_openai_path(
3538 state,
3539 headers,
3540 body,
3541 session_header,
3542 tenant,
3543 crate::upstream::OPENAI_RESPONSES_PATH,
3544 )
3545 .await
3546}
3547
3548#[cfg(test)]
3549mod tests {
3550 use bytes::Bytes;
3551
3552 use super::*;
3553
3554 fn test_config() -> ProxyConfig {
3555 ProxyConfig::from_lookup(|_| None).unwrap()
3556 }
3557
3558 #[test]
3559 fn build_trace_maps_request_and_response_fields() {
3560 let config = test_config();
3561 let req = Bytes::from_static(
3562 br#"{"model":"claude-haiku-4-5","tools":[{"name":"a"}],"messages":[]}"#,
3563 );
3564 let resp = Bytes::from_static(
3565 br#"{"model":"claude-haiku-4-5","usage":{"input_tokens":1200,"output_tokens":300}}"#,
3566 );
3567
3568 let trace = build_trace(&config, &req, &resp, 42, Some("sess-1"));
3569
3570 assert_eq!(trace.request.api, "anthropic.messages");
3571 assert_eq!(trace.session_id, "sess-1");
3572 assert_eq!(trace.attempts.len(), 1);
3573 let attempt = &trace.attempts[0];
3574 assert_eq!(attempt.model, "claude-haiku-4-5");
3575 assert_eq!(attempt.provider, "anthropic");
3576 assert_eq!(attempt.in_tokens, 1200);
3577 assert_eq!(attempt.out_tokens, 300);
3578 assert!(attempt.cost_usd > 0.0);
3579 assert_eq!(trace.request.features.tool_count, 1);
3580 assert!(!trace.request.features.has_images);
3581 assert_eq!(trace.final_.served_rung, Some(0));
3582 }
3583
3584 #[test]
3585 fn build_trace_falls_back_to_trace_id_session_when_header_absent() {
3586 let config = test_config();
3587 let req = Bytes::from_static(b"{}");
3588 let resp = Bytes::from_static(b"{}");
3589
3590 let trace = build_trace(&config, &req, &resp, 1, None);
3591
3592 assert_eq!(trace.session_id, trace.trace_id.to_string());
3593 }
3594
3595 #[test]
3596 fn build_error_trace_has_no_attempts_and_served_from_error() {
3597 let config = test_config();
3598 let req = Bytes::from_static(br#"{"model":"claude-haiku-4-5"}"#);
3599
3600 let trace = build_error_trace(&config, &req, 7, None);
3601
3602 assert!(trace.attempts.is_empty());
3603 assert_eq!(trace.final_.served_from, ServedFrom::Error);
3604 assert_eq!(trace.final_.served_rung, None);
3605 }
3606
3607 #[test]
3608 fn message_with_image_block_sets_has_images() {
3609 let req = br#"{"messages":[{"role":"user","content":[{"type":"image"}]}]}"#;
3610 let (_, _, has_images) = request_features(req);
3611 assert!(has_images);
3612 }
3613
3614 #[test]
3615 fn prompt_hash_never_contains_raw_prompt_text() {
3616 let hash = prompt_hash("salt", b"super secret prompt");
3617 assert!(!hash.contains("secret"));
3618 assert_eq!(hash.len(), 64);
3619 }
3620
3621 #[test]
3622 fn parse_model_request_preserves_content_verbatim_and_projects_text() {
3623 let body = br#"{"model":"m","system":"sys","max_tokens":50,
3624 "messages":[{"role":"user","content":[{"type":"text","text":"a"},{"type":"text","text":"b"}]},
3625 {"role":"assistant","content":"c"}]}"#;
3626 let req = parse_model_request(body).unwrap();
3627 assert_eq!(req.system.as_deref(), Some("sys"));
3628 assert_eq!(req.max_tokens, 50);
3629 assert_eq!(req.messages.len(), 2);
3630 assert_eq!(
3632 req.messages[0].content,
3633 serde_json::json!([{"type":"text","text":"a"},{"type":"text","text":"b"}])
3634 );
3635 assert_eq!(req.messages[1].content, Value::String("c".to_owned()));
3637 assert_eq!(req.messages[0].text_view(), "a\nb");
3639 assert_eq!(req.messages[1].text_view(), "c");
3640 }
3641
3642 #[test]
3643 fn tool_and_image_blocks_survive_the_request_round_trip() {
3644 let body = br#"{"model":"m","max_tokens":50,"messages":[
3646 {"role":"assistant","content":[{"type":"tool_use","id":"t1","name":"calc","input":{"x":1}}]},
3647 {"role":"user","content":[
3648 {"type":"tool_result","tool_use_id":"t1","content":"2"},
3649 {"type":"image","source":{"type":"base64","media_type":"image/png","data":"AA=="}}
3650 ]}]}"#;
3651 let req = parse_model_request(body).unwrap();
3652 let round_tripped = serde_json::to_value(&req.messages).unwrap();
3653 assert_eq!(
3654 round_tripped,
3655 serde_json::json!([
3656 {"role":"assistant","content":[{"type":"tool_use","id":"t1","name":"calc","input":{"x":1}}]},
3657 {"role":"user","content":[
3658 {"type":"tool_result","tool_use_id":"t1","content":"2"},
3659 {"type":"image","source":{"type":"base64","media_type":"image/png","data":"AA=="}}
3660 ]}
3661 ])
3662 );
3663 }
3664
3665 #[test]
3666 fn text_message_serializes_byte_identical_to_a_plain_string() {
3667 let m = ChatMessage::text("user", "hello");
3669 assert_eq!(
3670 serde_json::to_string(&m).unwrap(),
3671 r#"{"role":"user","content":"hello"}"#
3672 );
3673 }
3674
3675 #[test]
3676 fn parse_model_request_rejects_non_message_bodies() {
3677 assert!(parse_model_request(b"not json").is_none());
3678 assert!(parse_model_request(br#"{"no":"messages"}"#).is_none());
3679 }
3680
3681 use crate::provider::{MockProvider, ModelResponse, Provider, ProviderError, ProviderRegistry};
3684 use axum::extract::State;
3685 use std::collections::HashMap;
3686 use std::sync::Arc;
3687 use tokio::sync::mpsc;
3688
3689 fn model_resp(model: &str, text: &str) -> ModelResponse {
3690 ModelResponse {
3691 model: model.to_owned(),
3692 text: text.to_owned(),
3693 in_tokens: 1000,
3694 cache_write_tokens: 0,
3695 cache_read_tokens: 0,
3696 out_tokens: 400,
3697 raw: serde_json::Value::Null,
3698 }
3699 }
3700
3701 fn enforce_state(
3704 ladder: &[&str],
3705 gates: &[&str],
3706 outcomes: Vec<(&str, Result<ModelResponse, ProviderError>)>,
3707 ) -> (AppState, mpsc::Receiver<Trace>) {
3708 let toml = format!(
3709 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\nladder = [{}]\ngates = [{}]\n",
3710 ladder
3711 .iter()
3712 .map(|m| format!("\"{m}\""))
3713 .collect::<Vec<_>>()
3714 .join(", "),
3715 gates
3716 .iter()
3717 .map(|g| format!("\"{g}\""))
3718 .collect::<Vec<_>>()
3719 .join(", "),
3720 );
3721 let config = ProxyConfig::from_lookup(|k| match k {
3722 "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
3723 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
3724 _ => None,
3725 })
3726 .unwrap();
3727
3728 let mut outs = HashMap::new();
3729 for (model, out) in outcomes {
3730 outs.insert(model.to_owned(), out);
3731 }
3732 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
3733 map.insert(
3734 "anthropic".to_owned(),
3735 Arc::new(MockProvider::new("anthropic", outs)),
3736 );
3737 let providers = ProviderRegistry::from_map(map);
3738
3739 let (traces, rx) = mpsc::channel(64);
3740 let state = AppState {
3741 config: Arc::new(config),
3742 http: reqwest::Client::new(),
3743 providers,
3744 gate_health: Arc::new(GateHealthRegistry::new()),
3745 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
3746 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
3747 traces,
3748 adaptive: None,
3749 bandit: None,
3750 promoter: None,
3751 verified_cache: None,
3752 predictor: None,
3753 tenant_rate_limiter: None,
3754 spill: None,
3755 };
3756 (state, rx)
3757 }
3758
3759 fn user_body() -> Bytes {
3760 Bytes::from_static(
3761 br#"{"model":"ignored","max_tokens":64,"messages":[{"role":"user","content":"hi"}]}"#,
3762 )
3763 }
3764
3765 async fn body_json(resp: Response) -> Value {
3766 let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
3767 .await
3768 .unwrap();
3769 serde_json::from_slice(&bytes).unwrap()
3770 }
3771
3772 #[tokio::test]
3773 async fn mode_profile_stamped_on_trace_when_non_balanced() {
3774 let (state, mut rx) = enforce_state(
3775 &["anthropic/claude-haiku-4-5"],
3776 &["non-empty"],
3777 vec![(
3778 "anthropic/claude-haiku-4-5",
3779 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
3780 )],
3781 );
3782 let mut headers = HeaderMap::new();
3783 headers.insert("x-firstpass-mode", "quality".parse().unwrap());
3784 let resp = messages(
3785 State(state),
3786 Extension(TenantId("default".to_owned())),
3787 headers,
3788 user_body(),
3789 )
3790 .await;
3791 assert_eq!(resp.status(), axum::http::StatusCode::OK);
3792 let trace = rx.try_recv().expect("trace enqueued");
3793 assert_eq!(
3794 trace.policy.mode_profile.as_deref(),
3795 Some("quality"),
3796 "mode_profile must be stamped when quality mode is active"
3797 );
3798 }
3799
3800 #[tokio::test]
3801 async fn mode_profile_absent_from_trace_when_balanced() {
3802 let (state, mut rx) = enforce_state(
3803 &["anthropic/claude-haiku-4-5"],
3804 &["non-empty"],
3805 vec![(
3806 "anthropic/claude-haiku-4-5",
3807 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
3808 )],
3809 );
3810 let resp = messages(
3812 State(state),
3813 Extension(TenantId("default".to_owned())),
3814 HeaderMap::new(),
3815 user_body(),
3816 )
3817 .await;
3818 assert_eq!(resp.status(), axum::http::StatusCode::OK);
3819 let trace = rx.try_recv().expect("trace enqueued");
3820 assert!(
3821 trace.policy.mode_profile.is_none(),
3822 "mode_profile must be absent when Balanced (byte-identical invariant)"
3823 );
3824 }
3825
3826 #[tokio::test]
3827 async fn enforce_serves_first_pass_and_returns_anthropic_shape() {
3828 let (state, mut rx) = enforce_state(
3829 &["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"],
3830 &["non-empty"],
3831 vec![(
3832 "anthropic/claude-haiku-4-5",
3833 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
3834 )],
3835 );
3836 let resp = messages(
3837 State(state),
3838 Extension(TenantId("default".to_owned())),
3839 HeaderMap::new(),
3840 user_body(),
3841 )
3842 .await;
3843 assert_eq!(resp.status(), axum::http::StatusCode::OK);
3844 let json = body_json(resp).await;
3845 assert_eq!(json["type"], "message");
3846 assert_eq!(json["content"][0]["text"], "hello");
3847 assert_eq!(json["model"], "anthropic/claude-haiku-4-5");
3848
3849 let trace = rx.try_recv().expect("a trace was enqueued");
3850 assert_eq!(trace.mode, Mode::Enforce);
3851 assert_eq!(trace.final_.served_rung, Some(0));
3852 assert_eq!(trace.attempts.len(), 1);
3853 }
3854
3855 #[tokio::test]
3856 async fn enforce_escalates_then_serves_and_traces_two_attempts() {
3857 let (state, mut rx) = enforce_state(
3858 &["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"],
3859 &["non-empty"],
3860 vec![
3861 (
3862 "anthropic/claude-haiku-4-5",
3863 Ok(model_resp("anthropic/claude-haiku-4-5", " ")),
3864 ), (
3866 "anthropic/claude-sonnet-5",
3867 Ok(model_resp("anthropic/claude-sonnet-5", "answer")),
3868 ),
3869 ],
3870 );
3871 let resp = messages(
3872 State(state),
3873 Extension(TenantId("default".to_owned())),
3874 HeaderMap::new(),
3875 user_body(),
3876 )
3877 .await;
3878 let json = body_json(resp).await;
3879 assert_eq!(json["content"][0]["text"], "answer");
3880
3881 let trace = rx.try_recv().expect("trace enqueued");
3882 assert_eq!(trace.attempts.len(), 2);
3883 assert_eq!(trace.final_.escalations, 1);
3884 assert_eq!(trace.final_.served_rung, Some(1));
3885 }
3886
3887 #[tokio::test]
3888 async fn enforce_all_rungs_error_returns_502() {
3889 let (state, mut rx) = enforce_state(
3890 &["anthropic/claude-haiku-4-5"],
3891 &["non-empty"],
3892 vec![(
3893 "anthropic/claude-haiku-4-5",
3894 Err(ProviderError::Transport("down".into())),
3895 )],
3896 );
3897 let resp = messages(
3898 State(state),
3899 Extension(TenantId("default".to_owned())),
3900 HeaderMap::new(),
3901 user_body(),
3902 )
3903 .await;
3904 assert_eq!(resp.status(), axum::http::StatusCode::BAD_GATEWAY);
3905 assert!(rx.try_recv().is_ok());
3907 }
3908
3909 #[tokio::test]
3910 async fn no_routing_config_falls_through_to_observe_not_enforce() {
3911 let config = ProxyConfig::from_lookup(|k| match k {
3915 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
3916 _ => None,
3917 })
3918 .unwrap();
3919 let (traces, _rx) = mpsc::channel(64);
3920 let state = AppState {
3921 config: Arc::new(config),
3922 http: reqwest::Client::new(),
3923 providers: ProviderRegistry::new("http://127.0.0.1:1", "http://127.0.0.1:1"),
3924 gate_health: Arc::new(GateHealthRegistry::new()),
3925 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
3926 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
3927 traces,
3928 adaptive: None,
3929 bandit: None,
3930 promoter: None,
3931 verified_cache: None,
3932 predictor: None,
3933 tenant_rate_limiter: None,
3934 spill: None,
3935 };
3936 let resp = messages(
3937 State(state),
3938 Extension(TenantId("default".to_owned())),
3939 HeaderMap::new(),
3940 user_body(),
3941 )
3942 .await;
3943 assert_ne!(resp.status(), axum::http::StatusCode::OK);
3945 }
3946
3947 fn bare_enforce_route() -> Route {
3950 use firstpass_core::config::{Match, Mode};
3951 Route {
3952 match_: Match::default(),
3953 mode: Mode::Enforce,
3954 ladder: vec!["anthropic/claude-haiku-4-5".to_owned()],
3955 gates: vec![],
3956 deferred_gates: vec![],
3957 routing_mode: None,
3958 rollout: None,
3959 shadow: None,
3960 }
3961 }
3962
3963 #[test]
3964 fn balanced_preset_application_is_noop() {
3965 let base_max_rungs = 3u32;
3967 let base_speculation = 2u32;
3968 let preset = RoutingMode::Balanced.preset();
3969 let max_rungs = if let Some(d) = preset.max_rungs_delta {
3970 (base_max_rungs as i32 + d).max(1) as u32
3971 } else {
3972 base_max_rungs
3973 };
3974 let speculation = preset.speculation.unwrap_or(base_speculation);
3975 let start_at_top = preset.start_at_top;
3976 assert_eq!(max_rungs, 3, "Balanced must not change max_rungs");
3977 assert_eq!(speculation, 2, "Balanced must not change speculation");
3978 assert!(!start_at_top, "Balanced must not set start_at_top");
3979 }
3980
3981 #[test]
3982 fn resolve_mode_header_wins_over_route_and_global() {
3983 let mut headers = HeaderMap::new();
3984 headers.insert("x-firstpass-mode", "cost".parse().unwrap());
3985 let mut route = bare_enforce_route();
3986 route.routing_mode = Some(RoutingMode::Quality); let mut config = test_config();
3988 config.default_routing_mode = RoutingMode::Max; assert_eq!(
3990 resolve_mode(&headers, &route, &config),
3991 RoutingMode::Cost,
3992 "header must win"
3993 );
3994 }
3995
3996 #[test]
3997 fn resolve_mode_route_wins_over_global_when_no_header() {
3998 let mut route = bare_enforce_route();
3999 route.routing_mode = Some(RoutingMode::Latency);
4000 let mut config = test_config();
4001 config.default_routing_mode = RoutingMode::Max;
4002 assert_eq!(
4003 resolve_mode(&HeaderMap::new(), &route, &config),
4004 RoutingMode::Latency,
4005 "route must beat global default"
4006 );
4007 }
4008
4009 #[test]
4010 fn resolve_mode_global_when_header_and_route_absent() {
4011 let mut config = test_config();
4012 config.default_routing_mode = RoutingMode::Quality;
4013 assert_eq!(
4014 resolve_mode(&HeaderMap::new(), &bare_enforce_route(), &config),
4015 RoutingMode::Quality
4016 );
4017 }
4018
4019 #[test]
4020 fn resolve_mode_unknown_header_falls_through_to_route() {
4021 let mut headers = HeaderMap::new();
4022 headers.insert("x-firstpass-mode", "turbo-mode".parse().unwrap());
4024 let mut route = bare_enforce_route();
4025 route.routing_mode = Some(RoutingMode::Cost);
4026 let config = test_config();
4027 assert_eq!(
4028 resolve_mode(&headers, &route, &config),
4029 RoutingMode::Cost,
4030 "unknown header value must fall through to route"
4031 );
4032 }
4033
4034 #[test]
4035 fn resolve_mode_header_case_insensitive() {
4036 let mut headers = HeaderMap::new();
4037 headers.insert("x-firstpass-mode", "QUALITY".parse().unwrap());
4038 assert_eq!(
4039 resolve_mode(&headers, &bare_enforce_route(), &test_config()),
4040 RoutingMode::Quality
4041 );
4042 }
4043
4044 #[test]
4045 fn resolve_mode_no_mode_set_returns_balanced() {
4046 assert_eq!(
4048 resolve_mode(&HeaderMap::new(), &bare_enforce_route(), &test_config()),
4049 RoutingMode::Balanced
4050 );
4051 }
4052
4053 #[test]
4054 fn capabilities_json_includes_routing_modes() {
4055 let modes: Vec<&'static str> = RoutingMode::ALL.iter().map(|m| m.as_str()).collect();
4056 assert!(modes.contains(&"balanced"));
4057 assert!(modes.contains(&"cost"));
4058 assert!(modes.contains(&"quality"));
4059 assert!(modes.contains(&"latency"));
4060 assert!(modes.contains(&"max"));
4061 assert!(modes.contains(&"observe"));
4062 }
4063
4064 #[test]
4065 fn model_list_dedups_across_ladders_keeping_cheapest_first_order() {
4066 let config = firstpass_core::config::Config::parse(
4069 r#"
4070 [[route]]
4071 mode = "enforce"
4072 ladder = ["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"]
4073 [[route]]
4074 mode = "enforce"
4075 ladder = ["anthropic/claude-sonnet-5", "anthropic/claude-opus-4-8"]
4076 "#,
4077 )
4078 .expect("test config parses");
4079
4080 let out = model_list(Some(&config));
4081
4082 let ids: Vec<&str> = out["data"]
4083 .as_array()
4084 .expect("data is an array")
4085 .iter()
4086 .map(|m| m["id"].as_str().expect("id is a string"))
4087 .collect();
4088 assert_eq!(
4089 ids,
4090 [
4091 "anthropic/claude-haiku-4-5",
4092 "anthropic/claude-sonnet-5",
4093 "anthropic/claude-opus-4-8",
4094 ],
4095 "sonnet appears once, and ladder order is preserved"
4096 );
4097 assert_eq!(out["object"], "list");
4098 assert_eq!(out["data"][0]["owned_by"], "anthropic");
4099 let haiku_in = out["data"][0]["firstpass"]["input_per_mtok"]
4102 .as_f64()
4103 .expect("a first-party rung has a price");
4104 assert!(haiku_in > 0.0, "expected a real price, got {haiku_in}");
4105 let opus_in = out["data"][2]["firstpass"]["input_per_mtok"]
4107 .as_f64()
4108 .expect("a first-party rung has a price");
4109 assert!(haiku_in < opus_in, "{haiku_in} should undercut {opus_in}");
4110 }
4111
4112 #[tokio::test]
4113 async fn healthz_names_the_service_so_a_client_can_tell_us_from_a_stranger() {
4114 let body = healthz().await.into_response();
4118 let bytes = axum::body::to_bytes(body.into_body(), 64 * 1024)
4119 .await
4120 .expect("healthz body");
4121 let json: serde_json::Value = serde_json::from_slice(&bytes).expect("healthz is json");
4122 assert_eq!(json["status"], "ok");
4123 assert_eq!(json["service"], "firstpass");
4124 }
4125
4126 #[test]
4127 fn model_list_is_empty_without_routing_config() {
4128 let out = model_list(None);
4129 assert_eq!(out["object"], "list");
4130 assert_eq!(
4131 out["data"].as_array().expect("data is an array").len(),
4132 0,
4133 "an observe-only deployment advertises no models"
4134 );
4135 }
4136
4137 #[test]
4138 fn detects_stream_requests() {
4139 assert!(is_stream_request(br#"{"stream": true}"#));
4140 assert!(!is_stream_request(br#"{"stream": false}"#));
4141 assert!(!is_stream_request(br#"{"model":"m"}"#));
4142 assert!(!is_stream_request(b"not json"));
4143 }
4144
4145 #[test]
4146 fn detects_tool_blocks_in_messages() {
4147 let with =
4148 br#"{"messages":[{"role":"user","content":[{"type":"tool_result","content":"42"}]}]}"#;
4149 let without = br#"{"messages":[{"role":"user","content":"hi"}]}"#;
4150 assert!(messages_have_tool_blocks(with));
4151 assert!(!messages_have_tool_blocks(without));
4152 }
4153
4154 #[test]
4155 fn enforce_only_handles_plain_text() {
4156 let plain =
4157 Bytes::from_static(br#"{"model":"m","messages":[{"role":"user","content":"hi"}]}"#);
4158 let tools = Bytes::from_static(
4159 br#"{"model":"m","tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
4160 );
4161 let f_plain = extract_features(&HeaderMap::new(), &plain);
4162 let f_tools = extract_features(&HeaderMap::new(), &tools);
4163 let anthropic_ladder = vec!["anthropic/claude-haiku-4-5".to_owned()];
4164 let providers = test_registry();
4165 assert!(enforce_can_handle(
4167 &f_plain,
4168 &plain,
4169 false,
4170 &anthropic_ladder,
4171 &providers,
4172 Dialect::Anthropic,
4173 ));
4174 assert!(!enforce_can_handle(
4175 &f_tools,
4176 &tools,
4177 false,
4178 &anthropic_ladder,
4179 &providers,
4180 Dialect::Anthropic,
4181 ));
4182 }
4183
4184 #[test]
4185 fn structured_enforce_routes_tools_and_streaming() {
4186 let tools = Bytes::from_static(
4189 br#"{"model":"m","tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
4190 );
4191 let streaming_tools = Bytes::from_static(
4192 br#"{"model":"m","stream":true,"tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
4193 );
4194 let f = extract_features(&HeaderMap::new(), &tools);
4195 let anthropic_ladder = vec![
4196 "anthropic/claude-haiku-4-5".to_owned(),
4197 "anthropic/claude-sonnet-5".to_owned(),
4198 ];
4199 let providers = test_registry();
4200 assert!(enforce_can_handle(
4201 &f,
4202 &tools,
4203 true,
4204 &anthropic_ladder,
4205 &providers,
4206 Dialect::Anthropic,
4207 ));
4208 assert!(enforce_can_handle(
4209 &f,
4210 &streaming_tools,
4211 true,
4212 &anthropic_ladder,
4213 &providers,
4214 Dialect::Anthropic,
4215 ));
4216 }
4217
4218 fn test_registry() -> crate::provider::ProviderRegistry {
4220 crate::provider::ProviderRegistry::new("http://localhost", "http://localhost")
4221 }
4222
4223 #[test]
4224 fn fidelity_guard_blocks_structured_on_non_verbatim_ladder() {
4225 let tools = Bytes::from_static(
4229 br#"{"model":"m","tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
4230 );
4231 let plain =
4232 Bytes::from_static(br#"{"model":"m","messages":[{"role":"user","content":"hi"}]}"#);
4233 let f_tools = extract_features(&HeaderMap::new(), &tools);
4234 let f_plain = extract_features(&HeaderMap::new(), &plain);
4235 let providers = test_registry();
4236 let mixed_ladder = vec![
4237 "openai/gpt-4.1-mini".to_owned(),
4238 "anthropic/claude-sonnet-5".to_owned(),
4239 ];
4240 assert!(!enforce_can_handle(
4241 &f_tools,
4242 &tools,
4243 true,
4244 &mixed_ladder,
4245 &providers,
4246 Dialect::Anthropic,
4247 ));
4248 assert!(enforce_can_handle(
4249 &f_plain,
4250 &plain,
4251 true,
4252 &mixed_ladder,
4253 &providers,
4254 Dialect::Anthropic,
4255 ));
4256 }
4257
4258 #[test]
4259 fn enforce_sse_reemission_preserves_text_and_tool_use() {
4260 let resp = ModelResponse {
4263 model: "anthropic/claude-haiku-4-5".to_owned(),
4264 text: "let me check".to_owned(),
4265 in_tokens: 5,
4266 cache_write_tokens: 0,
4267 cache_read_tokens: 0,
4268 out_tokens: 7,
4269 raw: serde_json::json!({
4270 "content": [
4271 { "type": "text", "text": "let me check" },
4272 { "type": "tool_use", "id": "tu_1", "name": "get_weather", "input": { "city": "Paris" } }
4273 ]
4274 }),
4275 };
4276 let sse = anthropic_sse_from_message(&anthropic_response_json(&resp));
4277
4278 let frames: Vec<Value> = sse
4280 .lines()
4281 .filter_map(|l| l.strip_prefix("data: "))
4282 .map(|d| serde_json::from_str::<Value>(d).expect("each SSE data frame is valid JSON"))
4283 .collect();
4284
4285 assert_eq!(frames.first().unwrap()["type"], "message_start");
4287 assert_eq!(frames.last().unwrap()["type"], "message_stop");
4288 assert!(frames.iter().any(|f| f["delta"]["type"] == "text_delta"
4290 && f["delta"]["text"] == "let me check"));
4291 assert!(
4294 frames
4295 .iter()
4296 .any(|f| f["content_block"]["type"] == "tool_use"
4297 && f["content_block"]["name"] == "get_weather"
4298 && f["content_block"]["id"] == "tu_1")
4299 );
4300 assert!(
4301 frames
4302 .iter()
4303 .any(|f| f["delta"]["type"] == "input_json_delta"
4304 && f["delta"]["partial_json"] == r#"{"city":"Paris"}"#)
4305 );
4306 }
4307
4308 #[tokio::test]
4313 async fn enforce_falls_back_to_observe_for_tool_requests() {
4314 let toml = "[[price]]\nmodel = \"anthropic/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/m\"]\ngates = [\"non-empty\"]\n";
4315 let config = ProxyConfig::from_lookup(|k| match k {
4316 "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
4317 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
4318 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
4319 _ => None,
4320 })
4321 .unwrap();
4322 let mut outs = HashMap::new();
4323 outs.insert(
4324 "anthropic/m".to_owned(),
4325 Ok(model_resp("anthropic/m", "hello")),
4326 );
4327 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
4328 map.insert(
4329 "anthropic".to_owned(),
4330 Arc::new(MockProvider::new("anthropic", outs)),
4331 );
4332 let (traces, _rx) = mpsc::channel(64);
4333 let state = AppState {
4334 config: Arc::new(config),
4335 http: reqwest::Client::new(),
4336 providers: ProviderRegistry::from_map(map),
4337 gate_health: Arc::new(GateHealthRegistry::new()),
4338 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
4339 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
4340 traces,
4341 adaptive: None,
4342 bandit: None,
4343 promoter: None,
4344 verified_cache: None,
4345 predictor: None,
4346 tenant_rate_limiter: None,
4347 spill: None,
4348 };
4349
4350 let plain =
4352 Bytes::from_static(br#"{"model":"m","messages":[{"role":"user","content":"hi"}]}"#);
4353 let resp = messages(
4354 State(state.clone()),
4355 Extension(TenantId("default".to_owned())),
4356 HeaderMap::new(),
4357 plain,
4358 )
4359 .await;
4360 assert_eq!(
4361 resp.status(),
4362 axum::http::StatusCode::OK,
4363 "plain text should enforce"
4364 );
4365
4366 let tools = Bytes::from_static(
4369 br#"{"model":"m","tools":[{"name":"get_weather"}],"messages":[{"role":"user","content":"hi"}]}"#,
4370 );
4371 let resp = messages(
4372 State(state.clone()),
4373 Extension(TenantId("default".to_owned())),
4374 HeaderMap::new(),
4375 tools.clone(),
4376 )
4377 .await;
4378 assert_eq!(
4379 resp.status(),
4380 axum::http::StatusCode::OK,
4381 "tool request must route through enforce by default (ADR 0005 default-on)"
4382 );
4383
4384 let toml_off = "[[price]]\nmodel = \"anthropic/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/m\"]\ngates = [\"non-empty\"]\n[escalation]\nenforce_structured = false\n";
4387 let config_off = ProxyConfig::from_lookup(|k| match k {
4388 "FIRSTPASS_CONFIG_TOML" => Some(toml_off.to_owned()),
4389 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
4390 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
4391 _ => None,
4392 })
4393 .unwrap();
4394 let state_off = AppState {
4395 config: Arc::new(config_off),
4396 ..state.clone()
4397 };
4398 let resp = messages(
4399 State(state_off),
4400 Extension(TenantId("default".to_owned())),
4401 HeaderMap::new(),
4402 tools,
4403 )
4404 .await;
4405 assert_ne!(
4406 resp.status(),
4407 axum::http::StatusCode::OK,
4408 "with enforce_structured = false a tool request must fall back to observe"
4409 );
4410
4411 let toolres = Bytes::from_static(
4413 br#"{"model":"m","messages":[{"role":"user","content":[{"type":"tool_result","tool_use_id":"x","content":"42"}]}]}"#,
4414 );
4415 let resp = messages(
4416 State(state),
4417 Extension(TenantId("default".to_owned())),
4418 HeaderMap::new(),
4419 toolres,
4420 )
4421 .await;
4422 assert_eq!(
4423 resp.status(),
4424 axum::http::StatusCode::OK,
4425 "tool_result blocks route through enforce by default too (verbatim carry)"
4426 );
4427 }
4428
4429 #[tokio::test]
4433 async fn panel_is_public_markup_but_receipts_stay_behind_auth() {
4434 use tower::ServiceExt;
4435 let (state, _rx) = enforce_state(
4436 &["anthropic/claude-haiku-4-5"],
4437 &["non-empty"],
4438 vec![(
4439 "anthropic/claude-haiku-4-5",
4440 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
4441 )],
4442 );
4443 let router = app(state).expect("prometheus recorder installs");
4444
4445 let page = router
4446 .clone()
4447 .oneshot(
4448 axum::http::Request::builder()
4449 .uri("/panel")
4450 .body(Body::empty())
4451 .unwrap(),
4452 )
4453 .await
4454 .unwrap();
4455 assert_eq!(page.status(), axum::http::StatusCode::OK);
4456 let html = axum::body::to_bytes(page.into_body(), 1 << 20)
4457 .await
4458 .unwrap();
4459 let html = String::from_utf8_lossy(&html);
4460 assert!(
4461 html.contains("firstpass") && html.contains("/v1/receipts"),
4462 "the page must fetch its data rather than embed it"
4463 );
4464 assert!(
4467 !html.contains("http://") && !html.contains("https://"),
4468 "the panel must not reference any external origin"
4469 );
4470 assert!(html.contains("const esc ="), "the escaper must exist");
4475 for raw in [
4476 "${a.model}",
4477 "${a.rung}",
4478 "${a.verdict}",
4479 "${r.trace_id}",
4480 "${r.served_rung}",
4481 ] {
4482 assert!(
4483 !html.contains(raw),
4484 "receipt field interpolated unescaped into innerHTML: {raw}"
4485 );
4486 }
4487
4488 let data = router
4489 .oneshot(
4490 axum::http::Request::builder()
4491 .uri("/v1/receipts?limit=5")
4492 .body(Body::empty())
4493 .unwrap(),
4494 )
4495 .await
4496 .unwrap();
4497 assert_eq!(data.status(), axum::http::StatusCode::OK);
4501 let body = axum::body::to_bytes(data.into_body(), 1 << 20)
4502 .await
4503 .unwrap();
4504 let v: Value = serde_json::from_slice(&body).unwrap();
4505 assert!(v.get("receipts").and_then(Value::as_array).is_some());
4506 }
4507
4508 async fn feedback_state() -> (AppState, std::path::PathBuf, String) {
4512 let db = std::env::temp_dir().join(format!("firstpass-feedback-{}.db", Uuid::now_v7()));
4513 let (tx, handle) = crate::store::open(&db).unwrap();
4514
4515 let mut trace = build_error_trace(
4516 &ProxyConfig::from_lookup(|_| None).unwrap(),
4517 &Bytes::from_static(b"{}"),
4518 5,
4519 Some("sess-fb"),
4520 );
4521 trace.attempts.push(Attempt {
4522 rung: 0,
4523 model: "anthropic/claude-haiku-4-5".into(),
4524 provider: "anthropic".into(),
4525 in_tokens: 10,
4526 cache_write_tokens: 0,
4527 cache_read_tokens: 0,
4528 out_tokens: 5,
4529 cost_usd: 0.001,
4530 latency_ms: 5,
4531 gates: vec![],
4532 verdict: Verdict::Pass,
4533 });
4534 let trace_id = trace.trace_id.to_string();
4535 tx.try_send(trace).unwrap();
4536 drop(tx);
4537 handle.await.unwrap();
4538
4539 let db_str = db.to_string_lossy().into_owned();
4540 let config = ProxyConfig::from_lookup(move |k| match k {
4541 "FIRSTPASS_DB" => Some(db_str.clone()),
4542 _ => None,
4543 })
4544 .unwrap();
4545 let (traces, _rx) = mpsc::channel(64);
4546 let state = AppState {
4547 config: Arc::new(config),
4548 http: reqwest::Client::new(),
4549 providers: ProviderRegistry::new("http://127.0.0.1:1", "http://127.0.0.1:1"),
4550 gate_health: Arc::new(GateHealthRegistry::new()),
4551 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
4552 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
4553 traces,
4554 adaptive: None,
4555 bandit: None,
4556 promoter: None,
4557 verified_cache: None,
4558 predictor: None,
4559 tenant_rate_limiter: None,
4560 spill: None,
4561 };
4562 (state, db, trace_id)
4563 }
4564
4565 #[tokio::test]
4566 async fn feedback_nudges_the_adaptive_threshold() {
4567 use firstpass_core::conformal::AdaptiveConformal;
4568 let (mut state, _db, trace_id) = feedback_state().await;
4569 let aci = Arc::new(std::sync::Mutex::new(AdaptiveConformal::new(0.1, 0.2, 0.5)));
4570 state.adaptive = Some(aci.clone());
4571 let before = aci.lock().unwrap().threshold();
4572
4573 let fail = Bytes::from(
4575 serde_json::json!({ "trace_id": trace_id, "gate_id": "tests", "verdict": "fail", "reporter": "ci" })
4576 .to_string(),
4577 );
4578 assert_eq!(
4579 feedback(
4580 State(state.clone()),
4581 Extension(TenantId("default".to_owned())),
4582 fail
4583 )
4584 .await
4585 .status(),
4586 axum::http::StatusCode::ACCEPTED
4587 );
4588 let after_fail = aci.lock().unwrap().threshold();
4589 assert!(
4590 after_fail > before,
4591 "served fail should raise the live threshold: {before} -> {after_fail}"
4592 );
4593
4594 let pass = Bytes::from(
4596 serde_json::json!({ "trace_id": trace_id, "gate_id": "tests", "verdict": "pass", "reporter": "ci" })
4597 .to_string(),
4598 );
4599 let _ = feedback(
4600 State(state),
4601 Extension(TenantId("default".to_owned())),
4602 pass,
4603 )
4604 .await;
4605 assert!(aci.lock().unwrap().threshold() < after_fail);
4606 }
4607
4608 #[tokio::test]
4609 async fn feedback_records_a_deferred_verdict_without_breaking_the_chain() {
4610 let (state, db, trace_id) = feedback_state().await;
4611 let body = Bytes::from(
4612 serde_json::json!({
4613 "trace_id": trace_id,
4614 "gate_id": "tests",
4615 "verdict": "pass",
4616 "score": 1.0,
4617 "reporter": "ci",
4618 })
4619 .to_string(),
4620 );
4621 let resp = feedback(
4622 State(state),
4623 Extension(TenantId("default".to_owned())),
4624 body,
4625 )
4626 .await;
4627 assert_eq!(resp.status(), axum::http::StatusCode::ACCEPTED);
4628
4629 let view = crate::store::load_trace_view(&db, "default", &trace_id)
4631 .unwrap()
4632 .unwrap();
4633 assert_eq!(view.deferred.len(), 1);
4634 assert_eq!(view.deferred[0].gate_id, "tests");
4635 let traces = crate::store::load_all_traces(&db).unwrap();
4637 firstpass_core::verify_chain(&traces, GENESIS_HASH).unwrap();
4638
4639 let _ = std::fs::remove_file(&db);
4640 }
4641
4642 #[tokio::test]
4643 async fn feedback_for_unknown_trace_is_404() {
4644 let (state, db, _trace_id) = feedback_state().await;
4645 let body = Bytes::from(
4646 serde_json::json!({
4647 "trace_id": "does-not-exist",
4648 "gate_id": "tests",
4649 "verdict": "pass",
4650 "reporter": "ci",
4651 })
4652 .to_string(),
4653 );
4654 let resp = feedback(
4655 State(state),
4656 Extension(TenantId("default".to_owned())),
4657 body,
4658 )
4659 .await;
4660 assert_eq!(resp.status(), axum::http::StatusCode::NOT_FOUND);
4661 let _ = std::fs::remove_file(&db);
4662 }
4663
4664 #[tokio::test]
4667 async fn feedback_across_tenants_is_404_not_403() {
4668 let (state, db, trace_id) = feedback_state().await;
4669 let body = Bytes::from(
4670 serde_json::json!({
4671 "trace_id": trace_id,
4672 "gate_id": "tests",
4673 "verdict": "pass",
4674 "score": 1.0,
4675 "reporter": "attacker",
4676 })
4677 .to_string(),
4678 );
4679 let resp = feedback(
4681 State(state),
4682 Extension(TenantId("tenant-b".to_owned())),
4683 body,
4684 )
4685 .await;
4686 assert_eq!(
4687 resp.status(),
4688 axum::http::StatusCode::NOT_FOUND,
4689 "cross-tenant feedback must look exactly like a missing trace"
4690 );
4691 let _ = std::fs::remove_file(&db);
4692 }
4693
4694 #[tokio::test]
4695 async fn feedback_rejects_bad_verdict_and_score() {
4696 let (state, db, trace_id) = feedback_state().await;
4697 let bad_verdict = Bytes::from(
4698 serde_json::json!({ "trace_id": trace_id, "gate_id": "g", "verdict": "maybe", "reporter": "x" })
4699 .to_string(),
4700 );
4701 assert_eq!(
4702 feedback(
4703 State(state.clone()),
4704 Extension(TenantId("default".to_owned())),
4705 bad_verdict
4706 )
4707 .await
4708 .status(),
4709 axum::http::StatusCode::BAD_REQUEST
4710 );
4711 let bad_score = Bytes::from(
4712 serde_json::json!({ "trace_id": trace_id, "gate_id": "g", "verdict": "pass", "score": 9.0, "reporter": "x" })
4713 .to_string(),
4714 );
4715 assert_eq!(
4716 feedback(
4717 State(state),
4718 Extension(TenantId("default".to_owned())),
4719 bad_score
4720 )
4721 .await
4722 .status(),
4723 axum::http::StatusCode::BAD_REQUEST
4724 );
4725 let _ = std::fs::remove_file(&db);
4726 }
4727
4728 #[tokio::test]
4729 async fn metrics_endpoint_renders_after_a_real_request() {
4730 use tower::ServiceExt;
4731
4732 let (state, mut rx) = enforce_state(
4733 &["anthropic/claude-haiku-4-5"],
4734 &["non-empty"],
4735 vec![(
4736 "anthropic/claude-haiku-4-5",
4737 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
4738 )],
4739 );
4740 let router = app(state).expect("prometheus recorder installs");
4741
4742 let req = axum::http::Request::builder()
4743 .method("POST")
4744 .uri("/v1/messages")
4745 .header("content-type", "application/json")
4746 .body(Body::from(user_body()))
4747 .unwrap();
4748 let resp = router.clone().oneshot(req).await.unwrap();
4749 assert_eq!(resp.status(), axum::http::StatusCode::OK);
4750 rx.try_recv().expect("a trace was enqueued");
4751
4752 let metrics_req = axum::http::Request::builder()
4753 .method("GET")
4754 .uri("/metrics")
4755 .body(Body::empty())
4756 .unwrap();
4757 let metrics_resp = router.oneshot(metrics_req).await.unwrap();
4758 assert_eq!(metrics_resp.status(), axum::http::StatusCode::OK);
4759 let bytes = axum::body::to_bytes(metrics_resp.into_body(), 1 << 20)
4760 .await
4761 .unwrap();
4762 let body = String::from_utf8(bytes.to_vec()).unwrap();
4763 assert!(
4764 body.contains("firstpass_enforce_latency_ms"),
4765 "metrics body missing enforce latency histogram: {body}"
4766 );
4767 assert!(
4768 body.contains("firstpass_served_total"),
4769 "metrics body missing served counter: {body}"
4770 );
4771 }
4772
4773 #[tokio::test]
4779 async fn metrics_are_dimensioned_by_provider_rung_and_gate() {
4780 use tower::ServiceExt;
4781
4782 let (state, mut rx) = enforce_state(
4783 &["anthropic/claude-haiku-4-5"],
4784 &["non-empty"],
4785 vec![(
4786 "anthropic/claude-haiku-4-5",
4787 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
4788 )],
4789 );
4790 let router = app(state).expect("prometheus recorder installs");
4791
4792 let req = axum::http::Request::builder()
4793 .method("POST")
4794 .uri("/v1/messages")
4795 .header("content-type", "application/json")
4796 .body(Body::from(user_body()))
4797 .unwrap();
4798 assert_eq!(
4799 router.clone().oneshot(req).await.unwrap().status(),
4800 axum::http::StatusCode::OK
4801 );
4802 rx.try_recv().expect("a trace was enqueued");
4803
4804 let metrics_req = axum::http::Request::builder()
4805 .method("GET")
4806 .uri("/metrics")
4807 .body(Body::empty())
4808 .unwrap();
4809 let bytes = axum::body::to_bytes(
4810 router.oneshot(metrics_req).await.unwrap().into_body(),
4811 1 << 20,
4812 )
4813 .await
4814 .unwrap();
4815 let body = String::from_utf8(bytes.to_vec()).unwrap();
4816
4817 assert!(
4819 body.contains(r#"firstpass_attempt_total{provider="anthropic",rung="0"}"#),
4820 "attempt counter is not dimensioned by provider+rung: {body}"
4821 );
4822 assert!(
4823 body.contains("firstpass_attempt_latency_ms")
4824 && body.contains(r#"provider="anthropic""#),
4825 "attempt latency is not dimensioned by provider: {body}"
4826 );
4827 assert!(
4828 body.contains("firstpass_attempt_cost_usd_total"),
4829 "per-attempt cost series missing: {body}"
4830 );
4831 assert!(
4833 body.contains(r#"firstpass_gate_verdict_total{gate_id="non-empty",verdict="pass"}"#),
4834 "gate verdicts are not dimensioned by gate_id+verdict: {body}"
4835 );
4836 }
4837
4838 fn auth_state(require_auth: bool, keys_json: Option<String>) -> AppState {
4842 auth_state_rated(require_auth, keys_json, None)
4843 }
4844
4845 fn auth_state_rated(
4848 require_auth: bool,
4849 keys_json: Option<String>,
4850 rate_per_sec: Option<u32>,
4851 ) -> AppState {
4852 let config = ProxyConfig::from_lookup(|k| match k {
4853 "FIRSTPASS_REQUIRE_AUTH" => require_auth.then(|| "true".to_owned()),
4854 "FIRSTPASS_TENANT_KEYS_JSON" => keys_json.clone(),
4855 "FIRSTPASS_TENANT_RATE_PER_SEC" => rate_per_sec.map(|n| n.to_string()),
4856 _ => None,
4857 })
4858 .unwrap();
4859 let (traces, _rx) = mpsc::channel(64);
4860 std::mem::forget(_rx);
4863 let providers: HashMap<String, Arc<dyn Provider>> = HashMap::new();
4864 let tenant_rate_limiter = build_tenant_rate_limiter(&config);
4865 AppState {
4866 config: Arc::new(config),
4867 http: reqwest::Client::new(),
4868 providers: ProviderRegistry::from_map(providers),
4869 gate_health: Arc::new(GateHealthRegistry::new()),
4870 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
4871 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
4872 traces,
4873 adaptive: None,
4874 bandit: None,
4875 promoter: None,
4876 verified_cache: None,
4877 predictor: None,
4878 tenant_rate_limiter,
4879 spill: None,
4880 }
4881 }
4882
4883 fn cap_request(auth_header: Option<&str>) -> axum::http::Request<Body> {
4884 let mut b = axum::http::Request::builder()
4885 .method("GET")
4886 .uri("/v1/capabilities");
4887 if let Some(h) = auth_header {
4888 b = b.header("authorization", h);
4889 }
4890 b.body(Body::empty()).unwrap()
4891 }
4892
4893 #[tokio::test]
4894 async fn auth_off_allows_unauthenticated_request() {
4895 use tower::ServiceExt;
4896 let router = app(auth_state(false, None)).expect("router");
4897 let resp = router.oneshot(cap_request(None)).await.unwrap();
4898 assert_eq!(resp.status(), axum::http::StatusCode::OK);
4900 }
4901
4902 #[tokio::test]
4903 async fn auth_on_missing_key_is_401_opaque() {
4904 use tower::ServiceExt;
4905 let hash = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
4906 let keys = format!("{{\"tenant-a\": {hash:?}}}");
4907 let router = app(auth_state(true, Some(keys))).expect("router");
4908
4909 let resp = router.oneshot(cap_request(None)).await.unwrap();
4910 assert_eq!(resp.status(), axum::http::StatusCode::UNAUTHORIZED);
4911 let json = body_json(resp).await;
4912 assert_eq!(json["error"]["type"], "unauthorized");
4913 let msg = json["error"]["message"].as_str().unwrap();
4915 assert!(!msg.contains("tenant"), "no tenant oracle in body: {msg}");
4916 }
4917
4918 #[tokio::test]
4919 async fn auth_on_invalid_key_is_401() {
4920 use tower::ServiceExt;
4921 let hash = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
4922 let keys = format!("{{\"tenant-a\": {hash:?}}}");
4923 let router = app(auth_state(true, Some(keys))).expect("router");
4924
4925 let resp = router
4926 .oneshot(cap_request(Some("Bearer wrong-key")))
4927 .await
4928 .unwrap();
4929 assert_eq!(resp.status(), axum::http::StatusCode::UNAUTHORIZED);
4930 }
4931
4932 #[tokio::test]
4933 async fn auth_on_valid_key_proceeds() {
4934 use tower::ServiceExt;
4935 let hash = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
4936 let keys = format!("{{\"tenant-a\": {hash:?}}}");
4937 let router = app(auth_state(true, Some(keys))).expect("router");
4938
4939 let resp = router
4941 .oneshot(cap_request(Some("Bearer tenant-a.key-a")))
4942 .await
4943 .unwrap();
4944 assert_eq!(resp.status(), axum::http::StatusCode::OK);
4946 }
4947
4948 fn two_tenant_state(rate_per_sec: Option<u32>) -> AppState {
4950 let hash_a = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
4951 let hash_b = crate::tenant_auth::TenantKeys::hash_key("key-b").unwrap();
4952 let keys = format!("{{\"tenant-a\": {hash_a:?}, \"tenant-b\": {hash_b:?}}}");
4953 auth_state_rated(true, Some(keys), rate_per_sec)
4954 }
4955
4956 #[tokio::test]
4957 async fn tenant_exceeding_rate_limit_gets_429_opaque() {
4958 use tower::ServiceExt;
4959 let router = app(two_tenant_state(Some(1))).expect("router");
4965
4966 let (r1, r2, r3, r4) = tokio::join!(
4967 router
4968 .clone()
4969 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
4970 router
4971 .clone()
4972 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
4973 router
4974 .clone()
4975 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
4976 router
4977 .clone()
4978 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
4979 );
4980 let responses = [r1.unwrap(), r2.unwrap(), r3.unwrap(), r4.unwrap()];
4981 let ok = responses
4982 .iter()
4983 .filter(|r| r.status() == axum::http::StatusCode::OK)
4984 .count();
4985 assert!(ok >= 1, "the burst's first request must pass");
4986 let limited: Vec<_> = responses
4987 .into_iter()
4988 .filter(|r| r.status() == axum::http::StatusCode::TOO_MANY_REQUESTS)
4989 .collect();
4990 assert!(
4991 !limited.is_empty(),
4992 "a 4-request burst against 1 req/sec must trip the limiter"
4993 );
4994
4995 let json = body_json(limited.into_iter().next().unwrap()).await;
4996 assert_eq!(json["error"]["type"], "rate_limited");
4997 let msg = json["error"]["message"].as_str().unwrap();
4999 assert!(!msg.contains('1'), "no limit value in body: {msg}");
5000 }
5001
5002 #[tokio::test]
5003 async fn rate_limit_buckets_are_independent_per_tenant() {
5004 use tower::ServiceExt;
5005 let router = app(two_tenant_state(Some(1))).expect("router");
5006
5007 let (a1, a2, a3) = tokio::join!(
5010 router
5011 .clone()
5012 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
5013 router
5014 .clone()
5015 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
5016 router
5017 .clone()
5018 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
5019 );
5020 let a_limited = [a1.unwrap(), a2.unwrap(), a3.unwrap()]
5021 .iter()
5022 .filter(|r| r.status() == axum::http::StatusCode::TOO_MANY_REQUESTS)
5023 .count();
5024 assert!(a_limited >= 1, "tenant A's burst must trip its limiter");
5025
5026 let b1 = router
5028 .clone()
5029 .oneshot(cap_request(Some("Bearer tenant-b.key-b")))
5030 .await
5031 .unwrap();
5032 assert_eq!(b1.status(), axum::http::StatusCode::OK);
5033 }
5034
5035 #[tokio::test]
5036 async fn rate_limit_unset_never_429s() {
5037 use tower::ServiceExt;
5038 let router = app(two_tenant_state(None)).expect("router");
5041 for _ in 0..20 {
5042 let resp = router
5043 .clone()
5044 .oneshot(cap_request(Some("Bearer tenant-a.key-a")))
5045 .await
5046 .unwrap();
5047 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5048 }
5049 }
5050
5051 #[test]
5054 fn u01_is_deterministic_and_in_range() {
5055 let s1 = u01(0xDEAD_BEEF_CAFE_1234_u128);
5056 let s2 = u01(0xDEAD_BEEF_CAFE_1234_u128);
5057 assert_eq!(s1, s2, "u01 must be deterministic for the same seed");
5058 assert!((0.0..1.0).contains(&s1), "u01 must return [0, 1), got {s1}");
5059
5060 let s3 = u01(0x1234_5678_9ABC_DEF0_u128);
5062 assert_ne!(s1, s3, "different seeds should give different values");
5063
5064 for i in 0u64..256 {
5066 let v = u01(i as u128);
5067 assert!((0.0..1.0).contains(&v), "seed {i}: u01={v} out of [0,1)");
5068 }
5069 }
5070
5071 #[test]
5072 fn epsilon_propensity_formula() {
5073 let epsilon = 0.2_f64;
5074 let k = 3_usize;
5075 let greedy = 1_u32;
5076
5077 let p_greedy = epsilon_propensity(greedy, greedy, epsilon, k);
5079 let expected_greedy = (1.0 - epsilon) + epsilon / k as f64;
5080 assert!(
5081 (p_greedy - expected_greedy).abs() < 1e-12,
5082 "{p_greedy} != {expected_greedy}"
5083 );
5084
5085 let p_other = epsilon_propensity(0, greedy, epsilon, k);
5087 let expected_other = epsilon / k as f64;
5088 assert!(
5089 (p_other - expected_other).abs() < 1e-12,
5090 "{p_other} != {expected_other}"
5091 );
5092
5093 for chosen in 0..k as u32 {
5095 let p = epsilon_propensity(chosen, greedy, epsilon, k);
5096 assert!(
5097 p > 0.0 && p <= 1.0,
5098 "propensity {p} out of (0,1] for chosen={chosen}"
5099 );
5100 }
5101 }
5102
5103 #[test]
5104 fn epsilon_branch_and_greedy_branch_both_occur_over_many_seeds() {
5105 let epsilon = 0.3_f64;
5107 let mut saw_explore = false;
5108 let mut saw_greedy = false;
5109 for i in 0u64..200 {
5110 let u = u01(i as u128);
5111 if u < epsilon {
5112 saw_explore = true;
5113 } else {
5114 saw_greedy = true;
5115 }
5116 if saw_explore && saw_greedy {
5117 break;
5118 }
5119 }
5120 assert!(
5121 saw_explore,
5122 "epsilon branch must fire with epsilon=0.3 over 200 seeds"
5123 );
5124 assert!(
5125 saw_greedy,
5126 "greedy branch must occur with epsilon=0.3 over 200 seeds"
5127 );
5128 }
5129
5130 #[test]
5131 fn epsilon_propensity_sums_to_one_over_all_rungs() {
5132 let epsilon = 0.15_f64;
5134 let k = 4_usize;
5135 let greedy = 2_u32;
5136 let total: f64 = (0..k as u32)
5137 .map(|r| epsilon_propensity(r, greedy, epsilon, k))
5138 .sum();
5139 assert!(
5140 (total - 1.0).abs() < 1e-12,
5141 "propensities must sum to 1, got {total}"
5142 );
5143 }
5144
5145 #[tokio::test(start_paused = true)]
5148 async fn keepalive_stream_ticks_then_emits_final_frame() {
5149 let (tx, rx) = tokio::sync::oneshot::channel::<Result<Value, ProxyError>>();
5150 let mut ticks = tokio::time::interval(SSE_KEEPALIVE_EVERY);
5151 ticks.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
5152 ticks.reset();
5153 let mut stream = KeepaliveStream {
5154 rx: Some(rx),
5155 ticks,
5156 format_message: anthropic_sse_from_message,
5157 };
5158 async fn next(
5160 stream: &mut KeepaliveStream,
5161 ) -> Option<Option<Result<Bytes, std::convert::Infallible>>> {
5162 std::future::poll_fn(|cx| {
5163 std::task::Poll::Ready(
5164 match futures_core::Stream::poll_next(std::pin::Pin::new(&mut *stream), cx) {
5165 std::task::Poll::Ready(item) => Some(item),
5166 std::task::Poll::Pending => None,
5167 },
5168 )
5169 })
5170 .await
5171 }
5172
5173 assert!(
5175 next(&mut stream).await.is_none(),
5176 "no frame before an interval"
5177 );
5178
5179 tokio::time::advance(SSE_KEEPALIVE_EVERY + Duration::from_millis(1)).await;
5181 let frame = next(&mut stream)
5182 .await
5183 .expect("keepalive due")
5184 .unwrap()
5185 .unwrap();
5186 assert!(
5187 frame.starts_with(b": "),
5188 "keepalive must be an SSE comment (ignored by every conforming parser)"
5189 );
5190
5191 let message = serde_json::json!({
5193 "id": "msg_1", "type": "message", "role": "assistant", "model": "m",
5194 "content": [{ "type": "text", "text": "done" }],
5195 "usage": { "input_tokens": 1, "output_tokens": 1 }
5196 });
5197 tx.send(Ok(message)).unwrap();
5198 let frame = next(&mut stream)
5199 .await
5200 .expect("final frame")
5201 .unwrap()
5202 .unwrap();
5203 let text = String::from_utf8(frame.to_vec()).unwrap();
5204 assert!(text.contains("event: message_start"));
5205 assert!(text.contains("event: message_stop"));
5206 let eos = next(&mut stream)
5207 .await
5208 .expect("stream must end after the final frame");
5209 assert!(eos.is_none(), "end-of-stream after the final frame");
5210 }
5211
5212 #[tokio::test]
5215 async fn streaming_enforce_serves_full_sse_sequence() {
5216 let toml = "[[price]]\nmodel = \"anthropic/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/m\"]\ngates = [\"non-empty\"]\n";
5217 let config = ProxyConfig::from_lookup(|k| match k {
5218 "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
5219 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
5220 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
5221 _ => None,
5222 })
5223 .unwrap();
5224 let mut outs = HashMap::new();
5225 outs.insert(
5226 "anthropic/m".to_owned(),
5227 Ok(model_resp("anthropic/m", "gated answer")),
5228 );
5229 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
5230 map.insert(
5231 "anthropic".to_owned(),
5232 Arc::new(MockProvider::new("anthropic", outs)),
5233 );
5234 let (traces, _rx) = mpsc::channel(64);
5235 let state = AppState {
5236 config: Arc::new(config),
5237 http: reqwest::Client::new(),
5238 providers: ProviderRegistry::from_map(map),
5239 gate_health: Arc::new(GateHealthRegistry::new()),
5240 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
5241 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
5242 traces,
5243 adaptive: None,
5244 bandit: None,
5245 promoter: None,
5246 verified_cache: None,
5247 predictor: None,
5248 tenant_rate_limiter: None,
5249 spill: None,
5250 };
5251 let body = Bytes::from_static(
5252 br#"{"model":"m","stream":true,"messages":[{"role":"user","content":"hi"}]}"#,
5253 );
5254 let resp = messages(
5255 State(state),
5256 Extension(TenantId("default".to_owned())),
5257 HeaderMap::new(),
5258 body,
5259 )
5260 .await;
5261 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5262 assert!(
5263 resp.headers()
5264 .get(axum::http::header::CONTENT_TYPE)
5265 .and_then(|v| v.to_str().ok())
5266 .is_some_and(|ct| ct.starts_with("text/event-stream")),
5267 "streaming client must get SSE"
5268 );
5269 let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
5270 .await
5271 .unwrap();
5272 let text = String::from_utf8(bytes.to_vec()).unwrap();
5273 assert!(text.contains("event: message_start"));
5274 assert!(text.contains("gated answer"));
5275 assert!(text.contains("event: message_stop"));
5276 }
5277
5278 #[test]
5283 fn parse_openai_request_plain_text() {
5284 let body = br#"{"model":"gpt-4o","max_tokens":256,"messages":[{"role":"user","content":"hello"}]}"#;
5286 let req = parse_openai_request(body, false).expect("must parse");
5287 assert_eq!(req.model, "gpt-4o");
5288 assert_eq!(req.max_tokens, 256);
5289 assert_eq!(req.messages.len(), 1);
5290 assert_eq!(req.messages[0].role, "user");
5291 assert_eq!(req.messages[0].content, Value::String("hello".to_owned()));
5292 assert!(req.system.is_none());
5293 assert_eq!(req.raw, Value::Null);
5295 }
5296
5297 #[test]
5298 fn parse_openai_request_system_message() {
5299 let body = br#"{"model":"gpt-4o","messages":[{"role":"system","content":"be concise"},{"role":"user","content":"hi"}]}"#;
5300 let req = parse_openai_request(body, false).expect("must parse");
5301 assert_eq!(req.system.as_deref(), Some("be concise"));
5302 assert_eq!(req.messages.len(), 1);
5303 assert_eq!(req.messages[0].role, "user");
5304 }
5305
5306 #[test]
5307 fn parse_openai_request_tool_calls_translate_to_tool_use() {
5308 let body = br#"{
5309 "model":"gpt-4o",
5310 "messages":[
5311 {"role":"user","content":"what's the weather?"},
5312 {"role":"assistant","content":null,"tool_calls":[
5313 {"id":"call_1","type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Paris\"}"}}
5314 ]},
5315 {"role":"tool","tool_call_id":"call_1","content":"15C, cloudy"}
5316 ]
5317 }"#;
5318 let req = parse_openai_request(body, false).expect("must parse");
5319 assert_eq!(req.messages.len(), 3);
5321 let asst = &req.messages[1];
5323 assert_eq!(asst.role, "assistant");
5324 let blocks = asst.content.as_array().expect("content array");
5325 assert_eq!(blocks[0]["type"], "tool_use");
5326 assert_eq!(blocks[0]["name"], "get_weather");
5327 assert_eq!(blocks[0]["id"], "call_1");
5328 assert_eq!(blocks[0]["input"]["city"], "Paris");
5329 let tool_msg = &req.messages[2];
5331 assert_eq!(tool_msg.role, "user");
5332 let result_blocks = tool_msg.content.as_array().expect("result blocks");
5333 assert_eq!(result_blocks[0]["type"], "tool_result");
5334 assert_eq!(result_blocks[0]["tool_use_id"], "call_1");
5335 }
5336
5337 #[test]
5338 fn parse_openai_request_tools_translate_to_anthropic_format() {
5339 let body = br#"{
5340 "model":"gpt-4o",
5341 "messages":[{"role":"user","content":"use a tool"}],
5342 "tools":[{"type":"function","function":{"name":"search","description":"web search","parameters":{"type":"object","properties":{"q":{"type":"string"}}}}}]
5343 }"#;
5344 let req = parse_openai_request(body, false).expect("must parse");
5345 let tools = req.tools.as_array().expect("tools array");
5346 assert_eq!(tools.len(), 1);
5347 assert_eq!(tools[0]["name"], "search");
5348 assert_eq!(tools[0]["description"], "web search");
5349 assert_eq!(tools[0]["input_schema"]["type"], "object");
5350 }
5351
5352 #[test]
5353 fn parse_openai_request_raw_carry_preserves_full_body() {
5354 let body =
5356 br#"{"model":"gpt-4o","max_tokens":100,"messages":[{"role":"user","content":"hi"}]}"#;
5357 let req = parse_openai_request(body, true).expect("must parse");
5358 assert!(req.raw.is_object(), "raw must be the full JSON object");
5359 assert_eq!(req.raw["model"], "gpt-4o");
5360 assert_eq!(req.raw["max_tokens"], 100);
5361 assert!(req.tools.is_null(), "no tools in this request");
5363 }
5364
5365 #[test]
5366 fn parse_openai_request_http_image_returns_none() {
5367 let body = br#"{"model":"gpt-4o","messages":[{"role":"user","content":[
5369 {"type":"text","text":"describe this"},
5370 {"type":"image_url","image_url":{"url":"https://example.com/cat.png"}}
5371 ]}]}"#;
5372 let result = parse_openai_request(body, false);
5373 assert!(result.is_none(), "http image URL must fail translation");
5374 }
5375
5376 #[test]
5377 fn parse_openai_request_data_url_image_translates_to_anthropic_base64() {
5378 let body = br#"{"model":"gpt-4o","messages":[{"role":"user","content":[
5379 {"type":"text","text":"describe"},
5380 {"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgo="}}
5381 ]}]}"#;
5382 let req = parse_openai_request(body, false).expect("data URL must parse");
5383 let blocks = req.messages[0].content.as_array().expect("blocks");
5384 let img = blocks
5385 .iter()
5386 .find(|b| b["type"] == "image")
5387 .expect("image block");
5388 assert_eq!(img["source"]["type"], "base64");
5389 assert_eq!(img["source"]["media_type"], "image/png");
5390 assert_eq!(img["source"]["data"], "iVBORw0KGgo=");
5391 }
5392
5393 #[test]
5396 fn openai_response_json_renders_text_response() {
5397 let resp = ModelResponse {
5398 model: "gpt-4o".to_owned(),
5399 text: "Hello!".to_owned(),
5400 in_tokens: 10,
5401 cache_write_tokens: 0,
5402 cache_read_tokens: 0,
5403 out_tokens: 5,
5404 raw: serde_json::json!({
5405 "content": [{ "type": "text", "text": "Hello!" }]
5406 }),
5407 };
5408 let json = openai_response_json(&resp);
5409 assert_eq!(json["object"], "chat.completion");
5410 assert_eq!(json["model"], "gpt-4o");
5411 assert_eq!(json["choices"][0]["message"]["role"], "assistant");
5412 assert_eq!(json["choices"][0]["message"]["content"], "Hello!");
5413 assert_eq!(json["choices"][0]["finish_reason"], "stop");
5414 assert_eq!(json["usage"]["prompt_tokens"], 10);
5415 assert_eq!(json["usage"]["completion_tokens"], 5);
5416 }
5417
5418 #[test]
5419 fn openai_response_json_renders_tool_call() {
5420 let resp = ModelResponse {
5421 model: "gpt-4o".to_owned(),
5422 text: String::new(),
5423 in_tokens: 20,
5424 cache_write_tokens: 0,
5425 cache_read_tokens: 0,
5426 out_tokens: 15,
5427 raw: serde_json::json!({
5428 "content": [{
5429 "type": "tool_use",
5430 "id": "call_abc",
5431 "name": "search",
5432 "input": {"q": "Rust async"}
5433 }]
5434 }),
5435 };
5436 let json = openai_response_json(&resp);
5437 assert_eq!(json["choices"][0]["finish_reason"], "tool_calls");
5438 let tc = &json["choices"][0]["message"]["tool_calls"][0];
5439 assert_eq!(tc["id"], "call_abc");
5440 assert_eq!(tc["type"], "function");
5441 assert_eq!(tc["function"]["name"], "search");
5442 assert_eq!(json["choices"][0]["message"]["content"], Value::Null);
5444 }
5445
5446 #[test]
5447 fn openai_sse_from_message_plain_text_has_role_then_content_then_stop() {
5448 let resp = ModelResponse {
5449 model: "gpt-4o".to_owned(),
5450 text: "Hi there!".to_owned(),
5451 in_tokens: 5,
5452 cache_write_tokens: 0,
5453 cache_read_tokens: 0,
5454 out_tokens: 3,
5455 raw: serde_json::json!({
5456 "content": [{ "type": "text", "text": "Hi there!" }]
5457 }),
5458 };
5459 let sse = openai_sse_from_message(&openai_response_json(&resp));
5460 for line in sse.lines() {
5462 assert!(
5463 line.is_empty() || line.starts_with("data: "),
5464 "bad SSE line: {line:?}"
5465 );
5466 }
5467 let frames: Vec<&str> = sse
5468 .lines()
5469 .filter_map(|l| l.strip_prefix("data: "))
5470 .collect();
5471 assert_eq!(*frames.last().unwrap(), "[DONE]");
5473 let role_frame: Value = serde_json::from_str(frames[0]).unwrap();
5475 assert_eq!(role_frame["choices"][0]["delta"]["role"], "assistant");
5476 assert!(frames.iter().any(|f| {
5478 if *f == "[DONE]" {
5479 return false;
5480 }
5481 serde_json::from_str::<Value>(f)
5482 .ok()
5483 .is_some_and(|v| v["choices"][0]["delta"]["content"] == "Hi there!")
5484 }));
5485 assert!(frames.iter().any(|f| {
5487 if *f == "[DONE]" {
5488 return false;
5489 }
5490 serde_json::from_str::<Value>(f)
5491 .ok()
5492 .is_some_and(|v| v["choices"][0]["finish_reason"] == "stop")
5493 }));
5494 }
5495
5496 #[test]
5499 fn detects_openai_tool_calls() {
5500 let with_tool_calls = Bytes::from_static(br#"{"messages":[
5501 {"role":"user","content":"hi"},
5502 {"role":"assistant","content":null,"tool_calls":[{"id":"c1","type":"function","function":{"name":"f","arguments":"{}"}}]}
5503 ]}"#);
5504 let without = Bytes::from_static(br#"{"messages":[{"role":"user","content":"hi"}]}"#);
5505 let with_tool_msg = Bytes::from_static(
5506 br#"{"messages":[
5507 {"role":"tool","tool_call_id":"c1","content":"result"}
5508 ]}"#,
5509 );
5510 assert!(openai_messages_have_tool_calls(&with_tool_calls));
5511 assert!(!openai_messages_have_tool_calls(&without));
5512 assert!(openai_messages_have_tool_calls(&with_tool_msg));
5513 }
5514
5515 #[test]
5516 fn detects_openai_http_images() {
5517 let http_img = Bytes::from_static(
5518 br#"{"messages":[{"role":"user","content":[
5519 {"type":"image_url","image_url":{"url":"https://example.com/img.png"}}
5520 ]}]}"#,
5521 );
5522 let data_img = Bytes::from_static(
5523 br#"{"messages":[{"role":"user","content":[
5524 {"type":"image_url","image_url":{"url":"data:image/png;base64,abc"}}
5525 ]}]}"#,
5526 );
5527 let no_img = Bytes::from_static(br#"{"messages":[{"role":"user","content":"hi"}]}"#);
5528 assert!(openai_has_http_images(&http_img));
5529 assert!(!openai_has_http_images(&data_img));
5530 assert!(!openai_has_http_images(&no_img));
5531 }
5532
5533 #[test]
5534 fn enforce_can_handle_openai_inbound_all_openai_ladder() {
5535 let tools_body = Bytes::from_static(br#"{"model":"gpt-4o","messages":[{"role":"assistant","content":null,"tool_calls":[{"id":"c","type":"function","function":{"name":"f","arguments":"{}"}}]}]}"#);
5537 let f = extract_openai_features(&HeaderMap::new(), &tools_body);
5538 let ladder = vec!["openai/gpt-4o-mini".to_owned(), "openai/gpt-4o".to_owned()];
5539 let providers = crate::provider::ProviderRegistry::new("http://x", "http://x");
5540 assert!(enforce_can_handle(
5541 &f,
5542 &tools_body,
5543 true,
5544 &ladder,
5545 &providers,
5546 Dialect::Openai
5547 ));
5548 }
5549
5550 #[test]
5551 fn enforce_can_handle_openai_inbound_all_anthropic_ladder_no_http_image() {
5552 let tools_body = Bytes::from_static(br#"{"model":"gpt-4o","messages":[{"role":"assistant","content":null,"tool_calls":[{"id":"c","type":"function","function":{"name":"f","arguments":"{}"}}]}]}"#);
5554 let f = extract_openai_features(&HeaderMap::new(), &tools_body);
5555 let ladder = vec!["anthropic/claude-haiku-4-5".to_owned()];
5556 let providers = crate::provider::ProviderRegistry::new("http://x", "http://x");
5557 assert!(enforce_can_handle(
5558 &f,
5559 &tools_body,
5560 true,
5561 &ladder,
5562 &providers,
5563 Dialect::Openai,
5564 ));
5565 }
5566
5567 #[test]
5568 fn enforce_can_handle_openai_inbound_http_image_falls_back() {
5569 let img_body = Bytes::from_static(
5571 br#"{"model":"gpt-4o","messages":[{"role":"user","content":[
5572 {"type":"image_url","image_url":{"url":"https://example.com/img.png"}}
5573 ]}]}"#,
5574 );
5575 let f = extract_openai_features(&HeaderMap::new(), &img_body);
5576 let ladder = vec!["anthropic/claude-haiku-4-5".to_owned()];
5577 let providers = crate::provider::ProviderRegistry::new("http://x", "http://x");
5578 assert!(!enforce_can_handle(
5579 &f,
5580 &img_body,
5581 true,
5582 &ladder,
5583 &providers,
5584 Dialect::Openai,
5585 ));
5586 }
5587
5588 fn openai_enforce_state(mock_resp: ModelResponse) -> AppState {
5592 let toml = "[[price]]\nmodel = \"mock/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"mock/m\"]\ngates = [\"non-empty\"]\n";
5593 let config = ProxyConfig::from_lookup(|k| match k {
5594 "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
5595 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
5596 _ => None,
5597 })
5598 .unwrap();
5599 let mut outs = HashMap::new();
5600 outs.insert("mock/m".to_owned(), Ok(mock_resp));
5601 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
5602 map.insert("mock".to_owned(), Arc::new(MockProvider::new("mock", outs)));
5603 let (traces, _rx) = mpsc::channel(64);
5604 std::mem::forget(_rx);
5605 let tenant_rate_limiter = build_tenant_rate_limiter(&config);
5606 AppState {
5607 config: Arc::new(config),
5608 http: reqwest::Client::new(),
5609 providers: ProviderRegistry::from_map(map),
5610 gate_health: Arc::new(GateHealthRegistry::new()),
5611 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
5612 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
5613 traces,
5614 adaptive: None,
5615 bandit: None,
5616 promoter: None,
5617 verified_cache: None,
5618 predictor: None,
5619 tenant_rate_limiter,
5620 spill: None,
5621 }
5622 }
5623
5624 #[tokio::test]
5625 async fn chat_completions_plain_text_enforce_returns_openai_shape() {
5626 let mock = model_resp("mock/m", "gated answer");
5627 let state = openai_enforce_state(mock);
5628 let body = Bytes::from_static(
5629 br#"{"model":"gpt-4o","messages":[{"role":"user","content":"hello"}]}"#,
5630 );
5631 let resp = chat_completions(
5632 State(state),
5633 Extension(TenantId("default".to_owned())),
5634 HeaderMap::new(),
5635 body,
5636 )
5637 .await;
5638 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5639 let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
5640 .await
5641 .unwrap();
5642 let json: Value = serde_json::from_slice(&bytes).expect("must be JSON");
5643 assert_eq!(json["object"], "chat.completion");
5644 assert_eq!(json["choices"][0]["message"]["role"], "assistant");
5645 assert_eq!(json["choices"][0]["message"]["content"], "gated answer");
5646 assert_eq!(json["choices"][0]["finish_reason"], "stop");
5647 assert!(
5648 json["id"]
5649 .as_str()
5650 .is_some_and(|id| id.starts_with("chatcmpl-"))
5651 );
5652 }
5653
5654 #[tokio::test]
5655 async fn chat_completions_stream_true_returns_sse_with_openai_chunks() {
5656 let mock = model_resp("mock/m", "gated answer");
5657 let state = openai_enforce_state(mock);
5658 let body = Bytes::from_static(
5659 br#"{"model":"gpt-4o","stream":true,"messages":[{"role":"user","content":"hello"}]}"#,
5660 );
5661 let resp = chat_completions(
5662 State(state),
5663 Extension(TenantId("default".to_owned())),
5664 HeaderMap::new(),
5665 body,
5666 )
5667 .await;
5668 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5669 assert!(
5670 resp.headers()
5671 .get(axum::http::header::CONTENT_TYPE)
5672 .and_then(|v| v.to_str().ok())
5673 .is_some_and(|ct| ct.starts_with("text/event-stream")),
5674 "stream:true must return SSE"
5675 );
5676 let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
5677 .await
5678 .unwrap();
5679 let text = String::from_utf8(bytes.to_vec()).unwrap();
5680 assert!(
5682 text.contains("chat.completion.chunk"),
5683 "must have OpenAI chunk frames"
5684 );
5685 assert!(text.contains("[DONE]"), "must end with [DONE]");
5686 assert!(
5687 text.contains("gated answer"),
5688 "content must be in the stream"
5689 );
5690 assert!(
5692 !text.contains("message_start"),
5693 "must not have Anthropic event types"
5694 );
5695 }
5696
5697 fn probe_state(
5701 sample_rate: f64,
5702 k: u32,
5703 outcomes: Vec<(&str, Result<ModelResponse, ProviderError>)>,
5704 ) -> (AppState, mpsc::Receiver<Trace>) {
5705 let toml = format!(
5706 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\nladder = [\"anthropic/claude-haiku-4-5\"]\ngates = [\"non-empty\"]\n\
5707 [escalation.probe]\nk = {k}\nsample_rate = {sample_rate}\n"
5708 );
5709 let config = ProxyConfig::from_lookup(|k_| match k_ {
5710 "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
5711 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
5712 _ => None,
5713 })
5714 .unwrap();
5715 let mut outs = HashMap::new();
5716 for (model, out) in outcomes {
5717 outs.insert(model.to_owned(), out);
5718 }
5719 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
5720 map.insert(
5721 "anthropic".to_owned(),
5722 Arc::new(MockProvider::new("anthropic", outs)),
5723 );
5724 let (traces, rx) = mpsc::channel(64);
5725 let state = AppState {
5726 config: Arc::new(config),
5727 http: reqwest::Client::new(),
5728 providers: ProviderRegistry::from_map(map),
5729 gate_health: Arc::new(GateHealthRegistry::new()),
5730 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
5731 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
5732 traces,
5733 adaptive: None,
5734 bandit: None,
5735 promoter: None,
5736 verified_cache: None,
5737 predictor: None,
5738 tenant_rate_limiter: None,
5739 spill: None,
5740 };
5741 (state, rx)
5742 }
5743
5744 async fn run_enforce_get_trace(state: AppState, mut rx: mpsc::Receiver<Trace>) -> Trace {
5746 let resp = messages(
5747 State(state),
5748 Extension(TenantId("default".to_owned())),
5749 HeaderMap::new(),
5750 user_body(),
5751 )
5752 .await;
5753 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5754 rx.try_recv().expect("trace must be enqueued")
5755 }
5756
5757 #[tokio::test]
5759 async fn probe_off_trace_has_no_probe_field() {
5760 let (state, rx) = enforce_state(
5761 &["anthropic/claude-haiku-4-5"],
5762 &["non-empty"],
5763 vec![(
5764 "anthropic/claude-haiku-4-5",
5765 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
5766 )],
5767 );
5768 assert!(
5769 state
5770 .config
5771 .routing
5772 .as_ref()
5773 .unwrap()
5774 .escalation
5775 .probe
5776 .is_none(),
5777 "probe must default to None"
5778 );
5779 let trace = run_enforce_get_trace(state, rx).await;
5780 assert!(
5781 trace.probe.is_none(),
5782 "probe=None config must not set trace.probe"
5783 );
5784 }
5785
5786 #[tokio::test]
5788 async fn probe_sample_rate_zero_never_fires() {
5789 let (state, rx) = probe_state(
5791 0.0,
5792 5,
5793 vec![(
5794 "anthropic/claude-haiku-4-5",
5795 Ok(model_resp("anthropic/claude-haiku-4-5", "hi")),
5796 )],
5797 );
5798 let trace = run_enforce_get_trace(state, rx).await;
5799 assert!(
5800 trace.probe.is_none(),
5801 "sample_rate=0.0 must never set trace.probe"
5802 );
5803 }
5804
5805 #[tokio::test]
5808 async fn probe_on_all_pass_sets_confident_pass() {
5809 let model = "anthropic/claude-haiku-4-5";
5811 let (state, mut rx) = probe_state(1.0, 3, vec![(model, Ok(model_resp(model, "hello")))]);
5812 let resp = messages(
5813 State(state),
5814 Extension(TenantId("default".to_owned())),
5815 HeaderMap::new(),
5816 user_body(),
5817 )
5818 .await;
5819 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5820 let json = body_json(resp).await;
5822 assert_eq!(
5823 json["content"][0]["text"], "hello",
5824 "served content unchanged"
5825 );
5826
5827 let trace = rx.try_recv().expect("trace enqueued");
5828 let sig = trace.probe.expect("probe must be set when sample_rate=1.0");
5829 assert_eq!(sig.k, 3);
5830 assert_eq!(sig.gate_pass_count, 3, "all 3 samples must pass non-empty");
5831 assert_eq!(
5832 sig.regime,
5833 firstpass_core::ProbeRegime::ConfidentPass,
5834 "all-pass → ConfidentPass"
5835 );
5836 assert!(
5837 sig.probe_cost_usd > 0.0,
5838 "k model calls must cost something"
5839 );
5840 }
5841
5842 #[tokio::test]
5845 async fn probe_on_all_fail_sets_confident_fail() {
5846 let model = "anthropic/claude-haiku-4-5";
5847 let (state, mut rx) = probe_state(1.0, 3, vec![(model, Ok(model_resp(model, "")))]);
5849 let resp = messages(
5851 State(state),
5852 Extension(TenantId("default".to_owned())),
5853 HeaderMap::new(),
5854 user_body(),
5855 )
5856 .await;
5857 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5858
5859 let trace = rx.try_recv().expect("trace enqueued");
5860 let sig = trace.probe.expect("probe must be set when sample_rate=1.0");
5861 assert_eq!(
5862 sig.gate_pass_count, 0,
5863 "empty response fails non-empty: all 0 pass"
5864 );
5865 assert_eq!(
5866 sig.regime,
5867 firstpass_core::ProbeRegime::ConfidentFail,
5868 "0 passes → ConfidentFail"
5869 );
5870 }
5871
5872 #[tokio::test]
5875 async fn probe_on_served_output_identical_to_probe_off() {
5876 let model = "anthropic/claude-haiku-4-5";
5877 let mk = |sample_rate: f64| {
5878 probe_state(
5879 sample_rate,
5880 2,
5881 vec![(model, Ok(model_resp(model, "gated answer")))],
5882 )
5883 };
5884
5885 let (state_off, mut rx_off) = mk(0.0);
5887 let resp_off = messages(
5888 State(state_off),
5889 Extension(TenantId("default".to_owned())),
5890 HeaderMap::new(),
5891 user_body(),
5892 )
5893 .await;
5894 let json_off = body_json(resp_off).await;
5895 let trace_off = rx_off.try_recv().unwrap();
5896
5897 let (state_on, mut rx_on) = mk(1.0);
5899 let resp_on = messages(
5900 State(state_on),
5901 Extension(TenantId("default".to_owned())),
5902 HeaderMap::new(),
5903 user_body(),
5904 )
5905 .await;
5906 let json_on = body_json(resp_on).await;
5907 let trace_on = rx_on.try_recv().unwrap();
5908
5909 assert_eq!(
5911 json_off["content"][0]["text"], json_on["content"][0]["text"],
5912 "served text must be identical regardless of probe"
5913 );
5914 assert!(
5916 (trace_off.final_.total_cost_usd - trace_on.final_.total_cost_usd).abs() < 1e-12,
5917 "total_cost_usd must not include probe cost: off={} on={}",
5918 trace_off.final_.total_cost_usd,
5919 trace_on.final_.total_cost_usd
5920 );
5921 assert!(trace_off.probe.is_none());
5923 assert!(trace_on.probe.is_some());
5924 assert!(
5926 trace_on.probe.as_ref().unwrap().probe_cost_usd > 0.0,
5927 "probe cost must be positive"
5928 );
5929 }
5930
5931 #[tokio::test]
5941 async fn probe_does_not_mutate_gate_health() {
5942 let model = "anthropic/claude-haiku-4-5";
5943 let (mut state, rx) = probe_state(1.0, 2, vec![(model, Ok(model_resp(model, "answer")))]);
5944
5945 let registry = GateHealthRegistry::new().with_budget("non-empty", 2, 0.4);
5948 registry.record("default", "non-empty", true);
5951 assert!(
5952 registry.enabled("default", "non-empty"),
5953 "gate must start enabled (window not full yet)"
5954 );
5955 state.gate_health = Arc::new(registry);
5956
5957 let trace = run_enforce_get_trace(state, rx).await;
5969 let sig = trace.probe.expect("probe must fire with sample_rate=1.0");
5970 assert_eq!(sig.k, 2);
5971 assert!(
5972 sig.gate_pass_count <= 2,
5973 "gate_pass_count must be in [0, k]"
5974 );
5975 }
5976
5977 fn predictor_state(
5979 enabled: bool,
5980 outcome: Result<ModelResponse, ProviderError>,
5981 ) -> (AppState, mpsc::Receiver<Trace>) {
5982 let toml = "[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/claude-haiku-4-5\"]\ngates = [\"non-empty\"]\n";
5983 let config = ProxyConfig::from_lookup(|k_| match k_ {
5984 "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
5985 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
5986 _ => None,
5987 })
5988 .unwrap();
5989 let mut outs = HashMap::new();
5990 outs.insert("anthropic/claude-haiku-4-5".to_owned(), outcome);
5991 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
5992 map.insert(
5993 "anthropic".to_owned(),
5994 Arc::new(MockProvider::new("anthropic", outs)),
5995 );
5996 let (traces, rx) = mpsc::channel(64);
5997 let predictor = enabled.then(|| {
5998 Arc::new(std::sync::Mutex::new(firstpass_core::PassPredictor::new(
5999 0.05, 1e-4,
6000 )))
6001 });
6002 let state = AppState {
6003 config: Arc::new(config),
6004 http: reqwest::Client::new(),
6005 providers: ProviderRegistry::from_map(map),
6006 gate_health: Arc::new(GateHealthRegistry::new()),
6007 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
6008 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
6009 traces,
6010 adaptive: None,
6011 bandit: None,
6012 promoter: None,
6013 verified_cache: None,
6014 predictor,
6015 tenant_rate_limiter: None,
6016 spill: None,
6017 };
6018 (state, rx)
6019 }
6020
6021 #[tokio::test]
6022 async fn predictor_off_leaves_predicted_pass_none() {
6023 let (state, rx) =
6024 predictor_state(false, Ok(model_resp("anthropic/claude-haiku-4-5", "ok")));
6025 let trace = run_enforce_get_trace(state, rx).await;
6026 assert!(
6027 trace.predicted_pass.is_none(),
6028 "predictor off => no field (byte-identical)"
6029 );
6030 let j = serde_json::to_string(&trace).unwrap();
6032 assert!(!j.contains("predicted_pass"), "None must be omitted: {j}");
6033 }
6034
6035 #[tokio::test]
6036 async fn predictor_on_records_shadow_prediction_and_serves_identically() {
6037 let (state_off, rx_off) = predictor_state(
6040 false,
6041 Ok(model_resp("anthropic/claude-haiku-4-5", "served answer")),
6042 );
6043 let off = run_enforce_get_trace(state_off, rx_off).await;
6044
6045 let (state_on, rx_on) = predictor_state(
6046 true,
6047 Ok(model_resp("anthropic/claude-haiku-4-5", "served answer")),
6048 );
6049 let on = run_enforce_get_trace(state_on, rx_on).await;
6050
6051 assert_eq!(
6052 on.final_.served_rung, off.final_.served_rung,
6053 "served rung identical"
6054 );
6055 assert_eq!(on.attempts.len(), off.attempts.len(), "same attempts");
6056 assert_eq!(
6057 on.final_.total_cost_usd, off.final_.total_cost_usd,
6058 "predictor never adds served cost"
6059 );
6060 let p = on
6061 .predicted_pass
6062 .expect("predictor on => predicted_pass recorded");
6063 assert!(p > 0.0 && p < 1.0, "shadow prediction in (0,1): {p}");
6064 }
6065
6066 fn rollout_state(
6068 percent: f64,
6069 key: &str,
6070 outcomes: Vec<(&str, Result<ModelResponse, ProviderError>)>,
6071 ) -> (AppState, mpsc::Receiver<Trace>) {
6072 let toml = format!(
6073 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
6074 ladder = [\"anthropic/claude-haiku-4-5\"]\ngates = [\"non-empty\"]\n\
6075 [route.rollout]\npercent = {percent}\nkey = \"{key}\"\n"
6076 );
6077 let config = ProxyConfig::from_lookup(|k| match k {
6078 "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
6079 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
6080 _ => None,
6081 })
6082 .unwrap();
6083 let mut outs = HashMap::new();
6084 for (model, out) in outcomes {
6085 outs.insert(model.to_owned(), out);
6086 }
6087 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
6088 map.insert(
6089 "anthropic".to_owned(),
6090 Arc::new(MockProvider::new("anthropic", outs)),
6091 );
6092 let (traces, rx) = mpsc::channel(64);
6093 let state = AppState {
6094 config: Arc::new(config),
6095 http: reqwest::Client::new(),
6096 providers: ProviderRegistry::from_map(map),
6097 gate_health: Arc::new(GateHealthRegistry::new()),
6098 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
6099 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
6100 traces,
6101 adaptive: None,
6102 bandit: None,
6103 promoter: None,
6104 verified_cache: None,
6105 predictor: None,
6106 tenant_rate_limiter: None,
6107 spill: None,
6108 };
6109 (state, rx)
6110 }
6111
6112 #[tokio::test]
6116 async fn rollout_at_zero_percent_never_enforces() {
6117 for i in 0..25 {
6118 let (state, mut rx) = rollout_state(
6119 0.0,
6120 "session",
6121 vec![(
6122 "anthropic/claude-haiku-4-5",
6123 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
6124 )],
6125 );
6126 let mut headers = HeaderMap::new();
6127 headers.insert("x-firstpass-session", format!("s{i}").parse().unwrap());
6128 let resp = messages(
6129 State(state),
6130 Extension(TenantId("default".to_owned())),
6131 headers,
6132 user_body(),
6133 )
6134 .await;
6135 let _ = resp;
6138 if let Ok(t) = rx.try_recv() {
6139 assert!(
6140 t.attempts.is_empty(),
6141 "0% rollout produced an enforced decision for session s{i}"
6142 );
6143 }
6144 }
6145 }
6146
6147 #[tokio::test]
6150 async fn rollout_at_hundred_percent_always_enforces() {
6151 for i in 0..25 {
6152 let (state, mut rx) = rollout_state(
6153 100.0,
6154 "session",
6155 vec![(
6156 "anthropic/claude-haiku-4-5",
6157 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
6158 )],
6159 );
6160 let mut headers = HeaderMap::new();
6161 headers.insert("x-firstpass-session", format!("s{i}").parse().unwrap());
6162 let resp = messages(
6163 State(state),
6164 Extension(TenantId("default".to_owned())),
6165 headers,
6166 user_body(),
6167 )
6168 .await;
6169 assert_eq!(resp.status(), axum::http::StatusCode::OK);
6170 let t = rx.try_recv().expect("trace enqueued");
6171 assert!(
6172 !t.attempts.is_empty(),
6173 "100% rollout skipped enforcement for session s{i}"
6174 );
6175 }
6176 }
6177
6178 #[tokio::test]
6186 async fn dispatch_arm_matches_the_predicted_arm_per_session() {
6187 let mut checked_enforced = 0;
6188 let mut checked_control = 0;
6189 for i in 0..24 {
6190 let session = format!("session-{i}");
6191 let (state, mut rx) = rollout_state(
6192 50.0,
6193 "session",
6194 vec![(
6195 "anthropic/claude-haiku-4-5",
6196 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
6197 )],
6198 );
6199 let expected = firstpass_core::rollout::decide(
6202 &state.config.prompt_salt,
6203 &firstpass_core::Rollout {
6204 percent: 50.0,
6205 key: firstpass_core::RolloutKey::Session,
6206 },
6207 &session,
6208 )
6209 .enforced;
6210
6211 let mut headers = HeaderMap::new();
6212 headers.insert("x-firstpass-session", session.parse().unwrap());
6213 let _ = messages(
6214 State(state),
6215 Extension(TenantId("default".to_owned())),
6216 headers,
6217 user_body(),
6218 )
6219 .await;
6220 let observed = rx
6221 .try_recv()
6222 .map(|t| !t.attempts.is_empty())
6223 .unwrap_or(false);
6224
6225 assert_eq!(
6226 observed,
6227 expected,
6228 "{session}: dispatch put it in the {} arm, bucketing predicted the {} arm",
6229 if observed { "enforced" } else { "control" },
6230 if expected { "enforced" } else { "control" }
6231 );
6232 if expected {
6233 checked_enforced += 1;
6234 } else {
6235 checked_control += 1;
6236 }
6237 }
6238 assert!(
6240 checked_enforced > 0 && checked_control > 0,
6241 "test saw only one arm ({checked_enforced} enforced, {checked_control} control)"
6242 );
6243 }
6244}