1use std::sync::Arc;
5use std::time::Instant;
6
7use axum::body::Body;
8use axum::extract::{Request, State};
9use axum::http::HeaderMap;
10use axum::middleware::Next;
11use axum::response::{IntoResponse, Response};
12use axum::routing::{get, post};
13use axum::{Extension, Json, Router};
14use bytes::Bytes;
15use firstpass_core::features::{hour_bucket, token_bucket};
16use firstpass_core::hashchain::sha256_hex;
17use firstpass_core::{
18 Attempt, DeferredVerdict, Dialect, FEATURE_VERSION, Features, FinalOutcome, GENESIS_HASH, Mode,
19 ModelRef, PolicyRef, ProbeRegime, ProbeSignal, RequestInfo, RoutingMode, Score, ServedFrom,
20 TaskKind, Trace, Verdict,
21};
22use serde::Deserialize;
23use serde_json::Value;
24use std::future::Future;
25use std::time::Duration;
26use tokio::sync::mpsc::error::TrySendError;
27use uuid::Uuid;
28
29use crate::config::ProxyConfig;
30use crate::error::ProxyError;
31use crate::gate::{GateHealthRegistry, aggregate_with_policy, resolve_gates};
32use crate::provider::{Auth, ChatMessage, ModelRequest, ModelResponse, ProviderRegistry};
33use crate::router::{EnforceCtx, EngineOutcome, route_enforce};
34use crate::store;
35use crate::tenant_auth::{TenantId, auth_middleware};
36use crate::upstream::{forward_anthropic, forward_anthropic_streaming};
37use firstpass_core::Route;
38use firstpass_core::trace::ShadowSignal;
39
40#[derive(Clone)]
43pub struct AppState {
44 pub config: Arc<ProxyConfig>,
46 pub http: reqwest::Client,
48 pub providers: ProviderRegistry,
50 pub gate_health: Arc<GateHealthRegistry>,
52 pub shadow_ledger: Arc<crate::shadow::ShadowLedger>,
55 pub guardrails: Arc<crate::guard::GuardrailRegistry>,
58 pub traces: store::TraceSender,
60 pub adaptive: Option<Arc<std::sync::Mutex<firstpass_core::conformal::AdaptiveConformal>>>,
64 pub bandit: Option<Arc<std::sync::Mutex<crate::bandit::StartRungBandit>>>,
69 pub promoter: Option<Arc<crate::affinity::SessionPromoter>>,
74 pub verified_cache: Option<Arc<crate::verified_cache::VerifiedCache>>,
78 pub predictor: Option<Arc<std::sync::Mutex<firstpass_core::PassPredictor>>>,
84 pub tenant_rate_limiter: Option<Arc<governor::DefaultKeyedRateLimiter<String>>>,
88 pub spill: Option<store::SpillHandle>,
92}
93
94#[must_use]
110fn routing_cfg_condense(state: &AppState) -> Option<&firstpass_core::config::CondenseConfig> {
111 state
112 .config
113 .routing
114 .as_ref()
115 .and_then(|r| r.escalation.condense.as_ref())
116}
117
118#[must_use]
123fn mode_now() -> jiff::Timestamp {
125 jiff::Timestamp::now()
126}
127
128fn cache_hit_trace(
136 ctx: &crate::router::EnforceCtx<'_>,
137 entry: &crate::verified_cache::Entry,
138 ts: jiff::Timestamp,
139) -> Trace {
140 Trace {
141 trace_id: uuid::Uuid::now_v7(),
142 prev_hash: firstpass_core::GENESIS_HASH.to_owned(),
143 tenant_id: ctx.tenant_id.clone(),
144 session_id: ctx.session_id.clone(),
145 ts,
146 mode: Mode::Enforce,
147 policy: PolicyRef {
148 id: format!("{}+cache", ctx.policy_id),
149 explore: false,
150 propensity: None,
151 mode_profile: None,
152 },
153 request: RequestInfo {
154 api: ctx.api.clone(),
155 prompt_hash: ctx.prompt_hash.clone(),
156 features: ctx.features.clone(),
157 },
158 attempts: Vec::new(),
159 final_: FinalOutcome {
160 served_rung: None,
161 served_from: ServedFrom::Cache,
162 total_cost_usd: 0.0,
166 gate_cost_usd: 0.0,
167 total_latency_ms: 0,
168 escalations: 0,
169 counterfactual_baseline_usd: entry.original_cost_usd,
170 savings_usd: entry.original_cost_usd,
171 cache_source: Some(entry.source),
172 },
173 deferred: Vec::new(),
174 predicted_pass: None,
175 probe: None,
176 elastic: None,
177 rollout: None,
178 shadow: None,
179 route_ix: None,
180 }
181}
182
183pub fn build_verified_cache(
184 config: &ProxyConfig,
185) -> Option<Arc<crate::verified_cache::VerifiedCache>> {
186 let c = config
187 .routing
188 .as_ref()
189 .and_then(|r| r.escalation.verified_cache.as_ref())?;
190 Some(Arc::new(crate::verified_cache::VerifiedCache::new(
191 std::time::Duration::from_secs(c.ttl_secs),
192 c.max_entries,
193 )))
194}
195
196pub fn build_promoter(
197 config: &ProxyConfig,
198) -> firstpass_core::Result<Option<Arc<crate::affinity::SessionPromoter>>> {
199 match config
200 .routing
201 .as_ref()
202 .and_then(|r| r.escalation.session_promotion.as_ref())
203 {
204 Some(sp) => Ok(Some(Arc::new(crate::affinity::SessionPromoter::new(
205 sp.clone(),
206 )?))),
207 None => Ok(None),
208 }
209}
210
211#[must_use]
212pub fn build_tenant_rate_limiter(
213 config: &ProxyConfig,
214) -> Option<Arc<governor::DefaultKeyedRateLimiter<String>>> {
215 let per_sec = config.tenant_rate_per_sec?;
216 Some(Arc::new(governor::RateLimiter::keyed(
217 governor::Quota::per_second(per_sec),
218 )))
219}
220
221pub async fn tenant_rate_limit_middleware(
225 State(state): State<AppState>,
226 Extension(tenant): Extension<TenantId>,
227 req: Request,
228 next: Next,
229) -> Response {
230 if let Some(limiter) = &state.tenant_rate_limiter
231 && limiter.check_key(&tenant.0).is_err()
232 {
233 return ProxyError::RateLimited.into_response();
234 }
235 next.run(req).await
236}
237
238impl std::fmt::Debug for AppState {
239 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
240 f.debug_struct("AppState")
241 .field("config", &self.config)
242 .finish_non_exhaustive()
243 }
244}
245
246fn offer_trace(traces: &store::TraceSender, spill: Option<&store::SpillHandle>, trace: Trace) {
263 record_trace_metrics(&trace);
264 match traces.try_send(trace) {
265 Ok(()) => {}
266 Err(TrySendError::Full(t)) => {
267 if let Some(handle) = spill {
268 match store::append_to_spill(handle, &t) {
269 Ok(()) => {
270 metrics::counter!("firstpass_receipts_spilled_total").increment(1);
271 }
272 Err(e) => {
273 tracing::error!(%e, "durable mode: spill write failed; trace lost");
274 metrics::counter!("firstpass_traces_dropped_total").increment(1);
275 }
276 }
277 } else {
278 tracing::warn!("trace channel full; dropping trace (writer behind under load)");
279 metrics::counter!("firstpass_traces_dropped_total").increment(1);
280 }
281 }
282 Err(TrySendError::Closed(_)) => {
283 tracing::warn!("trace writer is gone; dropping trace");
284 }
285 }
286}
287
288fn record_trace_metrics(trace: &Trace) {
292 if trace.mode == Mode::Enforce {
293 metrics::histogram!("firstpass_enforce_latency_ms")
294 .record(trace.final_.total_latency_ms as f64);
295 if trace.final_.escalations > 0 {
296 metrics::counter!("firstpass_escalations_total")
297 .increment(u64::from(trace.final_.escalations));
298 }
299 }
300 let served_from = match trace.final_.served_from {
301 ServedFrom::Attempt => "attempt",
302 ServedFrom::BestAttempt => "best_attempt",
303 ServedFrom::Error => "error",
304 ServedFrom::Cache => "cache",
308 };
309 metrics::counter!("firstpass_served_total", "served_from" => served_from).increment(1);
310 if trace.final_.served_from == ServedFrom::Error {
311 metrics::counter!("firstpass_upstream_failures_total").increment(1);
312 }
313 metrics::gauge!("firstpass_cost_usd_total").increment(trace.final_.total_cost_usd);
317 metrics::gauge!("firstpass_gate_cost_usd_total").increment(trace.final_.gate_cost_usd);
318 metrics::gauge!("firstpass_baseline_usd_total")
319 .increment(trace.final_.counterfactual_baseline_usd);
320 metrics::gauge!("firstpass_savings_usd_total").increment(trace.final_.savings_usd);
321 if let Some(rung) = trace.final_.served_rung {
323 let model = trace
324 .attempts
325 .iter()
326 .find(|a| a.rung == rung)
327 .map(|a| a.model.clone())
328 .unwrap_or_else(|| "unknown".to_owned());
329 metrics::counter!(
330 "firstpass_served_rung_total",
331 "rung" => rung.to_string(),
332 "model" => model
333 )
334 .increment(1);
335 }
336}
337
338const MAX_BODY_BYTES: usize = 16 * 1024 * 1024;
342
343pub(crate) fn u01(seed: u128) -> f64 {
354 let lo = splitmix64_finalise(seed as u64);
355 let hi = splitmix64_finalise((seed >> 64) as u64);
356 ((lo ^ hi) >> 11) as f64 * (1.0_f64 / (1u64 << 53) as f64)
358}
359
360fn splitmix64_finalise(mut z: u64) -> u64 {
361 z = z.wrapping_add(0x9E37_79B9_7F4A_7C15);
362 z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
363 z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
364 z ^ (z >> 31)
365}
366
367#[must_use]
374pub(crate) fn epsilon_propensity(chosen: u32, greedy: u32, epsilon: f64, k: usize) -> f64 {
375 let greedy_term = if chosen == greedy { 1.0 - epsilon } else { 0.0 };
376 greedy_term + epsilon / k as f64
377}
378
379pub fn app(state: AppState) -> Result<Router, ProxyError> {
386 crate::metrics::install()?;
387 let max_concurrency = state.config.max_concurrency;
388
389 let business = Router::new()
398 .route("/v1/messages", post(messages))
399 .route("/v1/chat/completions", post(chat_completions))
400 .route("/v1/responses", post(responses))
401 .route("/v1/feedback", post(feedback))
402 .route("/v1/capabilities", get(capabilities))
403 .route("/v1/models", get(models))
404 .route("/v1/receipts", get(receipts))
407 .layer(axum::middleware::from_fn_with_state(
408 state.clone(),
409 tenant_rate_limit_middleware,
410 ))
411 .layer(axum::middleware::from_fn_with_state(
412 state.clone(),
413 auth_middleware,
414 ));
415
416 Ok(Router::new()
417 .merge(business)
418 .route("/healthz", get(healthz))
419 .route("/metrics", get(crate::metrics::handler))
420 .route("/panel", get(panel))
423 .layer(axum::extract::DefaultBodyLimit::max(MAX_BODY_BYTES))
425 .layer(tower::limit::GlobalConcurrencyLimitLayer::new(
428 max_concurrency,
429 ))
430 .with_state(state))
431}
432
433async fn healthz() -> impl IntoResponse {
435 Json(serde_json::json!({
441 "status": "ok",
442 "service": "firstpass",
443 "version": env!("CARGO_PKG_VERSION"),
444 }))
445}
446
447const PANEL_HTML: &str = include_str!("panel.html");
451
452async fn receipts(
458 State(state): State<AppState>,
459 Extension(TenantId(tenant)): Extension<TenantId>,
460 axum::extract::Query(q): axum::extract::Query<ReceiptsQuery>,
461) -> Response {
462 let limit = q.limit.unwrap_or(20).clamp(1, 200);
464 let db = state.config.db_path.clone();
465 let traces =
466 match tokio::task::spawn_blocking(move || crate::store::load_tenant_traces(&db, &tenant))
467 .await
468 {
469 Ok(Ok(t)) => t,
470 _ => Vec::new(),
473 };
474 let items: Vec<serde_json::Value> = traces
475 .iter()
476 .rev()
477 .take(limit)
478 .map(|t| {
479 let f = &t.final_;
480 serde_json::json!({
481 "trace_id": t.trace_id,
482 "served_rung": f.served_rung,
483 "served_model": f.served_rung
488 .and_then(|r| t.attempts.iter().find(|a| a.rung == r))
489 .map(|a| a.model.clone()),
490 "attempts": t.attempts.iter().map(|a| serde_json::json!({
491 "rung": a.rung,
492 "model": a.model,
493 "verdict": a.verdict,
494 "cost_usd": a.cost_usd,
495 "latency_ms": a.latency_ms,
496 })).collect::<Vec<_>>(),
497 "total_cost_usd": f.total_cost_usd,
498 "baseline_usd": f.counterfactual_baseline_usd,
499 "savings_usd": f.savings_usd,
500 })
501 })
502 .collect();
503 axum::Json(serde_json::json!({ "receipts": items })).into_response()
504}
505
506#[derive(Debug, serde::Deserialize)]
508struct ReceiptsQuery {
509 limit: Option<usize>,
511}
512
513async fn panel() -> impl IntoResponse {
519 (
520 [(axum::http::header::CONTENT_TYPE, "text/html; charset=utf-8")],
521 PANEL_HTML,
522 )
523}
524
525async fn capabilities(State(state): State<AppState>) -> impl IntoResponse {
526 let (ladder, gates) = state
529 .config
530 .routing
531 .as_ref()
532 .and_then(|c| c.routes.iter().find(|r| r.mode == Mode::Enforce))
533 .map(|r| (r.ladder.clone(), r.gates.clone()))
534 .unwrap_or_default();
535 let routing_modes: Vec<serde_json::Value> = RoutingMode::ALL
536 .iter()
537 .map(|m| {
538 let p = m.preset();
539 serde_json::json!({
540 "name": m.as_str(),
541 "description": p.description,
542 "tradeoff": p.tradeoff,
543 })
544 })
545 .collect();
546 Json(serde_json::json!({
547 "service": "firstpass",
548 "version": env!("CARGO_PKG_VERSION"),
549 "feature_version": FEATURE_VERSION,
550 "modes": ["observe", "enforce"],
551 "routing_modes": routing_modes,
552 "wire_apis": ["anthropic.messages", "openai.chat_completions", "openai.responses"],
553 "ladder": ladder,
554 "gates": gates,
555 "feedback_api": "POST /v1/feedback",
556 "offboarding": "unset ANTHROPIC_BASE_URL (or OPENAI_BASE_URL for OpenAI clients)",
557 }))
558}
559
560async fn models(State(state): State<AppState>) -> impl IntoResponse {
572 Json(model_list(state.config.routing.as_ref()))
573}
574
575fn model_list(routing: Option<&firstpass_core::config::Config>) -> serde_json::Value {
578 let prices = routing.map(|c| c.price_table()).unwrap_or_default();
579 let mut seen = std::collections::HashSet::new();
582 let data: Vec<serde_json::Value> = routing
583 .map(|c| c.routes.as_slice())
584 .unwrap_or_default()
585 .iter()
586 .flat_map(|r| r.ladder.iter())
587 .filter(|id| seen.insert(id.as_str()))
588 .map(|id| {
589 let price = prices.get(id);
590 serde_json::json!({
591 "id": id,
592 "object": "model",
593 "created": 0,
596 "owned_by": id.split_once('/').map_or("firstpass", |(p, _)| p),
597 "firstpass": {
598 "input_per_mtok": price.map(|p| p.input_per_mtok),
599 "output_per_mtok": price.map(|p| p.output_per_mtok),
600 },
601 })
602 })
603 .collect();
604 serde_json::json!({ "object": "list", "data": data })
605}
606
607#[derive(Debug, Deserialize)]
609struct FeedbackRequest {
610 trace_id: String,
612 gate_id: String,
614 verdict: String,
616 #[serde(default)]
618 score: Option<f64>,
619 reporter: String,
621}
622
623async fn feedback(
628 State(state): State<AppState>,
629 Extension(TenantId(tenant)): Extension<TenantId>,
630 body: Bytes,
631) -> Response {
632 let req: FeedbackRequest = match serde_json::from_slice(&body) {
633 Ok(r) => r,
634 Err(e) => {
635 return ProxyError::BadRequest(format!("invalid feedback body: {e}")).into_response();
636 }
637 };
638 let verdict = match req.verdict.as_str() {
639 "pass" => Verdict::Pass,
640 "fail" => Verdict::Fail,
641 "abstain" => Verdict::Abstain,
642 other => {
643 return ProxyError::BadRequest(format!("unknown verdict {other:?}")).into_response();
644 }
645 };
646 let score = match req.score {
647 Some(s) => match Score::new(s) {
648 Ok(sc) => Some(sc),
649 Err(_) => {
650 return ProxyError::BadRequest(format!("score {s} out of range [0,1]"))
651 .into_response();
652 }
653 },
654 None => None,
655 };
656
657 let db = state.config.db_path.clone();
658
659 let (db_check, tenant_check, tid_check) = (db.clone(), tenant.clone(), req.trace_id.clone());
664 match tokio::task::spawn_blocking(move || {
665 store::trace_exists(&db_check, &tenant_check, &tid_check)
666 })
667 .await
668 {
669 Ok(Ok(true)) => {}
670 Ok(Ok(false)) => {
671 return ProxyError::NotFound(format!("unknown trace_id {:?}", req.trace_id))
672 .into_response();
673 }
674 Ok(Err(e)) => {
675 tracing::error!(%e, "feedback: trace_exists check failed");
676 return ProxyError::Internal(e.to_string()).into_response();
677 }
678 Err(e) => {
679 tracing::error!(%e, "feedback: trace_exists task panicked");
680 return ProxyError::Internal(e.to_string()).into_response();
681 }
682 }
683
684 if verdict == Verdict::Fail
689 && let Some(cache) = state.verified_cache.as_ref()
690 {
691 let reported = req.trace_id.parse::<uuid::Uuid>().ok();
696 let origin = {
697 let (d, t, i) = (db.clone(), tenant.clone(), req.trace_id.clone());
698 tokio::task::spawn_blocking(move || store::trace_cache_source(&d, &t, &i))
699 .await
700 .ok()
701 .and_then(Result::ok)
702 .flatten()
703 .flatten()
704 };
705 let dropped: usize = reported
706 .into_iter()
707 .chain(origin)
708 .map(|id| cache.retract(id))
709 .sum();
710 if dropped > 0 {
711 tracing::info!(
712 trace = %req.trace_id,
713 ?origin,
714 dropped,
715 "verified cache: retracted a disproven answer"
716 );
717 metrics::counter!("firstpass_cache_retractions_total").increment(dropped as u64);
718 }
719 }
720
721 let feedback_signal = match verdict {
723 Verdict::Pass => Some(true),
724 Verdict::Fail => Some(false),
725 Verdict::Abstain => None,
726 };
727 let dv = DeferredVerdict {
728 gate_id: req.gate_id,
729 verdict,
730 score,
731 reported_at: jiff::Timestamp::now(),
732 reporter: req.reporter,
733 };
734 let trace_id = req.trace_id.clone();
735 match tokio::task::spawn_blocking(move || store::append_deferred(&db, &req.trace_id, &dv)).await
736 {
737 Ok(Ok(())) => {
738 if let (Some(a), Some(correct)) = (state.adaptive.as_ref(), feedback_signal)
740 && let Ok(mut g) = a.lock()
741 {
742 g.observe_served(correct);
743 metrics::gauge!("firstpass_serve_threshold").set(g.threshold());
744 metrics::gauge!("firstpass_realized_served_failure")
745 .set(g.realized_served_failure());
746 }
747
748 let attributed_route = tokio::task::spawn_blocking({
753 let (db, ten, tid) = (
754 state.config.db_path.clone(),
755 tenant.clone(),
756 trace_id.clone(),
757 );
758 move || store::trace_route(&db, &ten, &tid)
759 })
760 .await
761 .ok()
762 .and_then(Result::ok)
763 .flatten()
764 .flatten()
765 .unwrap_or(0) as usize;
766
767 if let (Some(cfg), Some(correct)) = (
772 state.config.routing.as_ref().and_then(|r| r.guardrail),
773 feedback_signal,
774 ) {
775 let cooldown = state
776 .config
777 .routing
778 .as_ref()
779 .map_or(3_600, |r| r.guardrail_cooldown_secs);
780 let reaction = state.guardrails.record(
781 &tenant,
782 attributed_route,
783 &cfg,
784 correct,
785 jiff::Timestamp::now().as_second(),
786 cooldown,
787 );
788 match &reaction {
789 crate::guard::Reaction::Demoted(v) => {
790 metrics::counter!("firstpass_guardrail_demotions_total").increment(1);
791 tracing::error!(
792 tenant = %tenant,
793 n = v.n,
794 rate = v.rate,
795 bound = v.bound,
796 alpha = cfg.alpha,
797 "GUARDRAIL: served-failure bound exceeded target — route demoted to observe; traffic now serves as it would without Firstpass"
798 );
799 }
800 crate::guard::Reaction::Alarmed(v) => {
801 metrics::counter!("firstpass_guardrail_breaches_total").increment(1);
802 tracing::error!(
803 tenant = %tenant,
804 n = v.n,
805 rate = v.rate,
806 bound = v.bound,
807 alpha = cfg.alpha,
808 "GUARDRAIL: served-failure bound exceeded target (alarm only — routing unchanged)"
809 );
810 }
811 crate::guard::Reaction::None => {}
812 }
813 }
814 (
815 axum::http::StatusCode::ACCEPTED,
816 Json(serde_json::json!({ "status": "recorded", "trace_id": trace_id })),
817 )
818 .into_response()
819 }
820 Ok(Err(e)) => {
821 tracing::error!(%e, "feedback: append_deferred failed");
822 ProxyError::Internal(e.to_string()).into_response()
823 }
824 Err(e) => {
825 tracing::error!(%e, "feedback: append_deferred task panicked");
826 ProxyError::Internal(e.to_string()).into_response()
827 }
828 }
829}
830
831const SESSION_HEADER: &str = "x-firstpass-session";
834
835const AGENT_HEADER: &str = "x-firstpass-agent";
837const SUBAGENT_HEADER: &str = "x-firstpass-subagent";
839#[expect(
845 clippy::too_many_arguments,
846 reason = "mirrors the enforce context it evaluates; a wrapper struct would only move the list"
847)]
848fn spawn_shadow(
849 state: &AppState,
850 route: &firstpass_core::Route,
851 route_ix: usize,
852 body: Bytes,
853 auth: Auth,
854 features: Features,
855 tenant: String,
856 session_id: String,
857 api: &str,
858) {
859 let Some(shadow) = route.shadow else {
860 return;
861 };
862 if !shadow.sampled(&state.config.prompt_salt, &session_id) {
865 return;
866 }
867 let (state, route, api) = (state.clone(), route.clone(), api.to_owned());
868 tokio::spawn(async move {
869 let signal = evaluate_shadow(
870 &state, &route, shadow, route_ix, &body, auth, &features, tenant, session_id, &api, 0.0,
871 )
872 .await;
873 tracing::debug!(
875 would_pass = signal.would_pass,
876 projected_usd = signal.projected_cost_usd,
877 skipped = ?signal.skipped,
878 "shadow evaluation complete"
879 );
880 });
881}
882
883#[expect(
890 clippy::too_many_arguments,
891 reason = "mirrors the enforce context; grouping them into a struct would only move the list"
892)]
893async fn evaluate_shadow(
894 state: &AppState,
895 route: &firstpass_core::Route,
896 shadow: firstpass_core::rollout::Shadow,
897 route_ix: usize,
898 body: &Bytes,
899 auth: Auth,
900 features: &Features,
901 tenant: String,
902 session_id: String,
903 api: &str,
904 actual_cost_usd: f64,
905) -> ShadowSignal {
906 let skipped = |why: &str| ShadowSignal {
907 would_serve_rung: None,
908 would_pass: false,
909 projected_cost_usd: 0.0,
910 actual_cost_usd,
911 skipped: Some(why.to_owned()),
912 };
913
914 let now = jiff::Timestamp::now();
915 if !state
916 .shadow_ledger
917 .may_spend(&tenant, route_ix, shadow.max_usd_per_day, now)
918 {
919 return skipped("budget_exhausted");
922 }
923
924 let Some(base_request) = parse_model_request(body) else {
925 return skipped("unparseable_request");
926 };
927 let gate_defs = state
928 .config
929 .routing
930 .as_ref()
931 .map_or(&[][..], |cfg| &cfg.gate_defs);
932 let gates = resolve_gates(
933 &route.gates,
934 gate_defs,
935 &state.providers,
936 &auth,
937 &state.config.prices,
938 );
939 let (budget, max_rungs) = state
940 .config
941 .routing
942 .as_ref()
943 .map_or((None, u32::MAX), |cfg| {
944 (
945 cfg.budget.per_request_usd,
946 cfg.escalation.max_rungs_per_request,
947 )
948 });
949
950 let mut base_request = base_request;
951 base_request.cache_prefix = state
955 .config
956 .routing
957 .as_ref()
958 .is_some_and(|r| r.escalation.prompt_cache);
959 let ctx = EnforceCtx {
960 condense: routing_cfg_condense(state),
961 ladder: &route.ladder,
962 gates: &gates,
963 health: &state.gate_health,
964 base_request: &base_request,
965 providers: &state.providers,
966 auth: &auth,
967 prices: &state.config.prices,
968 budget_per_request_usd: budget,
969 max_rungs,
970 speculation: 0,
973 serve_threshold: None,
974 elastic: None,
975 features: features.clone(),
976 start_rung: 0,
977 tenant_id: tenant.clone(),
978 session_id,
979 prompt_hash: prompt_hash(&state.config.prompt_salt, body),
980 api: api.to_owned(),
981 policy_id: "shadow".to_owned(),
982 };
983
984 let (outcome, trace) = route_enforce(ctx).await;
985 let spent = trace.final_.total_cost_usd;
986 state.shadow_ledger.debit(&tenant, route_ix, spent, now);
987
988 let would_pass = matches!(outcome, EngineOutcome::Served(_));
991 let would_serve_rung = if would_pass {
992 trace.final_.served_rung
993 } else {
994 None
995 };
996 ShadowSignal {
997 would_serve_rung,
998 would_pass,
999 projected_cost_usd: spent,
1000 actual_cost_usd,
1001 skipped: None,
1002 }
1003}
1004
1005const MODE_PROFILE_HEADER: &str = "x-firstpass-mode";
1008
1009fn resolve_mode(headers: &HeaderMap, route: &Route, config: &ProxyConfig) -> RoutingMode {
1018 if let Some(val) = header_str(headers, MODE_PROFILE_HEADER) {
1020 match val.trim().to_ascii_lowercase().as_str() {
1021 "observe" => return RoutingMode::Observe,
1022 "cost" => return RoutingMode::Cost,
1023 "balanced" => return RoutingMode::Balanced,
1024 "quality" => return RoutingMode::Quality,
1025 "latency" => return RoutingMode::Latency,
1026 "max" => return RoutingMode::Max,
1027 other => {
1028 tracing::warn!(
1029 value = other,
1030 "unknown x-firstpass-mode value; ignoring \
1031 (valid: observe|cost|balanced|quality|latency|max)"
1032 );
1033 }
1034 }
1035 }
1036 if let Some(m) = route.routing_mode {
1038 return m;
1039 }
1040 config.default_routing_mode
1042}
1043
1044async fn messages(
1049 State(state): State<AppState>,
1050 Extension(TenantId(tenant)): Extension<TenantId>,
1051 headers: HeaderMap,
1052 body: Bytes,
1053) -> Response {
1054 let session_header = header_str(&headers, SESSION_HEADER);
1055
1056 if let Some(routing) = state.config.routing.as_ref() {
1059 let features = extract_features(&headers, &body);
1060 if let Some(route) = routing
1061 .route_for(&features)
1062 .filter(|r| r.mode == Mode::Enforce && !r.ladder.is_empty())
1063 {
1064 let route_ix = routing
1067 .routes
1068 .iter()
1069 .position(|r| std::ptr::eq(r, route))
1070 .unwrap_or(0);
1071 let route = route.clone();
1074 let routing_mode = resolve_mode(&headers, &route, &state.config);
1076 if routing_mode == RoutingMode::Observe {
1078 return observe_passthrough(state, headers, body, session_header, tenant).await;
1079 }
1080 if state
1084 .guardrails
1085 .is_demoted(&tenant, route_ix, jiff::Timestamp::now().as_second())
1086 {
1087 return observe_passthrough(state, headers, body, session_header, tenant).await;
1088 }
1089 if let Some(rollout) = route.rollout.as_ref() {
1094 let key_value = match rollout.key {
1095 firstpass_core::RolloutKey::Session => session_header
1096 .clone()
1097 .unwrap_or_else(|| firstpass_core::rollout::request_identity(&body)),
1098 firstpass_core::RolloutKey::Request => {
1104 firstpass_core::rollout::request_identity(&body)
1105 }
1106 firstpass_core::RolloutKey::Tenant => tenant.to_string(),
1107 };
1108 let decision =
1109 firstpass_core::rollout::decide(&state.config.prompt_salt, rollout, &key_value);
1110 if !decision.enforced {
1111 let shadow_ctx = route.shadow.map(|_| {
1113 (
1114 body.clone(),
1115 Auth::from_headers(&headers),
1116 features.clone(),
1117 tenant.clone(),
1118 session_header
1119 .clone()
1120 .unwrap_or_else(|| Uuid::now_v7().to_string()),
1121 )
1122 });
1123 let resp =
1124 observe_passthrough(state.clone(), headers, body, session_header, tenant)
1125 .await;
1126 if let Some((b, auth, feats, ten, sess)) = shadow_ctx {
1129 spawn_shadow(
1130 &state,
1131 &route,
1132 route_ix,
1133 b,
1134 auth,
1135 feats,
1136 ten,
1137 sess,
1138 "anthropic",
1139 );
1140 }
1141 return resp;
1142 }
1143 }
1144 if enforce_can_handle(
1145 &features,
1146 &body,
1147 routing.escalation.enforce_structured,
1148 &route.ladder,
1149 &state.providers,
1150 Dialect::Anthropic,
1151 ) {
1152 return handle_enforce(
1153 &state,
1154 &headers,
1155 &body,
1156 features,
1157 &route,
1158 route_ix,
1159 session_header,
1160 tenant,
1161 routing_mode,
1162 )
1163 .await;
1164 }
1165 tracing::info!(
1169 "enforce route matched but structured request can't be routed faithfully (flag/ladder); serving via observe passthrough"
1170 );
1171 }
1172 }
1173 observe_passthrough(state, headers, body, session_header, tenant).await
1174}
1175
1176fn enforce_can_handle(
1190 features: &Features,
1191 body: &[u8],
1192 enforce_structured: bool,
1193 ladder: &[String],
1194 providers: &crate::provider::ProviderRegistry,
1195 inbound: Dialect,
1196) -> bool {
1197 let structured = features.tool_count > 0
1198 || features.has_images
1199 || match inbound {
1200 Dialect::Anthropic => messages_have_tool_blocks(body),
1201 Dialect::Openai => openai_messages_have_tool_calls(body),
1202 Dialect::Gemini => false,
1203 };
1204 if !structured {
1205 return true;
1206 }
1207 if !enforce_structured {
1208 return false;
1209 }
1210 let all_verbatim = ladder.iter().all(|rung| {
1212 let provider_id = rung.split('/').next().unwrap_or_default();
1213 providers
1214 .get(provider_id)
1215 .is_some_and(|p| p.carries_structured_verbatim(inbound))
1216 });
1217 if all_verbatim {
1218 return true;
1219 }
1220 if inbound == Dialect::Openai && !openai_has_http_images(body) {
1224 let all_anthropic = ladder.iter().all(|rung| {
1225 let pid = rung.split('/').next().unwrap_or_default();
1226 providers
1227 .get(pid)
1228 .is_some_and(|p| p.carries_structured_verbatim(Dialect::Anthropic))
1229 });
1230 if all_anthropic {
1231 return true;
1232 }
1233 }
1234 false
1235}
1236
1237fn messages_have_tool_blocks(body: &[u8]) -> bool {
1240 serde_json::from_slice::<Value>(body)
1241 .ok()
1242 .and_then(|json| {
1243 json.get("messages")
1244 .and_then(Value::as_array)
1245 .map(|messages| messages.iter().any(message_has_tool_block))
1246 })
1247 .unwrap_or(false)
1248}
1249
1250fn message_has_tool_block(message: &Value) -> bool {
1252 message
1253 .get("content")
1254 .and_then(Value::as_array)
1255 .is_some_and(|blocks| {
1256 blocks.iter().any(|block| {
1257 matches!(
1258 block.get("type").and_then(Value::as_str),
1259 Some("tool_use" | "tool_result")
1260 )
1261 })
1262 })
1263}
1264
1265fn is_stream_request(body: &[u8]) -> bool {
1267 serde_json::from_slice::<Value>(body)
1268 .ok()
1269 .and_then(|json| json.get("stream").and_then(Value::as_bool))
1270 .unwrap_or(false)
1271}
1272
1273fn header_str(headers: &HeaderMap, name: &str) -> Option<String> {
1275 headers
1276 .get(name)
1277 .and_then(|v| v.to_str().ok())
1278 .map(str::to_owned)
1279}
1280
1281fn extract_features(headers: &HeaderMap, body: &[u8]) -> Features {
1284 let (_model, tool_count, has_images) = request_features(body);
1285 let mut f = Features::new(TaskKind::Other);
1286 f.agent = header_str(headers, AGENT_HEADER);
1287 f.subagent = header_str(headers, SUBAGENT_HEADER);
1288 f.tool_count = tool_count;
1289 f.has_images = has_images;
1290 f.prompt_token_bucket = token_bucket(body.len() as u64);
1293 f.hour_bucket = hour_bucket(jiff::Timestamp::now());
1294 f
1295}
1296
1297#[allow(clippy::too_many_arguments)]
1300async fn handle_enforce(
1301 state: &AppState,
1302 headers: &HeaderMap,
1303 body: &Bytes,
1304 features: Features,
1305 route: &Route,
1306 route_ix: usize,
1309 session_header: Option<String>,
1310 tenant: String,
1311 routing_mode: RoutingMode,
1312) -> Response {
1313 if is_stream_request(body) {
1321 let (tx, rx) = tokio::sync::oneshot::channel::<Result<Value, ProxyError>>();
1322 let (state_c, headers_c, body_c, route_c) =
1323 (state.clone(), headers.clone(), body.clone(), route.clone());
1324 tokio::spawn(async move {
1325 let out = enforce_pipeline(
1326 &state_c,
1327 &headers_c,
1328 &body_c,
1329 features,
1330 &route_c,
1331 route_ix,
1332 session_header,
1333 tenant,
1334 routing_mode,
1335 )
1336 .await;
1337 let _ = tx.send(out);
1338 });
1339 return sse_keepalive_response(rx, anthropic_sse_from_message);
1340 }
1341 match enforce_pipeline(
1342 state,
1343 headers,
1344 body,
1345 features,
1346 route,
1347 route_ix,
1348 session_header,
1349 tenant,
1350 routing_mode,
1351 )
1352 .await
1353 {
1354 Ok(message) => (axum::http::StatusCode::OK, Json(message)).into_response(),
1355 Err(e) => e.into_response(),
1356 }
1357}
1358
1359#[allow(clippy::too_many_arguments)] async fn enforce_pipeline_inner(
1364 state: &AppState,
1365 body: &Bytes,
1366 base_request: ModelRequest,
1367 auth: Auth,
1368 features: Features,
1369 route: &Route,
1370 session_header: Option<String>,
1371 tenant: String,
1372 api: &str,
1373 routing_mode: RoutingMode,
1374 route_ix: usize,
1375) -> Result<ModelResponse, ProxyError> {
1376 let gate_defs = state
1377 .config
1378 .routing
1379 .as_ref()
1380 .map_or(&[][..], |cfg| &cfg.gate_defs);
1381 let gates = resolve_gates(
1382 &route.gates,
1383 gate_defs,
1384 &state.providers,
1385 &auth,
1386 &state.config.prices,
1387 );
1388 let session_id = session_header.unwrap_or_else(|| Uuid::now_v7().to_string());
1389 let (budget, max_rungs, speculation, serve_threshold, elastic) =
1390 match state.config.routing.as_ref() {
1391 Some(cfg) => (
1392 cfg.budget.per_request_usd,
1393 cfg.escalation.max_rungs_per_request,
1394 cfg.escalation.speculation,
1395 cfg.escalation.serve_threshold,
1396 cfg.escalation.elastic.as_ref(),
1397 ),
1398 None => (None, 3, 0, None, None),
1399 };
1400 let serve_threshold = state
1403 .adaptive
1404 .as_ref()
1405 .and_then(|a| a.lock().ok().map(|g| g.threshold()))
1406 .or(serve_threshold);
1407
1408 let preset = routing_mode.preset();
1411 let max_rungs = if let Some(delta) = preset.max_rungs_delta {
1412 (max_rungs as i32 + delta).max(1) as u32
1413 } else {
1414 max_rungs
1415 };
1416 let speculation = preset.speculation.unwrap_or(speculation);
1417
1418 let bandit_ctx = crate::bandit::ContextBucket::from_features(&features);
1422
1423 let (greedy_rung, base_policy_id, ts_propensity) = {
1427 let (chosen, ts_p) = state
1428 .bandit
1429 .as_ref()
1430 .and_then(|b| b.lock().ok())
1431 .map(|mut b| {
1432 b.choose_start_with_propensity(&bandit_ctx, &route.ladder, &state.config.prices)
1433 })
1434 .unwrap_or((0, None));
1435 let policy = if ts_p.is_some() {
1436 "bandit@v2-ts".to_owned()
1437 } else if chosen > 0 {
1438 "bandit@v1".to_owned()
1439 } else {
1440 "static-ladder@v0".to_owned()
1441 };
1442 (chosen, policy, ts_p)
1443 };
1444
1445 let exploration_epsilon = state
1450 .config
1451 .routing
1452 .as_ref()
1453 .and_then(|cfg| cfg.escalation.exploration.as_ref())
1454 .map(|e| e.epsilon);
1455
1456 let (start_rung, policy_id, explore_flag, propensity) = if ts_propensity.is_some() {
1457 if exploration_epsilon.is_some() {
1460 tracing::warn!(
1461 "bandit.algorithm = thompson already logs propensities; \
1462 [escalation.exploration] epsilon is ignored"
1463 );
1464 }
1465 (greedy_rung, base_policy_id, false, ts_propensity)
1466 } else if let Some(epsilon) = exploration_epsilon {
1467 let k = route.ladder.len().max(1);
1468 let u = u01(Uuid::now_v7().as_u128());
1470 let (chosen, eps_branch) = if u < epsilon {
1471 let idx = ((u / epsilon) * k as f64) as u32;
1473 (idx.min(k as u32 - 1), true)
1474 } else {
1475 (greedy_rung, false)
1476 };
1477 let p = epsilon_propensity(chosen, greedy_rung, epsilon, k);
1478 (chosen, format!("{base_policy_id}+eps"), eps_branch, Some(p))
1479 } else {
1480 (greedy_rung, base_policy_id, false, None)
1481 };
1482
1483 let promotion = state
1488 .promoter
1489 .as_ref()
1490 .map(|p| p.decide(&tenant, &session_id, std::time::Instant::now()))
1491 .unwrap_or(crate::affinity::Decision::Cold);
1492 let start_rung = start_rung.max(promotion.start_rung());
1493 let promo_key = state
1496 .promoter
1497 .as_ref()
1498 .map(|_| (tenant.clone(), session_id.clone()));
1499
1500 let start_rung = if preset.start_at_top {
1503 route.ladder.len().saturating_sub(1) as u32
1504 } else {
1505 start_rung
1506 };
1507
1508 let speculation = match state
1514 .config
1515 .routing
1516 .as_ref()
1517 .and_then(|cfg| cfg.escalation.speculation_band)
1518 {
1519 Some([lo, hi]) if speculation > 0 => {
1520 let estimate = state
1521 .bandit
1522 .as_ref()
1523 .and_then(|b| b.lock().ok())
1524 .and_then(|b| b.pass_estimate(&bandit_ctx, start_rung));
1525 match estimate {
1526 Some(p) if p < lo || p > hi => {
1527 metrics::counter!("firstpass_speculation_skipped_total").increment(1);
1528 0
1529 }
1530 _ => speculation,
1531 }
1532 }
1533 _ => speculation,
1534 };
1535
1536 if state.bandit.is_some() {
1538 metrics::counter!(
1539 "firstpass_bandit_start_rung",
1540 "rung" => start_rung.to_string()
1541 )
1542 .increment(1);
1543 }
1544
1545 let mut base_request = base_request;
1546 base_request.cache_prefix = state
1550 .config
1551 .routing
1552 .as_ref()
1553 .is_some_and(|r| r.escalation.prompt_cache);
1554 let ctx = EnforceCtx {
1555 condense: routing_cfg_condense(state),
1556 ladder: &route.ladder,
1557 gates: &gates,
1558 health: &state.gate_health,
1559 base_request: &base_request,
1560 providers: &state.providers,
1561 auth: &auth,
1562 prices: &state.config.prices,
1563 budget_per_request_usd: budget,
1564 max_rungs,
1565 speculation,
1566 serve_threshold,
1567 elastic,
1568 features,
1569 start_rung,
1570 tenant_id: tenant,
1573 session_id,
1574 prompt_hash: prompt_hash(&state.config.prompt_salt, body),
1575 api: api.to_owned(),
1576 policy_id,
1577 };
1578
1579 if let Some(cache) = state.verified_cache.as_ref()
1584 && let Some(entry) = cache.get(
1585 &ctx.tenant_id,
1586 &ctx.prompt_hash,
1587 &route.ladder,
1588 std::time::Instant::now(),
1589 )
1590 {
1591 metrics::counter!("firstpass_cache_hits_total").increment(1);
1592 let trace = cache_hit_trace(&ctx, &entry, mode_now());
1593 offer_trace(&state.traces, state.spill.as_ref(), trace);
1594 return Ok(entry.response);
1595 }
1596
1597 let (outcome, mut trace) = route_enforce(ctx).await;
1598
1599 trace.route_ix = Some(u32::try_from(route_ix).unwrap_or(0));
1603 trace.policy.explore = explore_flag;
1606 trace.policy.propensity = propensity;
1607 if routing_mode != RoutingMode::Balanced {
1610 trace.policy.mode_profile = Some(routing_mode.as_str().to_owned());
1611 }
1612
1613 if let Some(bandit) = state.bandit.as_ref()
1617 && let Ok(mut b) = bandit.lock()
1618 {
1619 for attempt in &trace.attempts {
1620 b.observe(&bandit_ctx, attempt.rung, attempt.verdict);
1621 }
1622 }
1623
1624 if let Some(promoter) = state.promoter.as_ref()
1629 && let Some((tenant_key, session_key)) = promo_key.as_ref()
1630 && let Some(served) = trace.final_.served_rung
1631 {
1632 promoter.record(
1633 tenant_key,
1634 session_key,
1635 served,
1636 served > start_rung,
1637 std::time::Instant::now(),
1638 );
1639 }
1640
1641 if let Some(predictor) = state.predictor.as_ref()
1647 && let Ok(mut p) = predictor.lock()
1648 {
1649 let predicted = p.predict(&trace.request.features, start_rung);
1651 for attempt in &trace.attempts {
1652 match attempt.verdict {
1653 Verdict::Pass => p.update(&trace.request.features, attempt.rung, true),
1654 Verdict::Fail => p.update(&trace.request.features, attempt.rung, false),
1655 Verdict::Abstain => {} }
1657 }
1658 trace.predicted_pass = Some(predicted);
1659 metrics::histogram!("firstpass_predictor_pass_prob").record(predicted);
1660 }
1661
1662 if let Some(probe_cfg) = state
1669 .config
1670 .routing
1671 .as_ref()
1672 .and_then(|c| c.escalation.probe)
1673 && u01(Uuid::now_v7().as_u128()) < probe_cfg.sample_rate
1674 {
1675 let probe_rung = (start_rung as usize).min(route.ladder.len().saturating_sub(1));
1677 if let Some(probe_model_str) = route.ladder.get(probe_rung).cloned() {
1678 let probe_provider = ModelRef::parse(&probe_model_str)
1679 .ok()
1680 .and_then(|m| state.providers.get(&m.provider));
1681
1682 if let Some(probe_provider) = probe_provider {
1683 let mut join_set = tokio::task::JoinSet::new();
1686 for _ in 0..probe_cfg.k {
1687 let mut probe_req = base_request.clone();
1688 probe_req.model = probe_model_str.clone();
1689 let probe_auth = auth.clone();
1690 let prov = probe_provider.clone();
1691 join_set.spawn(async move { prov.complete(&probe_req, &probe_auth).await });
1692 }
1693
1694 let fail_closed_owned: std::collections::HashSet<String> = gates
1697 .iter()
1698 .filter(|g| g.abstain_fails_closed())
1699 .map(|g| g.id().to_owned())
1700 .collect();
1701
1702 let mut gate_pass_count = 0u32;
1703 let mut probe_cost_usd = 0.0f64;
1704
1705 while let Some(task_result) = join_set.join_next().await {
1706 let Ok(Ok(probe_resp)) = task_result else {
1707 continue; };
1709 probe_cost_usd += state
1712 .config
1713 .prices
1714 .cost_usd_with_cache(
1715 &probe_model_str,
1716 probe_resp.in_tokens,
1717 probe_resp.cache_write_tokens,
1718 probe_resp.cache_read_tokens,
1719 probe_resp.out_tokens,
1720 )
1721 .unwrap_or(0.0);
1722
1723 let mut probe_gate_req = base_request.clone();
1724 probe_gate_req.model = probe_model_str.clone();
1725
1726 let mut probe_gate_results = Vec::with_capacity(gates.len());
1729 for g in &gates {
1730 if !state.gate_health.enabled(&trace.tenant_id, g.id()) {
1732 continue;
1733 }
1734 let r = g.evaluate(&probe_gate_req, &probe_resp).await;
1735 probe_gate_results.push(r);
1737 }
1738
1739 let fail_closed_refs: std::collections::HashSet<&str> =
1740 fail_closed_owned.iter().map(|s| s.as_str()).collect();
1741 let verdict = aggregate_with_policy(&probe_gate_results, &fail_closed_refs);
1742 let passes = match serve_threshold {
1745 None => verdict == Verdict::Pass,
1746 Some(t) => crate::calibrate::gate_score(&probe_gate_results, verdict) >= t,
1747 };
1748 if passes {
1749 gate_pass_count += 1;
1750 }
1751 }
1752
1753 let regime = ProbeRegime::classify(gate_pass_count, probe_cfg.k);
1754 let regime_label = match regime {
1755 ProbeRegime::ConfidentPass => "confident_pass",
1756 ProbeRegime::ConfidentFail => "confident_fail",
1757 ProbeRegime::Ambiguous => "ambiguous",
1758 };
1759 metrics::counter!(
1760 "firstpass_probe_regime_total",
1761 "regime" => regime_label
1762 )
1763 .increment(1);
1764 metrics::gauge!("firstpass_probe_cost_usd_total").increment(probe_cost_usd);
1765 trace.probe = Some(ProbeSignal {
1766 k: probe_cfg.k,
1767 gate_pass_count,
1768 regime,
1769 probe_cost_usd,
1770 });
1771 }
1772 }
1773 }
1774 if let Some(cache) = state.verified_cache.as_ref()
1780 && let EngineOutcome::Served(resp) = &outcome
1781 && cache.offer(&trace, resp, &route.ladder, std::time::Instant::now())
1782 {
1783 metrics::counter!("firstpass_cache_stores_total").increment(1);
1784 }
1785
1786 offer_trace(&state.traces, state.spill.as_ref(), trace);
1788
1789 match outcome {
1790 EngineOutcome::Served(resp) => Ok(resp),
1791 EngineOutcome::Failed(msg) => Err(ProxyError::Engine(msg)),
1792 }
1793}
1794
1795#[allow(clippy::too_many_arguments)]
1798async fn enforce_pipeline(
1799 state: &AppState,
1800 headers: &HeaderMap,
1801 body: &Bytes,
1802 features: Features,
1803 route: &Route,
1804 route_ix: usize,
1806 session_header: Option<String>,
1807 tenant: String,
1808 routing_mode: RoutingMode,
1809) -> Result<Value, ProxyError> {
1810 let Some(base_request) = parse_model_request(body) else {
1811 return Err(ProxyError::BadRequest(
1812 "request body is not a valid Anthropic Messages request".to_owned(),
1813 ));
1814 };
1815 let auth = Auth::from_headers(headers);
1816 let resp = enforce_pipeline_inner(
1817 state,
1818 body,
1819 base_request,
1820 auth,
1821 features,
1822 route,
1823 session_header,
1824 tenant,
1825 "anthropic.messages",
1826 routing_mode,
1827 route_ix,
1828 )
1829 .await?;
1830 Ok(anthropic_response_json(&resp))
1831}
1832
1833#[allow(clippy::too_many_arguments)]
1836async fn enforce_pipeline_openai(
1837 state: &AppState,
1838 headers: &HeaderMap,
1839 body: &Bytes,
1840 features: Features,
1841 route: &Route,
1842 route_ix: usize,
1844 session_header: Option<String>,
1845 tenant: String,
1846 routing_mode: RoutingMode,
1847) -> Result<Value, ProxyError> {
1848 enforce_pipeline_openai_as(
1849 state,
1850 headers,
1851 body,
1852 body,
1853 "openai.chat_completions",
1854 features,
1855 route,
1856 route_ix,
1857 session_header,
1858 tenant,
1859 routing_mode,
1860 )
1861 .await
1862}
1863
1864#[allow(clippy::too_many_arguments)]
1873async fn enforce_pipeline_openai_as(
1874 state: &AppState,
1875 headers: &HeaderMap,
1876 body: &Bytes,
1877 receipt_body: &Bytes,
1878 api: &str,
1879 features: Features,
1880 route: &Route,
1881 route_ix: usize,
1882 session_header: Option<String>,
1883 tenant: String,
1884 routing_mode: RoutingMode,
1885) -> Result<Value, ProxyError> {
1886 let providers = &state.providers;
1888 let all_openai = route.ladder.iter().all(|rung| {
1889 let pid = rung.split('/').next().unwrap_or_default();
1890 providers
1891 .get(pid)
1892 .is_some_and(|p| p.carries_structured_verbatim(Dialect::Openai))
1893 });
1894 let Some(base_request) = parse_openai_request(body, all_openai) else {
1895 return Err(ProxyError::BadRequest(
1896 "request body is not a valid OpenAI Chat Completions request".to_owned(),
1897 ));
1898 };
1899 let auth = Auth::from_headers(headers);
1900 let resp = enforce_pipeline_inner(
1901 state,
1902 receipt_body,
1903 base_request,
1904 auth,
1905 features,
1906 route,
1907 session_header,
1908 tenant,
1909 api,
1910 routing_mode,
1911 route_ix,
1912 )
1913 .await?;
1914 Ok(openai_response_json(&resp))
1915}
1916
1917const SSE_KEEPALIVE_EVERY: Duration = Duration::from_secs(5);
1919
1920fn sse_keepalive_response(
1929 rx: tokio::sync::oneshot::Receiver<Result<Value, ProxyError>>,
1930 format_message: fn(&Value) -> String,
1931) -> Response {
1932 let mut ticks = tokio::time::interval(SSE_KEEPALIVE_EVERY);
1933 ticks.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
1934 ticks.reset(); let stream = KeepaliveStream {
1936 rx: Some(rx),
1937 ticks,
1938 format_message,
1939 };
1940 (
1941 axum::http::StatusCode::OK,
1942 [(
1943 axum::http::header::CONTENT_TYPE,
1944 "text/event-stream; charset=utf-8",
1945 )],
1946 axum::body::Body::from_stream(stream),
1947 )
1948 .into_response()
1949}
1950
1951struct KeepaliveStream {
1954 rx: Option<tokio::sync::oneshot::Receiver<Result<Value, ProxyError>>>,
1956 ticks: tokio::time::Interval,
1957 format_message: fn(&Value) -> String,
1959}
1960
1961impl futures_core::Stream for KeepaliveStream {
1962 type Item = Result<Bytes, std::convert::Infallible>;
1963
1964 fn poll_next(
1965 mut self: std::pin::Pin<&mut Self>,
1966 cx: &mut std::task::Context<'_>,
1967 ) -> std::task::Poll<Option<Self::Item>> {
1968 use std::task::Poll;
1969 let Some(rx) = self.rx.as_mut() else {
1970 return Poll::Ready(None); };
1972 if let Poll::Ready(out) = std::pin::Pin::new(rx).poll(cx) {
1973 let fmt = self.format_message;
1974 let frame = match out {
1975 Ok(Ok(message)) => fmt(&message),
1976 Ok(Err(e)) => sse_error_event(&e),
1977 Err(_) => sse_error_event(&ProxyError::Internal(
1978 "enforce pipeline task dropped".to_owned(),
1979 )),
1980 };
1981 self.rx = None;
1982 return Poll::Ready(Some(Ok(Bytes::from(frame))));
1983 }
1984 if self.ticks.poll_tick(cx).is_ready() {
1985 return Poll::Ready(Some(Ok(Bytes::from_static(b": firstpass routing\n\n"))));
1986 }
1987 Poll::Pending
1988 }
1989}
1990
1991fn sse_error_event(e: &ProxyError) -> String {
1994 let mut out = String::new();
1995 sse_event(
1996 &mut out,
1997 "error",
1998 &serde_json::json!({
1999 "type": "error",
2000 "error": { "type": "api_error", "message": e.client_message() }
2001 }),
2002 );
2003 out
2004}
2005
2006fn parse_model_request(body: &[u8]) -> Option<ModelRequest> {
2015 let json: Value = serde_json::from_slice(body).ok()?;
2016 let raw = json.clone();
2017 let messages_json = json.get("messages")?.as_array()?;
2018 let messages = messages_json
2019 .iter()
2020 .map(|m| ChatMessage {
2021 role: m
2022 .get("role")
2023 .and_then(Value::as_str)
2024 .unwrap_or("user")
2025 .to_owned(),
2026 content: m
2027 .get("content")
2028 .cloned()
2029 .unwrap_or_else(|| Value::String(String::new())),
2030 })
2031 .collect();
2032 let system = json
2033 .get("system")
2034 .and_then(Value::as_str)
2035 .map(str::to_owned);
2036 let max_tokens = json
2037 .get("max_tokens")
2038 .and_then(Value::as_u64)
2039 .and_then(|n| u32::try_from(n).ok())
2040 .unwrap_or(1024);
2041 let tools = json.get("tools").cloned().unwrap_or(Value::Null);
2042 Some(ModelRequest {
2043 model: json
2044 .get("model")
2045 .and_then(Value::as_str)
2046 .unwrap_or_default()
2047 .to_owned(),
2048 system,
2049 messages,
2050 max_tokens,
2051 tools,
2052 raw,
2053 cache_prefix: false,
2054 })
2055}
2056
2057fn anthropic_response_json(resp: &ModelResponse) -> Value {
2067 let content = resp
2068 .raw
2069 .get("content")
2070 .filter(|c| c.is_array())
2071 .cloned()
2072 .unwrap_or_else(|| serde_json::json!([{ "type": "text", "text": resp.text }]));
2073 serde_json::json!({
2074 "id": format!("msg_{}", Uuid::now_v7()),
2075 "type": "message",
2076 "role": "assistant",
2077 "model": resp.model,
2078 "content": content,
2079 "usage": { "input_tokens": resp.in_tokens, "output_tokens": resp.out_tokens },
2080 })
2081}
2082
2083fn sse_event(out: &mut String, event: &str, data: &Value) {
2085 out.push_str("event: ");
2086 out.push_str(event);
2087 out.push_str("\ndata: ");
2088 out.push_str(&data.to_string());
2089 out.push_str("\n\n");
2090}
2091
2092fn anthropic_sse_from_message(message: &Value) -> String {
2099 let mut out = String::new();
2100
2101 let mut start_msg = message.clone();
2103 start_msg["content"] = Value::Array(Vec::new());
2104 sse_event(
2105 &mut out,
2106 "message_start",
2107 &serde_json::json!({ "type": "message_start", "message": start_msg }),
2108 );
2109
2110 let empty = Vec::new();
2111 let blocks = message
2112 .get("content")
2113 .and_then(Value::as_array)
2114 .unwrap_or(&empty);
2115 for (i, block) in blocks.iter().enumerate() {
2116 match block.get("type").and_then(Value::as_str) {
2117 Some("tool_use") => {
2118 let mut shell = block.clone();
2120 shell["input"] = serde_json::json!({});
2121 sse_event(
2122 &mut out,
2123 "content_block_start",
2124 &serde_json::json!({ "type": "content_block_start", "index": i, "content_block": shell }),
2125 );
2126 let input_json = block
2127 .get("input")
2128 .map_or_else(|| "{}".to_owned(), std::string::ToString::to_string);
2129 sse_event(
2130 &mut out,
2131 "content_block_delta",
2132 &serde_json::json!({ "type": "content_block_delta", "index": i,
2133 "delta": { "type": "input_json_delta", "partial_json": input_json } }),
2134 );
2135 }
2136 _ => {
2137 let text = block.get("text").and_then(Value::as_str).unwrap_or("");
2139 sse_event(
2140 &mut out,
2141 "content_block_start",
2142 &serde_json::json!({ "type": "content_block_start", "index": i,
2143 "content_block": { "type": "text", "text": "" } }),
2144 );
2145 sse_event(
2146 &mut out,
2147 "content_block_delta",
2148 &serde_json::json!({ "type": "content_block_delta", "index": i,
2149 "delta": { "type": "text_delta", "text": text } }),
2150 );
2151 }
2152 }
2153 sse_event(
2154 &mut out,
2155 "content_block_stop",
2156 &serde_json::json!({ "type": "content_block_stop", "index": i }),
2157 );
2158 }
2159
2160 let out_tokens = message
2161 .pointer("/usage/output_tokens")
2162 .cloned()
2163 .unwrap_or_else(|| Value::from(0));
2164 sse_event(
2165 &mut out,
2166 "message_delta",
2167 &serde_json::json!({ "type": "message_delta", "delta": { "stop_reason": "end_turn" },
2168 "usage": { "output_tokens": out_tokens } }),
2169 );
2170 sse_event(
2171 &mut out,
2172 "message_stop",
2173 &serde_json::json!({ "type": "message_stop" }),
2174 );
2175 out
2176}
2177
2178async fn observe_passthrough(
2180 state: AppState,
2181 headers: HeaderMap,
2182 body: Bytes,
2183 session_header: Option<String>,
2184 tenant: String,
2185) -> Response {
2186 if is_stream_request(&body) {
2188 return observe_stream(state, headers, body, session_header, tenant).await;
2189 }
2190 let start = Instant::now();
2191 let result = forward_anthropic(
2192 &state.http,
2193 &state.config.upstream_anthropic,
2194 &headers,
2195 body.clone(),
2196 )
2197 .await;
2198 let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
2199
2200 match result {
2201 Ok((status, resp_headers, resp_body)) => {
2202 spawn_trace(
2206 &state,
2207 body,
2208 Some(resp_body.clone()),
2209 latency_ms,
2210 session_header,
2211 tenant,
2212 );
2213 (status, resp_headers, resp_body).into_response()
2214 }
2215 Err(err) => {
2216 spawn_trace(&state, body, None, latency_ms, session_header, tenant);
2217 err.into_response()
2218 }
2219 }
2220}
2221
2222async fn observe_stream(
2231 state: AppState,
2232 headers: HeaderMap,
2233 body: Bytes,
2234 session_header: Option<String>,
2235 tenant: String,
2236) -> Response {
2237 let start = Instant::now();
2238 let result = forward_anthropic_streaming(
2239 &state.http,
2240 &state.config.upstream_anthropic,
2241 &headers,
2242 body.clone(),
2243 )
2244 .await;
2245 let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
2246
2247 match result {
2248 Ok((status, resp_headers, response)) => {
2249 spawn_stream_trace(&state, body, latency_ms, session_header, tenant);
2250 let stream_body = Body::from_stream(response.bytes_stream());
2251 (status, resp_headers, stream_body).into_response()
2252 }
2253 Err(err) => {
2254 spawn_trace(&state, body, None, latency_ms, session_header, tenant);
2255 err.into_response()
2256 }
2257 }
2258}
2259
2260fn spawn_stream_trace(
2262 state: &AppState,
2263 req_body: Bytes,
2264 latency_ms: u64,
2265 session_header: Option<String>,
2266 tenant: String,
2267) {
2268 let config = state.config.clone();
2269 let traces = state.traces.clone();
2270 let spill = state.spill.clone();
2271 tokio::spawn(async move {
2272 let mut trace =
2273 build_stream_trace(&config, &req_body, latency_ms, session_header.as_deref());
2274 trace.tenant_id = tenant;
2276 offer_trace(&traces, spill.as_ref(), trace);
2277 });
2278}
2279
2280fn spawn_trace(
2285 state: &AppState,
2286 req_body: Bytes,
2287 resp_body: Option<Bytes>,
2288 latency_ms: u64,
2289 session_header: Option<String>,
2290 tenant: String,
2291) {
2292 let config = state.config.clone();
2293 let traces = state.traces.clone();
2294 let spill = state.spill.clone();
2295 tokio::spawn(async move {
2296 let mut trace = match resp_body {
2297 Some(resp) => build_trace(
2298 &config,
2299 &req_body,
2300 &resp,
2301 latency_ms,
2302 session_header.as_deref(),
2303 ),
2304 None => build_error_trace(&config, &req_body, latency_ms, session_header.as_deref()),
2305 };
2306 trace.tenant_id = tenant;
2308 offer_trace(&traces, spill.as_ref(), trace);
2309 });
2310}
2311
2312fn session_id(session_header: Option<&str>, trace_id: Uuid) -> String {
2314 session_header
2315 .map(str::to_owned)
2316 .unwrap_or_else(|| trace_id.to_string())
2317}
2318
2319fn prompt_hash(salt: &str, body: &[u8]) -> String {
2322 let mut salted = Vec::with_capacity(salt.len() + body.len());
2323 salted.extend_from_slice(salt.as_bytes());
2324 salted.extend_from_slice(body);
2325 sha256_hex(&salted)
2326}
2327
2328fn request_features(body: &[u8]) -> (Option<String>, u32, bool) {
2332 let Ok(json) = serde_json::from_slice::<Value>(body) else {
2333 return (None, 0, false);
2334 };
2335 let model = json.get("model").and_then(Value::as_str).map(str::to_owned);
2336 let tool_count = json
2337 .get("tools")
2338 .and_then(Value::as_array)
2339 .map_or(0, |tools| u32::try_from(tools.len()).unwrap_or(u32::MAX));
2340 let has_images = json
2341 .get("messages")
2342 .and_then(Value::as_array)
2343 .is_some_and(|messages| messages.iter().any(message_has_image));
2344 (model, tool_count, has_images)
2345}
2346
2347fn message_has_image(message: &Value) -> bool {
2349 message
2350 .get("content")
2351 .and_then(Value::as_array)
2352 .is_some_and(|blocks| {
2353 blocks
2354 .iter()
2355 .any(|block| block.get("type").and_then(Value::as_str) == Some("image"))
2356 })
2357}
2358
2359#[derive(Debug, Default, Clone, PartialEq, Eq)]
2361struct ResponseUsage {
2362 model: Option<String>,
2363 in_tokens: u64,
2364 cache_write_tokens: u64,
2365 cache_read_tokens: u64,
2366 out_tokens: u64,
2367}
2368
2369fn response_usage(body: &[u8]) -> ResponseUsage {
2377 let Ok(json) = serde_json::from_slice::<Value>(body) else {
2378 return ResponseUsage::default();
2379 };
2380 let u = |ptr: &str| json.pointer(ptr).and_then(Value::as_u64).unwrap_or(0);
2381 ResponseUsage {
2382 model: json.get("model").and_then(Value::as_str).map(str::to_owned),
2383 in_tokens: u("/usage/input_tokens"),
2384 cache_write_tokens: u("/usage/cache_creation_input_tokens"),
2385 cache_read_tokens: u("/usage/cache_read_input_tokens"),
2386 out_tokens: u("/usage/output_tokens"),
2387 }
2388}
2389
2390fn build_trace(
2392 config: &ProxyConfig,
2393 req_body: &Bytes,
2394 resp_body: &Bytes,
2395 latency_ms: u64,
2396 session_header: Option<&str>,
2397) -> Trace {
2398 let (req_model, tool_count, has_images) = request_features(req_body);
2399 let usage = response_usage(resp_body);
2400 let (resp_model, in_tokens, out_tokens) = (usage.model, usage.in_tokens, usage.out_tokens);
2401 let (cache_write_tokens, cache_read_tokens) =
2402 (usage.cache_write_tokens, usage.cache_read_tokens);
2403 let model = resp_model
2404 .or(req_model)
2405 .unwrap_or_else(|| "unknown".to_owned());
2406
2407 let cost_usd = config
2408 .prices
2409 .cost_usd_with_cache(
2410 &format!("anthropic/{model}"),
2411 in_tokens,
2412 cache_write_tokens,
2413 cache_read_tokens,
2414 out_tokens,
2415 )
2416 .unwrap_or(0.0);
2417
2418 let attempt = Attempt {
2419 rung: 0,
2420 model,
2421 provider: "anthropic".to_owned(),
2422 in_tokens,
2423 cache_write_tokens,
2424 cache_read_tokens,
2425 out_tokens,
2426 cost_usd,
2427 latency_ms,
2428 gates: Vec::new(),
2429 verdict: Verdict::Pass,
2430 };
2431
2432 let mut trace = base_trace(config, req_body, latency_ms, session_header);
2433 trace.request.features.prompt_token_bucket = token_bucket(in_tokens);
2434 trace.request.features.tool_count = tool_count;
2435 trace.request.features.has_images = has_images;
2436 trace.attempts.push(attempt);
2437 trace.final_ = FinalOutcome {
2438 served_rung: Some(0),
2439 served_from: ServedFrom::Attempt,
2440 total_cost_usd: cost_usd,
2441 gate_cost_usd: 0.0,
2442 total_latency_ms: latency_ms,
2443 escalations: 0,
2444 counterfactual_baseline_usd: cost_usd,
2445 savings_usd: 0.0,
2446 cache_source: None,
2447 };
2448 trace.recompute_savings();
2449 trace
2450}
2451
2452fn build_stream_trace(
2456 config: &ProxyConfig,
2457 req_body: &Bytes,
2458 latency_ms: u64,
2459 session_header: Option<&str>,
2460) -> Trace {
2461 let (req_model, tool_count, has_images) = request_features(req_body);
2462 let model = req_model.unwrap_or_else(|| "unknown".to_owned());
2463
2464 let attempt = Attempt {
2465 rung: 0,
2466 model,
2467 provider: "anthropic".to_owned(),
2468 in_tokens: 0,
2469 cache_write_tokens: 0,
2471 cache_read_tokens: 0,
2472 out_tokens: 0,
2473 cost_usd: 0.0,
2474 latency_ms,
2475 gates: Vec::new(),
2476 verdict: Verdict::Pass,
2477 };
2478
2479 let mut trace = base_trace(config, req_body, latency_ms, session_header);
2480 trace.request.features.tool_count = tool_count;
2481 trace.request.features.has_images = has_images;
2482 trace.attempts.push(attempt);
2483 trace.final_ = FinalOutcome {
2484 served_rung: Some(0),
2485 served_from: ServedFrom::Attempt,
2486 total_cost_usd: 0.0,
2487 gate_cost_usd: 0.0,
2488 total_latency_ms: latency_ms,
2489 escalations: 0,
2490 counterfactual_baseline_usd: 0.0,
2491 savings_usd: 0.0,
2492 cache_source: None,
2493 };
2494 trace.recompute_savings();
2495 trace
2496}
2497
2498fn build_error_trace(
2502 config: &ProxyConfig,
2503 req_body: &Bytes,
2504 latency_ms: u64,
2505 session_header: Option<&str>,
2506) -> Trace {
2507 let (_, tool_count, has_images) = request_features(req_body);
2508 let mut trace = base_trace(config, req_body, latency_ms, session_header);
2509 trace.request.features.tool_count = tool_count;
2510 trace.request.features.has_images = has_images;
2511 trace.final_ = FinalOutcome {
2512 served_rung: None,
2513 served_from: ServedFrom::Error,
2514 total_cost_usd: 0.0,
2515 gate_cost_usd: 0.0,
2516 total_latency_ms: latency_ms,
2517 escalations: 0,
2518 counterfactual_baseline_usd: 0.0,
2519 savings_usd: 0.0,
2520 cache_source: None,
2521 };
2522 trace.recompute_savings();
2523 trace
2524}
2525
2526fn base_trace(
2529 config: &ProxyConfig,
2530 req_body: &Bytes,
2531 latency_ms: u64,
2532 session_header: Option<&str>,
2533) -> Trace {
2534 let trace_id = Uuid::now_v7();
2535 let mut features = Features::new(TaskKind::Other);
2536 features.hour_bucket = hour_bucket(jiff::Timestamp::now());
2537
2538 Trace {
2539 trace_id,
2540 prev_hash: GENESIS_HASH.to_owned(),
2541 tenant_id: config.tenant_id.clone(),
2542 session_id: session_id(session_header, trace_id),
2543 ts: jiff::Timestamp::now(),
2544 mode: Mode::Observe,
2545 policy: PolicyRef {
2546 id: "observe-passthrough@v0".to_owned(),
2547 explore: false,
2548 propensity: None,
2549 mode_profile: None,
2550 },
2551 request: RequestInfo {
2552 api: "anthropic.messages".to_owned(),
2553 prompt_hash: prompt_hash(&config.prompt_salt, req_body),
2554 features,
2555 },
2556 attempts: Vec::new(),
2557 deferred: Vec::new(),
2558 final_: FinalOutcome {
2559 served_rung: None,
2560 served_from: ServedFrom::Error,
2561 total_cost_usd: 0.0,
2562 gate_cost_usd: 0.0,
2563 total_latency_ms: latency_ms,
2564 escalations: 0,
2565 counterfactual_baseline_usd: 0.0,
2566 savings_usd: 0.0,
2567 cache_source: None,
2568 },
2569 probe: None,
2570 rollout: None,
2571 shadow: None,
2572 route_ix: None,
2573 predicted_pass: None,
2574 elastic: None,
2575 }
2576}
2577
2578fn openai_messages_have_tool_calls(body: &[u8]) -> bool {
2583 serde_json::from_slice::<Value>(body)
2584 .ok()
2585 .and_then(|json| {
2586 json.get("messages").and_then(Value::as_array).map(|msgs| {
2587 msgs.iter().any(|m| {
2588 m.get("tool_calls").is_some()
2589 || m.get("role").and_then(Value::as_str) == Some("tool")
2590 })
2591 })
2592 })
2593 .unwrap_or(false)
2594}
2595
2596fn openai_has_http_images(body: &[u8]) -> bool {
2600 serde_json::from_slice::<Value>(body)
2601 .ok()
2602 .and_then(|json| {
2603 json.get("messages").and_then(Value::as_array).map(|msgs| {
2604 msgs.iter().any(|m| {
2605 m.get("content")
2606 .and_then(Value::as_array)
2607 .is_some_and(|parts| {
2608 parts.iter().any(|p| {
2609 p.get("type").and_then(Value::as_str) == Some("image_url")
2610 && p.pointer("/image_url/url")
2611 .and_then(Value::as_str)
2612 .is_some_and(|u| {
2613 u.starts_with("http://") || u.starts_with("https://")
2614 })
2615 })
2616 })
2617 })
2618 })
2619 })
2620 .unwrap_or(false)
2621}
2622
2623fn openai_messages_have_images(body: &[u8]) -> bool {
2626 serde_json::from_slice::<Value>(body)
2627 .ok()
2628 .and_then(|json| {
2629 json.get("messages").and_then(Value::as_array).map(|msgs| {
2630 msgs.iter().any(|m| {
2631 m.get("content")
2632 .and_then(Value::as_array)
2633 .is_some_and(|parts| {
2634 parts
2635 .iter()
2636 .any(|p| p.get("type").and_then(Value::as_str) == Some("image_url"))
2637 })
2638 })
2639 })
2640 })
2641 .unwrap_or(false)
2642}
2643
2644fn extract_openai_features(headers: &HeaderMap, body: &[u8]) -> Features {
2647 let Ok(json) = serde_json::from_slice::<Value>(body) else {
2648 let mut f = Features::new(TaskKind::Other);
2649 f.hour_bucket = hour_bucket(jiff::Timestamp::now());
2650 return f;
2651 };
2652 let tool_count = json
2653 .get("tools")
2654 .and_then(Value::as_array)
2655 .map_or(0, |tools| u32::try_from(tools.len()).unwrap_or(u32::MAX));
2656 let has_images = openai_messages_have_images(body);
2657 let mut f = Features::new(TaskKind::Other);
2658 f.agent = header_str(headers, AGENT_HEADER);
2659 f.subagent = header_str(headers, SUBAGENT_HEADER);
2660 f.tool_count = tool_count;
2661 f.has_images = has_images;
2662 f.prompt_token_bucket = token_bucket(body.len() as u64);
2663 f.hour_bucket = hour_bucket(jiff::Timestamp::now());
2664 f
2665}
2666
2667fn parse_data_url(url: &str) -> Option<(&str, &str)> {
2671 let rest = url.strip_prefix("data:")?;
2672 let (meta, data) = rest.split_once(',')?;
2673 let media_type = meta.strip_suffix(";base64")?;
2674 Some((media_type, data))
2675}
2676
2677fn translate_openai_user_content(content: &Value) -> Option<Value> {
2680 match content {
2681 Value::String(_) => Some(content.clone()),
2683 Value::Array(parts) => {
2684 let mut blocks: Vec<Value> = Vec::with_capacity(parts.len());
2685 for part in parts {
2686 match part.get("type").and_then(Value::as_str) {
2687 Some("text") => {
2688 let text = part.get("text").and_then(Value::as_str).unwrap_or("");
2689 blocks.push(serde_json::json!({ "type": "text", "text": text }));
2690 }
2691 Some("image_url") => {
2692 let url = part.pointer("/image_url/url").and_then(Value::as_str)?;
2693 if url.starts_with("http://") || url.starts_with("https://") {
2694 return None; }
2696 let (media_type, data) = parse_data_url(url)?;
2698 blocks.push(serde_json::json!({
2699 "type": "image",
2700 "source": { "type": "base64", "media_type": media_type, "data": data }
2701 }));
2702 }
2703 _ => {} }
2705 }
2706 Some(Value::Array(blocks))
2707 }
2708 _ => Some(Value::String(String::new())),
2709 }
2710}
2711
2712fn translate_openai_tools(tools: &Value) -> Value {
2716 let Some(arr) = tools.as_array() else {
2717 return Value::Null;
2718 };
2719 let converted: Vec<Value> = arr
2720 .iter()
2721 .map(|tool| {
2722 let func = tool.get("function").unwrap_or(&Value::Null);
2723 let mut out = serde_json::json!({
2724 "name": func.get("name").cloned().unwrap_or(Value::String(String::new())),
2725 "input_schema": func.get("parameters").cloned()
2726 .unwrap_or_else(|| serde_json::json!({ "type": "object" })),
2727 });
2728 if let Some(desc) = func.get("description") {
2729 out["description"] = desc.clone();
2730 }
2731 out
2732 })
2733 .collect();
2734 Value::Array(converted)
2735}
2736
2737fn translate_openai_tool_choice(tc: &Value) -> Value {
2739 match tc {
2740 Value::String(s) => match s.as_str() {
2741 "auto" => serde_json::json!({ "type": "auto" }),
2742 "required" => serde_json::json!({ "type": "any" }),
2743 _ => serde_json::json!({ "type": "auto" }),
2745 },
2746 Value::Object(_) => {
2747 if tc.get("type").and_then(Value::as_str) == Some("function") {
2749 let name = tc.pointer("/function/name").cloned().unwrap_or(Value::Null);
2750 serde_json::json!({ "type": "tool", "name": name })
2751 } else {
2752 serde_json::json!({ "type": "auto" })
2753 }
2754 }
2755 _ => serde_json::json!({ "type": "auto" }),
2756 }
2757}
2758
2759pub fn parse_openai_request(body: &[u8], carry_raw: bool) -> Option<ModelRequest> {
2772 let json: Value = serde_json::from_slice(body).ok()?;
2773 let raw = if carry_raw { json.clone() } else { Value::Null };
2774
2775 let messages_json = json.get("messages")?.as_array()?;
2776
2777 let mut system: Option<String> = None;
2778 let mut messages: Vec<ChatMessage> = Vec::with_capacity(messages_json.len());
2779 let mut tools = Value::Null;
2780 let mut tool_choice_override: Option<Value> = None;
2781
2782 for msg in messages_json {
2783 let role = msg.get("role").and_then(Value::as_str).unwrap_or("user");
2784 match role {
2785 "system" => {
2786 if let Some(s) = msg.get("content").and_then(Value::as_str) {
2791 system = Some(s.to_owned());
2792 }
2793 }
2794 "user" => {
2795 let content_val = msg.get("content").unwrap_or(&Value::Null);
2796 let translated = translate_openai_user_content(content_val)?;
2797 messages.push(ChatMessage {
2798 role: "user".to_owned(),
2799 content: translated,
2800 });
2801 }
2802 "assistant" => {
2803 if let Some(tc_arr) = msg.get("tool_calls").and_then(Value::as_array) {
2804 let mut blocks: Vec<Value> = Vec::new();
2806 if let Some(text) = msg.get("content").and_then(Value::as_str)
2808 && !text.is_empty()
2809 {
2810 blocks.push(serde_json::json!({ "type": "text", "text": text }));
2811 }
2812 for tc in tc_arr {
2813 let id = tc.get("id").and_then(Value::as_str).unwrap_or("");
2814 let func = tc.get("function").unwrap_or(&Value::Null);
2815 let name = func.get("name").and_then(Value::as_str).unwrap_or("");
2816 let args_str = func
2817 .get("arguments")
2818 .and_then(Value::as_str)
2819 .unwrap_or("{}");
2820 let input: Value = serde_json::from_str(args_str)
2821 .unwrap_or_else(|_| serde_json::json!({}));
2822 blocks.push(serde_json::json!({
2823 "type": "tool_use",
2824 "id": id,
2825 "name": name,
2826 "input": input,
2827 }));
2828 }
2829 messages.push(ChatMessage {
2830 role: "assistant".to_owned(),
2831 content: Value::Array(blocks),
2832 });
2833 } else {
2834 let content = msg
2836 .get("content")
2837 .cloned()
2838 .unwrap_or_else(|| Value::String(String::new()));
2839 messages.push(ChatMessage {
2840 role: "assistant".to_owned(),
2841 content,
2842 });
2843 }
2844 }
2845 "tool" => {
2846 let tool_call_id = msg
2848 .get("tool_call_id")
2849 .and_then(Value::as_str)
2850 .unwrap_or("");
2851 let content = msg
2852 .get("content")
2853 .cloned()
2854 .unwrap_or_else(|| Value::String(String::new()));
2855 let result_block = serde_json::json!({
2856 "type": "tool_result",
2857 "tool_use_id": tool_call_id,
2858 "content": content,
2859 });
2860 messages.push(ChatMessage {
2861 role: "user".to_owned(),
2862 content: Value::Array(vec![result_block]),
2863 });
2864 }
2865 _ => {} }
2867 }
2868
2869 if !carry_raw {
2871 if let Some(t) = json.get("tools") {
2872 tools = translate_openai_tools(t);
2873 }
2874 if let Some(tc) = json.get("tool_choice") {
2875 tool_choice_override = Some(translate_openai_tool_choice(tc));
2876 }
2877 } else {
2878 tools = json.get("tools").cloned().unwrap_or(Value::Null);
2879 }
2880
2881 let max_tokens = json
2882 .get("max_tokens")
2883 .and_then(Value::as_u64)
2884 .and_then(|n| u32::try_from(n).ok())
2885 .unwrap_or(1024);
2886
2887 let model = json
2888 .get("model")
2889 .and_then(Value::as_str)
2890 .unwrap_or_default()
2891 .to_owned();
2892
2893 let _ = tool_choice_override; Some(ModelRequest {
2904 model,
2905 system,
2906 messages,
2907 max_tokens,
2908 tools,
2909 raw,
2910 cache_prefix: false,
2911 })
2912}
2913
2914fn extract_openai_content_and_tools(raw: &Value, text: &str) -> (Value, Option<Value>) {
2921 if let Some(blocks) = raw.get("content").and_then(Value::as_array) {
2923 let mut text_parts: Vec<&str> = Vec::new();
2924 let mut tool_calls: Vec<Value> = Vec::new();
2925 for block in blocks {
2926 match block.get("type").and_then(Value::as_str) {
2927 Some("text") => {
2928 if let Some(t) = block.get("text").and_then(Value::as_str) {
2929 text_parts.push(t);
2930 }
2931 }
2932 Some("tool_use") => {
2933 let id = block.get("id").and_then(Value::as_str).unwrap_or("");
2934 let name = block.get("name").and_then(Value::as_str).unwrap_or("");
2935 let input_str = block
2936 .get("input")
2937 .map_or_else(|| "{}".to_owned(), std::string::ToString::to_string);
2938 tool_calls.push(serde_json::json!({
2939 "id": id,
2940 "type": "function",
2941 "function": { "name": name, "arguments": input_str },
2942 }));
2943 }
2944 _ => {}
2945 }
2946 }
2947 let content_text = if tool_calls.is_empty() || !text_parts.is_empty() {
2948 Value::String(text_parts.join(""))
2949 } else {
2950 Value::Null };
2952 let tc = if tool_calls.is_empty() {
2953 None
2954 } else {
2955 Some(Value::Array(tool_calls))
2956 };
2957 return (content_text, tc);
2958 }
2959
2960 if let Some(msg) = raw.pointer("/choices/0/message") {
2962 let content = msg
2963 .get("content")
2964 .cloned()
2965 .unwrap_or(Value::String(text.to_owned()));
2966 let tc = msg.get("tool_calls").cloned();
2967 return (content, tc);
2968 }
2969
2970 (Value::String(text.to_owned()), None)
2972}
2973
2974fn openai_response_json(resp: &ModelResponse) -> Value {
2977 let (content_text, tool_calls) = extract_openai_content_and_tools(&resp.raw, &resp.text);
2978 let finish_reason = if tool_calls.is_some() {
2979 "tool_calls"
2980 } else {
2981 "stop"
2982 };
2983 let mut message = serde_json::json!({
2984 "role": "assistant",
2985 "content": content_text,
2986 });
2987 if let Some(tc) = tool_calls {
2988 message["tool_calls"] = tc;
2989 }
2990 serde_json::json!({
2991 "id": format!("chatcmpl-{}", Uuid::now_v7()),
2992 "object": "chat.completion",
2993 "created": jiff::Timestamp::now().as_second(),
2994 "model": resp.model,
2995 "choices": [{
2996 "index": 0,
2997 "message": message,
2998 "finish_reason": finish_reason,
2999 }],
3000 "usage": {
3001 "prompt_tokens": resp.in_tokens,
3002 "completion_tokens": resp.out_tokens,
3003 "total_tokens": resp.in_tokens + resp.out_tokens,
3004 }
3005 })
3006}
3007
3008fn openai_sse_from_message(message: &Value) -> String {
3012 let id = message
3013 .get("id")
3014 .and_then(Value::as_str)
3015 .unwrap_or("chatcmpl-unknown")
3016 .to_owned();
3017 let created = message.get("created").cloned().unwrap_or(Value::from(0));
3018 let model = message
3019 .get("model")
3020 .and_then(Value::as_str)
3021 .unwrap_or("unknown");
3022
3023 let choices = message.get("choices").and_then(Value::as_array);
3024 let msg = choices
3025 .and_then(|c| c.first())
3026 .and_then(|c| c.get("message"));
3027 let content = msg
3028 .and_then(|m| m.get("content"))
3029 .cloned()
3030 .unwrap_or(Value::Null);
3031 let tool_calls = msg.and_then(|m| m.get("tool_calls")).cloned();
3032 let finish_reason = choices
3033 .and_then(|c| c.first())
3034 .and_then(|c| c.get("finish_reason"))
3035 .cloned()
3036 .unwrap_or_else(|| Value::String("stop".to_owned()));
3037
3038 let mut out = String::new();
3039 let chunk = |delta: Value| {
3040 serde_json::json!({
3041 "id": id,
3042 "object": "chat.completion.chunk",
3043 "created": created,
3044 "model": model,
3045 "choices": [{ "index": 0, "delta": delta, "finish_reason": Value::Null }]
3046 })
3047 };
3048
3049 let role_chunk = chunk(serde_json::json!({ "role": "assistant", "content": "" }));
3051 out.push_str("data: ");
3052 out.push_str(&role_chunk.to_string());
3053 out.push_str("\n\n");
3054
3055 if let Value::String(text) = &content
3057 && !text.is_empty()
3058 {
3059 let content_chunk = chunk(serde_json::json!({ "content": text }));
3060 out.push_str("data: ");
3061 out.push_str(&content_chunk.to_string());
3062 out.push_str("\n\n");
3063 }
3064
3065 if let Some(tc) = tool_calls {
3067 let tc_chunk = chunk(serde_json::json!({ "tool_calls": tc }));
3068 out.push_str("data: ");
3069 out.push_str(&tc_chunk.to_string());
3070 out.push_str("\n\n");
3071 }
3072
3073 let finish_chunk = serde_json::json!({
3075 "id": id,
3076 "object": "chat.completion.chunk",
3077 "created": created,
3078 "model": model,
3079 "choices": [{ "index": 0, "delta": {}, "finish_reason": finish_reason }]
3080 });
3081 out.push_str("data: ");
3082 out.push_str(&finish_chunk.to_string());
3083 out.push_str("\n\n");
3084
3085 out.push_str("data: [DONE]\n\n");
3086 out
3087}
3088
3089#[allow(clippy::too_many_arguments)]
3094async fn handle_enforce_openai(
3095 state: &AppState,
3096 headers: &HeaderMap,
3097 body: &Bytes,
3098 features: Features,
3099 route: &Route,
3100 route_ix: usize,
3103 session_header: Option<String>,
3104 tenant: String,
3105 routing_mode: RoutingMode,
3106) -> Response {
3107 if is_stream_request(body) {
3108 let (tx, rx) = tokio::sync::oneshot::channel::<Result<Value, ProxyError>>();
3109 let (state_c, headers_c, body_c, route_c) =
3110 (state.clone(), headers.clone(), body.clone(), route.clone());
3111 tokio::spawn(async move {
3112 let out = enforce_pipeline_openai(
3113 &state_c,
3114 &headers_c,
3115 &body_c,
3116 features,
3117 &route_c,
3118 route_ix,
3119 session_header,
3120 tenant,
3121 routing_mode,
3122 )
3123 .await;
3124 let _ = tx.send(out);
3125 });
3126 return sse_keepalive_response(rx, openai_sse_from_message);
3127 }
3128 match enforce_pipeline_openai(
3129 state,
3130 headers,
3131 body,
3132 features,
3133 route,
3134 route_ix,
3135 session_header,
3136 tenant,
3137 routing_mode,
3138 )
3139 .await
3140 {
3141 Ok(message) => (axum::http::StatusCode::OK, Json(message)).into_response(),
3142 Err(e) => e.into_response(),
3143 }
3144}
3145
3146async fn observe_passthrough_openai(
3152 state: AppState,
3153 headers: HeaderMap,
3154 body: Bytes,
3155 session_header: Option<String>,
3156 tenant: String,
3157) -> Response {
3158 observe_passthrough_openai_path(
3159 state,
3160 headers,
3161 body,
3162 session_header,
3163 tenant,
3164 crate::upstream::OPENAI_CHAT_PATH,
3165 )
3166 .await
3167}
3168
3169async fn observe_passthrough_openai_path(
3175 state: AppState,
3176 headers: HeaderMap,
3177 body: Bytes,
3178 session_header: Option<String>,
3179 tenant: String,
3180 path: &'static str,
3181) -> Response {
3182 if is_stream_request(&body) {
3183 return observe_stream_openai_path(state, headers, body, session_header, tenant, path)
3184 .await;
3185 }
3186 let start = Instant::now();
3187 let result = crate::upstream::forward_openai_path(
3188 &state.http,
3189 &state.config.upstream_openai,
3190 path,
3191 &headers,
3192 body.clone(),
3193 )
3194 .await;
3195 let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
3196 match result {
3197 Ok((status, resp_headers, resp_body)) => {
3198 spawn_trace(
3199 &state,
3200 body,
3201 Some(resp_body.clone()),
3202 latency_ms,
3203 session_header,
3204 tenant,
3205 );
3206 (status, resp_headers, resp_body).into_response()
3207 }
3208 Err(err) => {
3209 spawn_trace(&state, body, None, latency_ms, session_header, tenant);
3210 err.into_response()
3211 }
3212 }
3213}
3214
3215async fn observe_stream_openai_path(
3219 state: AppState,
3220 headers: HeaderMap,
3221 body: Bytes,
3222 session_header: Option<String>,
3223 tenant: String,
3224 path: &'static str,
3225) -> Response {
3226 let start = Instant::now();
3227 let result = crate::upstream::forward_openai_streaming_path(
3228 &state.http,
3229 &state.config.upstream_openai,
3230 path,
3231 &headers,
3232 body.clone(),
3233 )
3234 .await;
3235 let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
3236 match result {
3237 Ok((status, resp_headers, response)) => {
3238 spawn_stream_trace(&state, body, latency_ms, session_header, tenant);
3239 let stream_body = Body::from_stream(response.bytes_stream());
3240 (status, resp_headers, stream_body).into_response()
3241 }
3242 Err(err) => {
3243 spawn_trace(&state, body, None, latency_ms, session_header, tenant);
3244 err.into_response()
3245 }
3246 }
3247}
3248
3249async fn chat_completions(
3254 State(state): State<AppState>,
3255 Extension(TenantId(tenant)): Extension<TenantId>,
3256 headers: HeaderMap,
3257 body: Bytes,
3258) -> Response {
3259 let session_header = header_str(&headers, SESSION_HEADER);
3260
3261 if let Some(routing) = state.config.routing.as_ref() {
3262 let features = extract_openai_features(&headers, &body);
3263 if let Some(route) = routing
3264 .route_for(&features)
3265 .filter(|r| r.mode == Mode::Enforce && !r.ladder.is_empty())
3266 {
3267 let route_ix = routing
3269 .routes
3270 .iter()
3271 .position(|r| std::ptr::eq(r, route))
3272 .unwrap_or(0);
3273 let route = route.clone();
3274 let routing_mode = resolve_mode(&headers, &route, &state.config);
3276 if routing_mode == RoutingMode::Observe {
3278 return observe_passthrough_openai(state, headers, body, session_header, tenant)
3279 .await;
3280 }
3281 if enforce_can_handle(
3282 &features,
3283 &body,
3284 routing.escalation.enforce_structured,
3285 &route.ladder,
3286 &state.providers,
3287 Dialect::Openai,
3288 ) {
3289 return handle_enforce_openai(
3290 &state,
3291 &headers,
3292 &body,
3293 features,
3294 &route,
3295 route_ix,
3296 session_header,
3297 tenant,
3298 routing_mode,
3299 )
3300 .await;
3301 }
3302 tracing::info!(
3303 "enforce route matched but OpenAI structured request can't be routed faithfully (flag/ladder); serving via observe passthrough"
3304 );
3305 }
3306 }
3307 observe_passthrough_openai(state, headers, body, session_header, tenant).await
3308}
3309
3310async fn responses(
3319 State(state): State<AppState>,
3320 Extension(TenantId(tenant)): Extension<TenantId>,
3321 headers: HeaderMap,
3322 body: Bytes,
3323) -> Response {
3324 let session_header = header_str(&headers, SESSION_HEADER);
3325
3326 if is_stream_request(&body) {
3330 return observe_passthrough_openai_path(
3331 state,
3332 headers,
3333 body,
3334 session_header,
3335 tenant,
3336 crate::upstream::OPENAI_RESPONSES_PATH,
3337 )
3338 .await;
3339 }
3340
3341 let Ok(parsed) = serde_json::from_slice::<Value>(&body) else {
3342 return observe_passthrough_openai_path(
3343 state,
3344 headers,
3345 body,
3346 session_header,
3347 tenant,
3348 crate::upstream::OPENAI_RESPONSES_PATH,
3349 )
3350 .await;
3351 };
3352 if crate::responses::has_untranslatable_content(&parsed) {
3356 return observe_passthrough_openai_path(
3357 state,
3358 headers,
3359 body,
3360 session_header,
3361 tenant,
3362 crate::upstream::OPENAI_RESPONSES_PATH,
3363 )
3364 .await;
3365 }
3366 let chat_body = Bytes::from(crate::responses::request_to_chat(&parsed).to_string());
3367
3368 if let Some(routing) = state.config.routing.as_ref() {
3369 let features = extract_openai_features(&headers, &chat_body);
3370 if let Some(route) = routing
3371 .route_for(&features)
3372 .filter(|r| r.mode == Mode::Enforce && !r.ladder.is_empty())
3373 {
3374 let route_ix = routing
3375 .routes
3376 .iter()
3377 .position(|r| std::ptr::eq(r, route))
3378 .unwrap_or(0);
3379 let route = route.clone();
3380 let routing_mode = resolve_mode(&headers, &route, &state.config);
3381 if routing_mode != RoutingMode::Observe
3382 && enforce_can_handle(
3383 &features,
3384 &chat_body,
3385 routing.escalation.enforce_structured,
3386 &route.ladder,
3387 &state.providers,
3388 Dialect::Openai,
3389 )
3390 {
3391 return match enforce_pipeline_openai_as(
3392 &state,
3393 &headers,
3394 &chat_body,
3395 &body,
3396 "openai.responses",
3397 features,
3398 &route,
3399 route_ix,
3400 session_header,
3401 tenant,
3402 routing_mode,
3403 )
3404 .await
3405 {
3406 Ok(chat) => Json(crate::responses::response_from_chat(&chat)).into_response(),
3407 Err(e) => e.into_response(),
3408 };
3409 }
3410 }
3411 }
3412 observe_passthrough_openai_path(
3413 state,
3414 headers,
3415 body,
3416 session_header,
3417 tenant,
3418 crate::upstream::OPENAI_RESPONSES_PATH,
3419 )
3420 .await
3421}
3422
3423#[cfg(test)]
3424mod tests {
3425 use bytes::Bytes;
3426
3427 use super::*;
3428
3429 fn test_config() -> ProxyConfig {
3430 ProxyConfig::from_lookup(|_| None).unwrap()
3431 }
3432
3433 #[test]
3434 fn build_trace_maps_request_and_response_fields() {
3435 let config = test_config();
3436 let req = Bytes::from_static(
3437 br#"{"model":"claude-haiku-4-5","tools":[{"name":"a"}],"messages":[]}"#,
3438 );
3439 let resp = Bytes::from_static(
3440 br#"{"model":"claude-haiku-4-5","usage":{"input_tokens":1200,"output_tokens":300}}"#,
3441 );
3442
3443 let trace = build_trace(&config, &req, &resp, 42, Some("sess-1"));
3444
3445 assert_eq!(trace.request.api, "anthropic.messages");
3446 assert_eq!(trace.session_id, "sess-1");
3447 assert_eq!(trace.attempts.len(), 1);
3448 let attempt = &trace.attempts[0];
3449 assert_eq!(attempt.model, "claude-haiku-4-5");
3450 assert_eq!(attempt.provider, "anthropic");
3451 assert_eq!(attempt.in_tokens, 1200);
3452 assert_eq!(attempt.out_tokens, 300);
3453 assert!(attempt.cost_usd > 0.0);
3454 assert_eq!(trace.request.features.tool_count, 1);
3455 assert!(!trace.request.features.has_images);
3456 assert_eq!(trace.final_.served_rung, Some(0));
3457 }
3458
3459 #[test]
3460 fn build_trace_falls_back_to_trace_id_session_when_header_absent() {
3461 let config = test_config();
3462 let req = Bytes::from_static(b"{}");
3463 let resp = Bytes::from_static(b"{}");
3464
3465 let trace = build_trace(&config, &req, &resp, 1, None);
3466
3467 assert_eq!(trace.session_id, trace.trace_id.to_string());
3468 }
3469
3470 #[test]
3471 fn build_error_trace_has_no_attempts_and_served_from_error() {
3472 let config = test_config();
3473 let req = Bytes::from_static(br#"{"model":"claude-haiku-4-5"}"#);
3474
3475 let trace = build_error_trace(&config, &req, 7, None);
3476
3477 assert!(trace.attempts.is_empty());
3478 assert_eq!(trace.final_.served_from, ServedFrom::Error);
3479 assert_eq!(trace.final_.served_rung, None);
3480 }
3481
3482 #[test]
3483 fn message_with_image_block_sets_has_images() {
3484 let req = br#"{"messages":[{"role":"user","content":[{"type":"image"}]}]}"#;
3485 let (_, _, has_images) = request_features(req);
3486 assert!(has_images);
3487 }
3488
3489 #[test]
3490 fn prompt_hash_never_contains_raw_prompt_text() {
3491 let hash = prompt_hash("salt", b"super secret prompt");
3492 assert!(!hash.contains("secret"));
3493 assert_eq!(hash.len(), 64);
3494 }
3495
3496 #[test]
3497 fn parse_model_request_preserves_content_verbatim_and_projects_text() {
3498 let body = br#"{"model":"m","system":"sys","max_tokens":50,
3499 "messages":[{"role":"user","content":[{"type":"text","text":"a"},{"type":"text","text":"b"}]},
3500 {"role":"assistant","content":"c"}]}"#;
3501 let req = parse_model_request(body).unwrap();
3502 assert_eq!(req.system.as_deref(), Some("sys"));
3503 assert_eq!(req.max_tokens, 50);
3504 assert_eq!(req.messages.len(), 2);
3505 assert_eq!(
3507 req.messages[0].content,
3508 serde_json::json!([{"type":"text","text":"a"},{"type":"text","text":"b"}])
3509 );
3510 assert_eq!(req.messages[1].content, Value::String("c".to_owned()));
3512 assert_eq!(req.messages[0].text_view(), "a\nb");
3514 assert_eq!(req.messages[1].text_view(), "c");
3515 }
3516
3517 #[test]
3518 fn tool_and_image_blocks_survive_the_request_round_trip() {
3519 let body = br#"{"model":"m","max_tokens":50,"messages":[
3521 {"role":"assistant","content":[{"type":"tool_use","id":"t1","name":"calc","input":{"x":1}}]},
3522 {"role":"user","content":[
3523 {"type":"tool_result","tool_use_id":"t1","content":"2"},
3524 {"type":"image","source":{"type":"base64","media_type":"image/png","data":"AA=="}}
3525 ]}]}"#;
3526 let req = parse_model_request(body).unwrap();
3527 let round_tripped = serde_json::to_value(&req.messages).unwrap();
3528 assert_eq!(
3529 round_tripped,
3530 serde_json::json!([
3531 {"role":"assistant","content":[{"type":"tool_use","id":"t1","name":"calc","input":{"x":1}}]},
3532 {"role":"user","content":[
3533 {"type":"tool_result","tool_use_id":"t1","content":"2"},
3534 {"type":"image","source":{"type":"base64","media_type":"image/png","data":"AA=="}}
3535 ]}
3536 ])
3537 );
3538 }
3539
3540 #[test]
3541 fn text_message_serializes_byte_identical_to_a_plain_string() {
3542 let m = ChatMessage::text("user", "hello");
3544 assert_eq!(
3545 serde_json::to_string(&m).unwrap(),
3546 r#"{"role":"user","content":"hello"}"#
3547 );
3548 }
3549
3550 #[test]
3551 fn parse_model_request_rejects_non_message_bodies() {
3552 assert!(parse_model_request(b"not json").is_none());
3553 assert!(parse_model_request(br#"{"no":"messages"}"#).is_none());
3554 }
3555
3556 use crate::provider::{MockProvider, ModelResponse, Provider, ProviderError, ProviderRegistry};
3559 use axum::extract::State;
3560 use std::collections::HashMap;
3561 use std::sync::Arc;
3562 use tokio::sync::mpsc;
3563
3564 fn model_resp(model: &str, text: &str) -> ModelResponse {
3565 ModelResponse {
3566 model: model.to_owned(),
3567 text: text.to_owned(),
3568 in_tokens: 1000,
3569 cache_write_tokens: 0,
3570 cache_read_tokens: 0,
3571 out_tokens: 400,
3572 raw: serde_json::Value::Null,
3573 }
3574 }
3575
3576 fn enforce_state(
3579 ladder: &[&str],
3580 gates: &[&str],
3581 outcomes: Vec<(&str, Result<ModelResponse, ProviderError>)>,
3582 ) -> (AppState, mpsc::Receiver<Trace>) {
3583 let toml = format!(
3584 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\nladder = [{}]\ngates = [{}]\n",
3585 ladder
3586 .iter()
3587 .map(|m| format!("\"{m}\""))
3588 .collect::<Vec<_>>()
3589 .join(", "),
3590 gates
3591 .iter()
3592 .map(|g| format!("\"{g}\""))
3593 .collect::<Vec<_>>()
3594 .join(", "),
3595 );
3596 let config = ProxyConfig::from_lookup(|k| match k {
3597 "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
3598 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
3599 _ => None,
3600 })
3601 .unwrap();
3602
3603 let mut outs = HashMap::new();
3604 for (model, out) in outcomes {
3605 outs.insert(model.to_owned(), out);
3606 }
3607 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
3608 map.insert(
3609 "anthropic".to_owned(),
3610 Arc::new(MockProvider::new("anthropic", outs)),
3611 );
3612 let providers = ProviderRegistry::from_map(map);
3613
3614 let (traces, rx) = mpsc::channel(64);
3615 let state = AppState {
3616 config: Arc::new(config),
3617 http: reqwest::Client::new(),
3618 providers,
3619 gate_health: Arc::new(GateHealthRegistry::new()),
3620 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
3621 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
3622 traces,
3623 adaptive: None,
3624 bandit: None,
3625 promoter: None,
3626 verified_cache: None,
3627 predictor: None,
3628 tenant_rate_limiter: None,
3629 spill: None,
3630 };
3631 (state, rx)
3632 }
3633
3634 fn user_body() -> Bytes {
3635 Bytes::from_static(
3636 br#"{"model":"ignored","max_tokens":64,"messages":[{"role":"user","content":"hi"}]}"#,
3637 )
3638 }
3639
3640 async fn body_json(resp: Response) -> Value {
3641 let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
3642 .await
3643 .unwrap();
3644 serde_json::from_slice(&bytes).unwrap()
3645 }
3646
3647 #[tokio::test]
3648 async fn mode_profile_stamped_on_trace_when_non_balanced() {
3649 let (state, mut rx) = enforce_state(
3650 &["anthropic/claude-haiku-4-5"],
3651 &["non-empty"],
3652 vec![(
3653 "anthropic/claude-haiku-4-5",
3654 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
3655 )],
3656 );
3657 let mut headers = HeaderMap::new();
3658 headers.insert("x-firstpass-mode", "quality".parse().unwrap());
3659 let resp = messages(
3660 State(state),
3661 Extension(TenantId("default".to_owned())),
3662 headers,
3663 user_body(),
3664 )
3665 .await;
3666 assert_eq!(resp.status(), axum::http::StatusCode::OK);
3667 let trace = rx.try_recv().expect("trace enqueued");
3668 assert_eq!(
3669 trace.policy.mode_profile.as_deref(),
3670 Some("quality"),
3671 "mode_profile must be stamped when quality mode is active"
3672 );
3673 }
3674
3675 #[tokio::test]
3676 async fn mode_profile_absent_from_trace_when_balanced() {
3677 let (state, mut rx) = enforce_state(
3678 &["anthropic/claude-haiku-4-5"],
3679 &["non-empty"],
3680 vec![(
3681 "anthropic/claude-haiku-4-5",
3682 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
3683 )],
3684 );
3685 let resp = messages(
3687 State(state),
3688 Extension(TenantId("default".to_owned())),
3689 HeaderMap::new(),
3690 user_body(),
3691 )
3692 .await;
3693 assert_eq!(resp.status(), axum::http::StatusCode::OK);
3694 let trace = rx.try_recv().expect("trace enqueued");
3695 assert!(
3696 trace.policy.mode_profile.is_none(),
3697 "mode_profile must be absent when Balanced (byte-identical invariant)"
3698 );
3699 }
3700
3701 #[tokio::test]
3702 async fn enforce_serves_first_pass_and_returns_anthropic_shape() {
3703 let (state, mut rx) = enforce_state(
3704 &["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"],
3705 &["non-empty"],
3706 vec![(
3707 "anthropic/claude-haiku-4-5",
3708 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
3709 )],
3710 );
3711 let resp = messages(
3712 State(state),
3713 Extension(TenantId("default".to_owned())),
3714 HeaderMap::new(),
3715 user_body(),
3716 )
3717 .await;
3718 assert_eq!(resp.status(), axum::http::StatusCode::OK);
3719 let json = body_json(resp).await;
3720 assert_eq!(json["type"], "message");
3721 assert_eq!(json["content"][0]["text"], "hello");
3722 assert_eq!(json["model"], "anthropic/claude-haiku-4-5");
3723
3724 let trace = rx.try_recv().expect("a trace was enqueued");
3725 assert_eq!(trace.mode, Mode::Enforce);
3726 assert_eq!(trace.final_.served_rung, Some(0));
3727 assert_eq!(trace.attempts.len(), 1);
3728 }
3729
3730 #[tokio::test]
3731 async fn enforce_escalates_then_serves_and_traces_two_attempts() {
3732 let (state, mut rx) = enforce_state(
3733 &["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"],
3734 &["non-empty"],
3735 vec![
3736 (
3737 "anthropic/claude-haiku-4-5",
3738 Ok(model_resp("anthropic/claude-haiku-4-5", " ")),
3739 ), (
3741 "anthropic/claude-sonnet-5",
3742 Ok(model_resp("anthropic/claude-sonnet-5", "answer")),
3743 ),
3744 ],
3745 );
3746 let resp = messages(
3747 State(state),
3748 Extension(TenantId("default".to_owned())),
3749 HeaderMap::new(),
3750 user_body(),
3751 )
3752 .await;
3753 let json = body_json(resp).await;
3754 assert_eq!(json["content"][0]["text"], "answer");
3755
3756 let trace = rx.try_recv().expect("trace enqueued");
3757 assert_eq!(trace.attempts.len(), 2);
3758 assert_eq!(trace.final_.escalations, 1);
3759 assert_eq!(trace.final_.served_rung, Some(1));
3760 }
3761
3762 #[tokio::test]
3763 async fn enforce_all_rungs_error_returns_502() {
3764 let (state, mut rx) = enforce_state(
3765 &["anthropic/claude-haiku-4-5"],
3766 &["non-empty"],
3767 vec![(
3768 "anthropic/claude-haiku-4-5",
3769 Err(ProviderError::Transport("down".into())),
3770 )],
3771 );
3772 let resp = messages(
3773 State(state),
3774 Extension(TenantId("default".to_owned())),
3775 HeaderMap::new(),
3776 user_body(),
3777 )
3778 .await;
3779 assert_eq!(resp.status(), axum::http::StatusCode::BAD_GATEWAY);
3780 assert!(rx.try_recv().is_ok());
3782 }
3783
3784 #[tokio::test]
3785 async fn no_routing_config_falls_through_to_observe_not_enforce() {
3786 let config = ProxyConfig::from_lookup(|k| match k {
3790 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
3791 _ => None,
3792 })
3793 .unwrap();
3794 let (traces, _rx) = mpsc::channel(64);
3795 let state = AppState {
3796 config: Arc::new(config),
3797 http: reqwest::Client::new(),
3798 providers: ProviderRegistry::new("http://127.0.0.1:1", "http://127.0.0.1:1"),
3799 gate_health: Arc::new(GateHealthRegistry::new()),
3800 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
3801 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
3802 traces,
3803 adaptive: None,
3804 bandit: None,
3805 promoter: None,
3806 verified_cache: None,
3807 predictor: None,
3808 tenant_rate_limiter: None,
3809 spill: None,
3810 };
3811 let resp = messages(
3812 State(state),
3813 Extension(TenantId("default".to_owned())),
3814 HeaderMap::new(),
3815 user_body(),
3816 )
3817 .await;
3818 assert_ne!(resp.status(), axum::http::StatusCode::OK);
3820 }
3821
3822 fn bare_enforce_route() -> Route {
3825 use firstpass_core::config::{Match, Mode};
3826 Route {
3827 match_: Match::default(),
3828 mode: Mode::Enforce,
3829 ladder: vec!["anthropic/claude-haiku-4-5".to_owned()],
3830 gates: vec![],
3831 deferred_gates: vec![],
3832 routing_mode: None,
3833 rollout: None,
3834 shadow: None,
3835 }
3836 }
3837
3838 #[test]
3839 fn balanced_preset_application_is_noop() {
3840 let base_max_rungs = 3u32;
3842 let base_speculation = 2u32;
3843 let preset = RoutingMode::Balanced.preset();
3844 let max_rungs = if let Some(d) = preset.max_rungs_delta {
3845 (base_max_rungs as i32 + d).max(1) as u32
3846 } else {
3847 base_max_rungs
3848 };
3849 let speculation = preset.speculation.unwrap_or(base_speculation);
3850 let start_at_top = preset.start_at_top;
3851 assert_eq!(max_rungs, 3, "Balanced must not change max_rungs");
3852 assert_eq!(speculation, 2, "Balanced must not change speculation");
3853 assert!(!start_at_top, "Balanced must not set start_at_top");
3854 }
3855
3856 #[test]
3857 fn resolve_mode_header_wins_over_route_and_global() {
3858 let mut headers = HeaderMap::new();
3859 headers.insert("x-firstpass-mode", "cost".parse().unwrap());
3860 let mut route = bare_enforce_route();
3861 route.routing_mode = Some(RoutingMode::Quality); let mut config = test_config();
3863 config.default_routing_mode = RoutingMode::Max; assert_eq!(
3865 resolve_mode(&headers, &route, &config),
3866 RoutingMode::Cost,
3867 "header must win"
3868 );
3869 }
3870
3871 #[test]
3872 fn resolve_mode_route_wins_over_global_when_no_header() {
3873 let mut route = bare_enforce_route();
3874 route.routing_mode = Some(RoutingMode::Latency);
3875 let mut config = test_config();
3876 config.default_routing_mode = RoutingMode::Max;
3877 assert_eq!(
3878 resolve_mode(&HeaderMap::new(), &route, &config),
3879 RoutingMode::Latency,
3880 "route must beat global default"
3881 );
3882 }
3883
3884 #[test]
3885 fn resolve_mode_global_when_header_and_route_absent() {
3886 let mut config = test_config();
3887 config.default_routing_mode = RoutingMode::Quality;
3888 assert_eq!(
3889 resolve_mode(&HeaderMap::new(), &bare_enforce_route(), &config),
3890 RoutingMode::Quality
3891 );
3892 }
3893
3894 #[test]
3895 fn resolve_mode_unknown_header_falls_through_to_route() {
3896 let mut headers = HeaderMap::new();
3897 headers.insert("x-firstpass-mode", "turbo-mode".parse().unwrap());
3899 let mut route = bare_enforce_route();
3900 route.routing_mode = Some(RoutingMode::Cost);
3901 let config = test_config();
3902 assert_eq!(
3903 resolve_mode(&headers, &route, &config),
3904 RoutingMode::Cost,
3905 "unknown header value must fall through to route"
3906 );
3907 }
3908
3909 #[test]
3910 fn resolve_mode_header_case_insensitive() {
3911 let mut headers = HeaderMap::new();
3912 headers.insert("x-firstpass-mode", "QUALITY".parse().unwrap());
3913 assert_eq!(
3914 resolve_mode(&headers, &bare_enforce_route(), &test_config()),
3915 RoutingMode::Quality
3916 );
3917 }
3918
3919 #[test]
3920 fn resolve_mode_no_mode_set_returns_balanced() {
3921 assert_eq!(
3923 resolve_mode(&HeaderMap::new(), &bare_enforce_route(), &test_config()),
3924 RoutingMode::Balanced
3925 );
3926 }
3927
3928 #[test]
3929 fn capabilities_json_includes_routing_modes() {
3930 let modes: Vec<&'static str> = RoutingMode::ALL.iter().map(|m| m.as_str()).collect();
3931 assert!(modes.contains(&"balanced"));
3932 assert!(modes.contains(&"cost"));
3933 assert!(modes.contains(&"quality"));
3934 assert!(modes.contains(&"latency"));
3935 assert!(modes.contains(&"max"));
3936 assert!(modes.contains(&"observe"));
3937 }
3938
3939 #[test]
3940 fn model_list_dedups_across_ladders_keeping_cheapest_first_order() {
3941 let config = firstpass_core::config::Config::parse(
3944 r#"
3945 [[route]]
3946 mode = "enforce"
3947 ladder = ["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"]
3948 [[route]]
3949 mode = "enforce"
3950 ladder = ["anthropic/claude-sonnet-5", "anthropic/claude-opus-4-8"]
3951 "#,
3952 )
3953 .expect("test config parses");
3954
3955 let out = model_list(Some(&config));
3956
3957 let ids: Vec<&str> = out["data"]
3958 .as_array()
3959 .expect("data is an array")
3960 .iter()
3961 .map(|m| m["id"].as_str().expect("id is a string"))
3962 .collect();
3963 assert_eq!(
3964 ids,
3965 [
3966 "anthropic/claude-haiku-4-5",
3967 "anthropic/claude-sonnet-5",
3968 "anthropic/claude-opus-4-8",
3969 ],
3970 "sonnet appears once, and ladder order is preserved"
3971 );
3972 assert_eq!(out["object"], "list");
3973 assert_eq!(out["data"][0]["owned_by"], "anthropic");
3974 let haiku_in = out["data"][0]["firstpass"]["input_per_mtok"]
3977 .as_f64()
3978 .expect("a first-party rung has a price");
3979 assert!(haiku_in > 0.0, "expected a real price, got {haiku_in}");
3980 let opus_in = out["data"][2]["firstpass"]["input_per_mtok"]
3982 .as_f64()
3983 .expect("a first-party rung has a price");
3984 assert!(haiku_in < opus_in, "{haiku_in} should undercut {opus_in}");
3985 }
3986
3987 #[tokio::test]
3988 async fn healthz_names_the_service_so_a_client_can_tell_us_from_a_stranger() {
3989 let body = healthz().await.into_response();
3993 let bytes = axum::body::to_bytes(body.into_body(), 64 * 1024)
3994 .await
3995 .expect("healthz body");
3996 let json: serde_json::Value = serde_json::from_slice(&bytes).expect("healthz is json");
3997 assert_eq!(json["status"], "ok");
3998 assert_eq!(json["service"], "firstpass");
3999 }
4000
4001 #[test]
4002 fn model_list_is_empty_without_routing_config() {
4003 let out = model_list(None);
4004 assert_eq!(out["object"], "list");
4005 assert_eq!(
4006 out["data"].as_array().expect("data is an array").len(),
4007 0,
4008 "an observe-only deployment advertises no models"
4009 );
4010 }
4011
4012 #[test]
4013 fn detects_stream_requests() {
4014 assert!(is_stream_request(br#"{"stream": true}"#));
4015 assert!(!is_stream_request(br#"{"stream": false}"#));
4016 assert!(!is_stream_request(br#"{"model":"m"}"#));
4017 assert!(!is_stream_request(b"not json"));
4018 }
4019
4020 #[test]
4021 fn detects_tool_blocks_in_messages() {
4022 let with =
4023 br#"{"messages":[{"role":"user","content":[{"type":"tool_result","content":"42"}]}]}"#;
4024 let without = br#"{"messages":[{"role":"user","content":"hi"}]}"#;
4025 assert!(messages_have_tool_blocks(with));
4026 assert!(!messages_have_tool_blocks(without));
4027 }
4028
4029 #[test]
4030 fn enforce_only_handles_plain_text() {
4031 let plain =
4032 Bytes::from_static(br#"{"model":"m","messages":[{"role":"user","content":"hi"}]}"#);
4033 let tools = Bytes::from_static(
4034 br#"{"model":"m","tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
4035 );
4036 let f_plain = extract_features(&HeaderMap::new(), &plain);
4037 let f_tools = extract_features(&HeaderMap::new(), &tools);
4038 let anthropic_ladder = vec!["anthropic/claude-haiku-4-5".to_owned()];
4039 let providers = test_registry();
4040 assert!(enforce_can_handle(
4042 &f_plain,
4043 &plain,
4044 false,
4045 &anthropic_ladder,
4046 &providers,
4047 Dialect::Anthropic,
4048 ));
4049 assert!(!enforce_can_handle(
4050 &f_tools,
4051 &tools,
4052 false,
4053 &anthropic_ladder,
4054 &providers,
4055 Dialect::Anthropic,
4056 ));
4057 }
4058
4059 #[test]
4060 fn structured_enforce_routes_tools_and_streaming() {
4061 let tools = Bytes::from_static(
4064 br#"{"model":"m","tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
4065 );
4066 let streaming_tools = Bytes::from_static(
4067 br#"{"model":"m","stream":true,"tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
4068 );
4069 let f = extract_features(&HeaderMap::new(), &tools);
4070 let anthropic_ladder = vec![
4071 "anthropic/claude-haiku-4-5".to_owned(),
4072 "anthropic/claude-sonnet-5".to_owned(),
4073 ];
4074 let providers = test_registry();
4075 assert!(enforce_can_handle(
4076 &f,
4077 &tools,
4078 true,
4079 &anthropic_ladder,
4080 &providers,
4081 Dialect::Anthropic,
4082 ));
4083 assert!(enforce_can_handle(
4084 &f,
4085 &streaming_tools,
4086 true,
4087 &anthropic_ladder,
4088 &providers,
4089 Dialect::Anthropic,
4090 ));
4091 }
4092
4093 fn test_registry() -> crate::provider::ProviderRegistry {
4095 crate::provider::ProviderRegistry::new("http://localhost", "http://localhost")
4096 }
4097
4098 #[test]
4099 fn fidelity_guard_blocks_structured_on_non_verbatim_ladder() {
4100 let tools = Bytes::from_static(
4104 br#"{"model":"m","tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
4105 );
4106 let plain =
4107 Bytes::from_static(br#"{"model":"m","messages":[{"role":"user","content":"hi"}]}"#);
4108 let f_tools = extract_features(&HeaderMap::new(), &tools);
4109 let f_plain = extract_features(&HeaderMap::new(), &plain);
4110 let providers = test_registry();
4111 let mixed_ladder = vec![
4112 "openai/gpt-4.1-mini".to_owned(),
4113 "anthropic/claude-sonnet-5".to_owned(),
4114 ];
4115 assert!(!enforce_can_handle(
4116 &f_tools,
4117 &tools,
4118 true,
4119 &mixed_ladder,
4120 &providers,
4121 Dialect::Anthropic,
4122 ));
4123 assert!(enforce_can_handle(
4124 &f_plain,
4125 &plain,
4126 true,
4127 &mixed_ladder,
4128 &providers,
4129 Dialect::Anthropic,
4130 ));
4131 }
4132
4133 #[test]
4134 fn enforce_sse_reemission_preserves_text_and_tool_use() {
4135 let resp = ModelResponse {
4138 model: "anthropic/claude-haiku-4-5".to_owned(),
4139 text: "let me check".to_owned(),
4140 in_tokens: 5,
4141 cache_write_tokens: 0,
4142 cache_read_tokens: 0,
4143 out_tokens: 7,
4144 raw: serde_json::json!({
4145 "content": [
4146 { "type": "text", "text": "let me check" },
4147 { "type": "tool_use", "id": "tu_1", "name": "get_weather", "input": { "city": "Paris" } }
4148 ]
4149 }),
4150 };
4151 let sse = anthropic_sse_from_message(&anthropic_response_json(&resp));
4152
4153 let frames: Vec<Value> = sse
4155 .lines()
4156 .filter_map(|l| l.strip_prefix("data: "))
4157 .map(|d| serde_json::from_str::<Value>(d).expect("each SSE data frame is valid JSON"))
4158 .collect();
4159
4160 assert_eq!(frames.first().unwrap()["type"], "message_start");
4162 assert_eq!(frames.last().unwrap()["type"], "message_stop");
4163 assert!(frames.iter().any(|f| f["delta"]["type"] == "text_delta"
4165 && f["delta"]["text"] == "let me check"));
4166 assert!(
4169 frames
4170 .iter()
4171 .any(|f| f["content_block"]["type"] == "tool_use"
4172 && f["content_block"]["name"] == "get_weather"
4173 && f["content_block"]["id"] == "tu_1")
4174 );
4175 assert!(
4176 frames
4177 .iter()
4178 .any(|f| f["delta"]["type"] == "input_json_delta"
4179 && f["delta"]["partial_json"] == r#"{"city":"Paris"}"#)
4180 );
4181 }
4182
4183 #[tokio::test]
4188 async fn enforce_falls_back_to_observe_for_tool_requests() {
4189 let toml = "[[price]]\nmodel = \"anthropic/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/m\"]\ngates = [\"non-empty\"]\n";
4190 let config = ProxyConfig::from_lookup(|k| match k {
4191 "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
4192 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
4193 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
4194 _ => None,
4195 })
4196 .unwrap();
4197 let mut outs = HashMap::new();
4198 outs.insert(
4199 "anthropic/m".to_owned(),
4200 Ok(model_resp("anthropic/m", "hello")),
4201 );
4202 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
4203 map.insert(
4204 "anthropic".to_owned(),
4205 Arc::new(MockProvider::new("anthropic", outs)),
4206 );
4207 let (traces, _rx) = mpsc::channel(64);
4208 let state = AppState {
4209 config: Arc::new(config),
4210 http: reqwest::Client::new(),
4211 providers: ProviderRegistry::from_map(map),
4212 gate_health: Arc::new(GateHealthRegistry::new()),
4213 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
4214 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
4215 traces,
4216 adaptive: None,
4217 bandit: None,
4218 promoter: None,
4219 verified_cache: None,
4220 predictor: None,
4221 tenant_rate_limiter: None,
4222 spill: None,
4223 };
4224
4225 let plain =
4227 Bytes::from_static(br#"{"model":"m","messages":[{"role":"user","content":"hi"}]}"#);
4228 let resp = messages(
4229 State(state.clone()),
4230 Extension(TenantId("default".to_owned())),
4231 HeaderMap::new(),
4232 plain,
4233 )
4234 .await;
4235 assert_eq!(
4236 resp.status(),
4237 axum::http::StatusCode::OK,
4238 "plain text should enforce"
4239 );
4240
4241 let tools = Bytes::from_static(
4244 br#"{"model":"m","tools":[{"name":"get_weather"}],"messages":[{"role":"user","content":"hi"}]}"#,
4245 );
4246 let resp = messages(
4247 State(state.clone()),
4248 Extension(TenantId("default".to_owned())),
4249 HeaderMap::new(),
4250 tools.clone(),
4251 )
4252 .await;
4253 assert_eq!(
4254 resp.status(),
4255 axum::http::StatusCode::OK,
4256 "tool request must route through enforce by default (ADR 0005 default-on)"
4257 );
4258
4259 let toml_off = "[[price]]\nmodel = \"anthropic/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/m\"]\ngates = [\"non-empty\"]\n[escalation]\nenforce_structured = false\n";
4262 let config_off = ProxyConfig::from_lookup(|k| match k {
4263 "FIRSTPASS_CONFIG_TOML" => Some(toml_off.to_owned()),
4264 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
4265 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
4266 _ => None,
4267 })
4268 .unwrap();
4269 let state_off = AppState {
4270 config: Arc::new(config_off),
4271 ..state.clone()
4272 };
4273 let resp = messages(
4274 State(state_off),
4275 Extension(TenantId("default".to_owned())),
4276 HeaderMap::new(),
4277 tools,
4278 )
4279 .await;
4280 assert_ne!(
4281 resp.status(),
4282 axum::http::StatusCode::OK,
4283 "with enforce_structured = false a tool request must fall back to observe"
4284 );
4285
4286 let toolres = Bytes::from_static(
4288 br#"{"model":"m","messages":[{"role":"user","content":[{"type":"tool_result","tool_use_id":"x","content":"42"}]}]}"#,
4289 );
4290 let resp = messages(
4291 State(state),
4292 Extension(TenantId("default".to_owned())),
4293 HeaderMap::new(),
4294 toolres,
4295 )
4296 .await;
4297 assert_eq!(
4298 resp.status(),
4299 axum::http::StatusCode::OK,
4300 "tool_result blocks route through enforce by default too (verbatim carry)"
4301 );
4302 }
4303
4304 #[tokio::test]
4308 async fn panel_is_public_markup_but_receipts_stay_behind_auth() {
4309 use tower::ServiceExt;
4310 let (state, _rx) = enforce_state(
4311 &["anthropic/claude-haiku-4-5"],
4312 &["non-empty"],
4313 vec![(
4314 "anthropic/claude-haiku-4-5",
4315 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
4316 )],
4317 );
4318 let router = app(state).expect("prometheus recorder installs");
4319
4320 let page = router
4321 .clone()
4322 .oneshot(
4323 axum::http::Request::builder()
4324 .uri("/panel")
4325 .body(Body::empty())
4326 .unwrap(),
4327 )
4328 .await
4329 .unwrap();
4330 assert_eq!(page.status(), axum::http::StatusCode::OK);
4331 let html = axum::body::to_bytes(page.into_body(), 1 << 20)
4332 .await
4333 .unwrap();
4334 let html = String::from_utf8_lossy(&html);
4335 assert!(
4336 html.contains("firstpass") && html.contains("/v1/receipts"),
4337 "the page must fetch its data rather than embed it"
4338 );
4339 assert!(
4342 !html.contains("http://") && !html.contains("https://"),
4343 "the panel must not reference any external origin"
4344 );
4345 assert!(html.contains("const esc ="), "the escaper must exist");
4350 for raw in [
4351 "${a.model}",
4352 "${a.rung}",
4353 "${a.verdict}",
4354 "${r.trace_id}",
4355 "${r.served_rung}",
4356 ] {
4357 assert!(
4358 !html.contains(raw),
4359 "receipt field interpolated unescaped into innerHTML: {raw}"
4360 );
4361 }
4362
4363 let data = router
4364 .oneshot(
4365 axum::http::Request::builder()
4366 .uri("/v1/receipts?limit=5")
4367 .body(Body::empty())
4368 .unwrap(),
4369 )
4370 .await
4371 .unwrap();
4372 assert_eq!(data.status(), axum::http::StatusCode::OK);
4376 let body = axum::body::to_bytes(data.into_body(), 1 << 20)
4377 .await
4378 .unwrap();
4379 let v: Value = serde_json::from_slice(&body).unwrap();
4380 assert!(v.get("receipts").and_then(Value::as_array).is_some());
4381 }
4382
4383 async fn feedback_state() -> (AppState, std::path::PathBuf, String) {
4387 let db = std::env::temp_dir().join(format!("firstpass-feedback-{}.db", Uuid::now_v7()));
4388 let (tx, handle) = crate::store::open(&db).unwrap();
4389
4390 let mut trace = build_error_trace(
4391 &ProxyConfig::from_lookup(|_| None).unwrap(),
4392 &Bytes::from_static(b"{}"),
4393 5,
4394 Some("sess-fb"),
4395 );
4396 trace.attempts.push(Attempt {
4397 rung: 0,
4398 model: "anthropic/claude-haiku-4-5".into(),
4399 provider: "anthropic".into(),
4400 in_tokens: 10,
4401 cache_write_tokens: 0,
4402 cache_read_tokens: 0,
4403 out_tokens: 5,
4404 cost_usd: 0.001,
4405 latency_ms: 5,
4406 gates: vec![],
4407 verdict: Verdict::Pass,
4408 });
4409 let trace_id = trace.trace_id.to_string();
4410 tx.try_send(trace).unwrap();
4411 drop(tx);
4412 handle.await.unwrap();
4413
4414 let db_str = db.to_string_lossy().into_owned();
4415 let config = ProxyConfig::from_lookup(move |k| match k {
4416 "FIRSTPASS_DB" => Some(db_str.clone()),
4417 _ => None,
4418 })
4419 .unwrap();
4420 let (traces, _rx) = mpsc::channel(64);
4421 let state = AppState {
4422 config: Arc::new(config),
4423 http: reqwest::Client::new(),
4424 providers: ProviderRegistry::new("http://127.0.0.1:1", "http://127.0.0.1:1"),
4425 gate_health: Arc::new(GateHealthRegistry::new()),
4426 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
4427 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
4428 traces,
4429 adaptive: None,
4430 bandit: None,
4431 promoter: None,
4432 verified_cache: None,
4433 predictor: None,
4434 tenant_rate_limiter: None,
4435 spill: None,
4436 };
4437 (state, db, trace_id)
4438 }
4439
4440 #[tokio::test]
4441 async fn feedback_nudges_the_adaptive_threshold() {
4442 use firstpass_core::conformal::AdaptiveConformal;
4443 let (mut state, _db, trace_id) = feedback_state().await;
4444 let aci = Arc::new(std::sync::Mutex::new(AdaptiveConformal::new(0.1, 0.2, 0.5)));
4445 state.adaptive = Some(aci.clone());
4446 let before = aci.lock().unwrap().threshold();
4447
4448 let fail = Bytes::from(
4450 serde_json::json!({ "trace_id": trace_id, "gate_id": "tests", "verdict": "fail", "reporter": "ci" })
4451 .to_string(),
4452 );
4453 assert_eq!(
4454 feedback(
4455 State(state.clone()),
4456 Extension(TenantId("default".to_owned())),
4457 fail
4458 )
4459 .await
4460 .status(),
4461 axum::http::StatusCode::ACCEPTED
4462 );
4463 let after_fail = aci.lock().unwrap().threshold();
4464 assert!(
4465 after_fail > before,
4466 "served fail should raise the live threshold: {before} -> {after_fail}"
4467 );
4468
4469 let pass = Bytes::from(
4471 serde_json::json!({ "trace_id": trace_id, "gate_id": "tests", "verdict": "pass", "reporter": "ci" })
4472 .to_string(),
4473 );
4474 let _ = feedback(
4475 State(state),
4476 Extension(TenantId("default".to_owned())),
4477 pass,
4478 )
4479 .await;
4480 assert!(aci.lock().unwrap().threshold() < after_fail);
4481 }
4482
4483 #[tokio::test]
4484 async fn feedback_records_a_deferred_verdict_without_breaking_the_chain() {
4485 let (state, db, trace_id) = feedback_state().await;
4486 let body = Bytes::from(
4487 serde_json::json!({
4488 "trace_id": trace_id,
4489 "gate_id": "tests",
4490 "verdict": "pass",
4491 "score": 1.0,
4492 "reporter": "ci",
4493 })
4494 .to_string(),
4495 );
4496 let resp = feedback(
4497 State(state),
4498 Extension(TenantId("default".to_owned())),
4499 body,
4500 )
4501 .await;
4502 assert_eq!(resp.status(), axum::http::StatusCode::ACCEPTED);
4503
4504 let view = crate::store::load_trace_view(&db, "default", &trace_id)
4506 .unwrap()
4507 .unwrap();
4508 assert_eq!(view.deferred.len(), 1);
4509 assert_eq!(view.deferred[0].gate_id, "tests");
4510 let traces = crate::store::load_all_traces(&db).unwrap();
4512 firstpass_core::verify_chain(&traces, GENESIS_HASH).unwrap();
4513
4514 let _ = std::fs::remove_file(&db);
4515 }
4516
4517 #[tokio::test]
4518 async fn feedback_for_unknown_trace_is_404() {
4519 let (state, db, _trace_id) = feedback_state().await;
4520 let body = Bytes::from(
4521 serde_json::json!({
4522 "trace_id": "does-not-exist",
4523 "gate_id": "tests",
4524 "verdict": "pass",
4525 "reporter": "ci",
4526 })
4527 .to_string(),
4528 );
4529 let resp = feedback(
4530 State(state),
4531 Extension(TenantId("default".to_owned())),
4532 body,
4533 )
4534 .await;
4535 assert_eq!(resp.status(), axum::http::StatusCode::NOT_FOUND);
4536 let _ = std::fs::remove_file(&db);
4537 }
4538
4539 #[tokio::test]
4542 async fn feedback_across_tenants_is_404_not_403() {
4543 let (state, db, trace_id) = feedback_state().await;
4544 let body = Bytes::from(
4545 serde_json::json!({
4546 "trace_id": trace_id,
4547 "gate_id": "tests",
4548 "verdict": "pass",
4549 "score": 1.0,
4550 "reporter": "attacker",
4551 })
4552 .to_string(),
4553 );
4554 let resp = feedback(
4556 State(state),
4557 Extension(TenantId("tenant-b".to_owned())),
4558 body,
4559 )
4560 .await;
4561 assert_eq!(
4562 resp.status(),
4563 axum::http::StatusCode::NOT_FOUND,
4564 "cross-tenant feedback must look exactly like a missing trace"
4565 );
4566 let _ = std::fs::remove_file(&db);
4567 }
4568
4569 #[tokio::test]
4570 async fn feedback_rejects_bad_verdict_and_score() {
4571 let (state, db, trace_id) = feedback_state().await;
4572 let bad_verdict = Bytes::from(
4573 serde_json::json!({ "trace_id": trace_id, "gate_id": "g", "verdict": "maybe", "reporter": "x" })
4574 .to_string(),
4575 );
4576 assert_eq!(
4577 feedback(
4578 State(state.clone()),
4579 Extension(TenantId("default".to_owned())),
4580 bad_verdict
4581 )
4582 .await
4583 .status(),
4584 axum::http::StatusCode::BAD_REQUEST
4585 );
4586 let bad_score = Bytes::from(
4587 serde_json::json!({ "trace_id": trace_id, "gate_id": "g", "verdict": "pass", "score": 9.0, "reporter": "x" })
4588 .to_string(),
4589 );
4590 assert_eq!(
4591 feedback(
4592 State(state),
4593 Extension(TenantId("default".to_owned())),
4594 bad_score
4595 )
4596 .await
4597 .status(),
4598 axum::http::StatusCode::BAD_REQUEST
4599 );
4600 let _ = std::fs::remove_file(&db);
4601 }
4602
4603 #[tokio::test]
4604 async fn metrics_endpoint_renders_after_a_real_request() {
4605 use tower::ServiceExt;
4606
4607 let (state, mut rx) = enforce_state(
4608 &["anthropic/claude-haiku-4-5"],
4609 &["non-empty"],
4610 vec![(
4611 "anthropic/claude-haiku-4-5",
4612 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
4613 )],
4614 );
4615 let router = app(state).expect("prometheus recorder installs");
4616
4617 let req = axum::http::Request::builder()
4618 .method("POST")
4619 .uri("/v1/messages")
4620 .header("content-type", "application/json")
4621 .body(Body::from(user_body()))
4622 .unwrap();
4623 let resp = router.clone().oneshot(req).await.unwrap();
4624 assert_eq!(resp.status(), axum::http::StatusCode::OK);
4625 rx.try_recv().expect("a trace was enqueued");
4626
4627 let metrics_req = axum::http::Request::builder()
4628 .method("GET")
4629 .uri("/metrics")
4630 .body(Body::empty())
4631 .unwrap();
4632 let metrics_resp = router.oneshot(metrics_req).await.unwrap();
4633 assert_eq!(metrics_resp.status(), axum::http::StatusCode::OK);
4634 let bytes = axum::body::to_bytes(metrics_resp.into_body(), 1 << 20)
4635 .await
4636 .unwrap();
4637 let body = String::from_utf8(bytes.to_vec()).unwrap();
4638 assert!(
4639 body.contains("firstpass_enforce_latency_ms"),
4640 "metrics body missing enforce latency histogram: {body}"
4641 );
4642 assert!(
4643 body.contains("firstpass_served_total"),
4644 "metrics body missing served counter: {body}"
4645 );
4646 }
4647
4648 fn auth_state(require_auth: bool, keys_json: Option<String>) -> AppState {
4652 auth_state_rated(require_auth, keys_json, None)
4653 }
4654
4655 fn auth_state_rated(
4658 require_auth: bool,
4659 keys_json: Option<String>,
4660 rate_per_sec: Option<u32>,
4661 ) -> AppState {
4662 let config = ProxyConfig::from_lookup(|k| match k {
4663 "FIRSTPASS_REQUIRE_AUTH" => require_auth.then(|| "true".to_owned()),
4664 "FIRSTPASS_TENANT_KEYS_JSON" => keys_json.clone(),
4665 "FIRSTPASS_TENANT_RATE_PER_SEC" => rate_per_sec.map(|n| n.to_string()),
4666 _ => None,
4667 })
4668 .unwrap();
4669 let (traces, _rx) = mpsc::channel(64);
4670 std::mem::forget(_rx);
4673 let providers: HashMap<String, Arc<dyn Provider>> = HashMap::new();
4674 let tenant_rate_limiter = build_tenant_rate_limiter(&config);
4675 AppState {
4676 config: Arc::new(config),
4677 http: reqwest::Client::new(),
4678 providers: ProviderRegistry::from_map(providers),
4679 gate_health: Arc::new(GateHealthRegistry::new()),
4680 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
4681 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
4682 traces,
4683 adaptive: None,
4684 bandit: None,
4685 promoter: None,
4686 verified_cache: None,
4687 predictor: None,
4688 tenant_rate_limiter,
4689 spill: None,
4690 }
4691 }
4692
4693 fn cap_request(auth_header: Option<&str>) -> axum::http::Request<Body> {
4694 let mut b = axum::http::Request::builder()
4695 .method("GET")
4696 .uri("/v1/capabilities");
4697 if let Some(h) = auth_header {
4698 b = b.header("authorization", h);
4699 }
4700 b.body(Body::empty()).unwrap()
4701 }
4702
4703 #[tokio::test]
4704 async fn auth_off_allows_unauthenticated_request() {
4705 use tower::ServiceExt;
4706 let router = app(auth_state(false, None)).expect("router");
4707 let resp = router.oneshot(cap_request(None)).await.unwrap();
4708 assert_eq!(resp.status(), axum::http::StatusCode::OK);
4710 }
4711
4712 #[tokio::test]
4713 async fn auth_on_missing_key_is_401_opaque() {
4714 use tower::ServiceExt;
4715 let hash = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
4716 let keys = format!("{{\"tenant-a\": {hash:?}}}");
4717 let router = app(auth_state(true, Some(keys))).expect("router");
4718
4719 let resp = router.oneshot(cap_request(None)).await.unwrap();
4720 assert_eq!(resp.status(), axum::http::StatusCode::UNAUTHORIZED);
4721 let json = body_json(resp).await;
4722 assert_eq!(json["error"]["type"], "unauthorized");
4723 let msg = json["error"]["message"].as_str().unwrap();
4725 assert!(!msg.contains("tenant"), "no tenant oracle in body: {msg}");
4726 }
4727
4728 #[tokio::test]
4729 async fn auth_on_invalid_key_is_401() {
4730 use tower::ServiceExt;
4731 let hash = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
4732 let keys = format!("{{\"tenant-a\": {hash:?}}}");
4733 let router = app(auth_state(true, Some(keys))).expect("router");
4734
4735 let resp = router
4736 .oneshot(cap_request(Some("Bearer wrong-key")))
4737 .await
4738 .unwrap();
4739 assert_eq!(resp.status(), axum::http::StatusCode::UNAUTHORIZED);
4740 }
4741
4742 #[tokio::test]
4743 async fn auth_on_valid_key_proceeds() {
4744 use tower::ServiceExt;
4745 let hash = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
4746 let keys = format!("{{\"tenant-a\": {hash:?}}}");
4747 let router = app(auth_state(true, Some(keys))).expect("router");
4748
4749 let resp = router
4751 .oneshot(cap_request(Some("Bearer tenant-a.key-a")))
4752 .await
4753 .unwrap();
4754 assert_eq!(resp.status(), axum::http::StatusCode::OK);
4756 }
4757
4758 fn two_tenant_state(rate_per_sec: Option<u32>) -> AppState {
4760 let hash_a = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
4761 let hash_b = crate::tenant_auth::TenantKeys::hash_key("key-b").unwrap();
4762 let keys = format!("{{\"tenant-a\": {hash_a:?}, \"tenant-b\": {hash_b:?}}}");
4763 auth_state_rated(true, Some(keys), rate_per_sec)
4764 }
4765
4766 #[tokio::test]
4767 async fn tenant_exceeding_rate_limit_gets_429_opaque() {
4768 use tower::ServiceExt;
4769 let router = app(two_tenant_state(Some(1))).expect("router");
4775
4776 let (r1, r2, r3, r4) = tokio::join!(
4777 router
4778 .clone()
4779 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
4780 router
4781 .clone()
4782 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
4783 router
4784 .clone()
4785 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
4786 router
4787 .clone()
4788 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
4789 );
4790 let responses = [r1.unwrap(), r2.unwrap(), r3.unwrap(), r4.unwrap()];
4791 let ok = responses
4792 .iter()
4793 .filter(|r| r.status() == axum::http::StatusCode::OK)
4794 .count();
4795 assert!(ok >= 1, "the burst's first request must pass");
4796 let limited: Vec<_> = responses
4797 .into_iter()
4798 .filter(|r| r.status() == axum::http::StatusCode::TOO_MANY_REQUESTS)
4799 .collect();
4800 assert!(
4801 !limited.is_empty(),
4802 "a 4-request burst against 1 req/sec must trip the limiter"
4803 );
4804
4805 let json = body_json(limited.into_iter().next().unwrap()).await;
4806 assert_eq!(json["error"]["type"], "rate_limited");
4807 let msg = json["error"]["message"].as_str().unwrap();
4809 assert!(!msg.contains('1'), "no limit value in body: {msg}");
4810 }
4811
4812 #[tokio::test]
4813 async fn rate_limit_buckets_are_independent_per_tenant() {
4814 use tower::ServiceExt;
4815 let router = app(two_tenant_state(Some(1))).expect("router");
4816
4817 let (a1, a2, a3) = tokio::join!(
4820 router
4821 .clone()
4822 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
4823 router
4824 .clone()
4825 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
4826 router
4827 .clone()
4828 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
4829 );
4830 let a_limited = [a1.unwrap(), a2.unwrap(), a3.unwrap()]
4831 .iter()
4832 .filter(|r| r.status() == axum::http::StatusCode::TOO_MANY_REQUESTS)
4833 .count();
4834 assert!(a_limited >= 1, "tenant A's burst must trip its limiter");
4835
4836 let b1 = router
4838 .clone()
4839 .oneshot(cap_request(Some("Bearer tenant-b.key-b")))
4840 .await
4841 .unwrap();
4842 assert_eq!(b1.status(), axum::http::StatusCode::OK);
4843 }
4844
4845 #[tokio::test]
4846 async fn rate_limit_unset_never_429s() {
4847 use tower::ServiceExt;
4848 let router = app(two_tenant_state(None)).expect("router");
4851 for _ in 0..20 {
4852 let resp = router
4853 .clone()
4854 .oneshot(cap_request(Some("Bearer tenant-a.key-a")))
4855 .await
4856 .unwrap();
4857 assert_eq!(resp.status(), axum::http::StatusCode::OK);
4858 }
4859 }
4860
4861 #[test]
4864 fn u01_is_deterministic_and_in_range() {
4865 let s1 = u01(0xDEAD_BEEF_CAFE_1234_u128);
4866 let s2 = u01(0xDEAD_BEEF_CAFE_1234_u128);
4867 assert_eq!(s1, s2, "u01 must be deterministic for the same seed");
4868 assert!((0.0..1.0).contains(&s1), "u01 must return [0, 1), got {s1}");
4869
4870 let s3 = u01(0x1234_5678_9ABC_DEF0_u128);
4872 assert_ne!(s1, s3, "different seeds should give different values");
4873
4874 for i in 0u64..256 {
4876 let v = u01(i as u128);
4877 assert!((0.0..1.0).contains(&v), "seed {i}: u01={v} out of [0,1)");
4878 }
4879 }
4880
4881 #[test]
4882 fn epsilon_propensity_formula() {
4883 let epsilon = 0.2_f64;
4884 let k = 3_usize;
4885 let greedy = 1_u32;
4886
4887 let p_greedy = epsilon_propensity(greedy, greedy, epsilon, k);
4889 let expected_greedy = (1.0 - epsilon) + epsilon / k as f64;
4890 assert!(
4891 (p_greedy - expected_greedy).abs() < 1e-12,
4892 "{p_greedy} != {expected_greedy}"
4893 );
4894
4895 let p_other = epsilon_propensity(0, greedy, epsilon, k);
4897 let expected_other = epsilon / k as f64;
4898 assert!(
4899 (p_other - expected_other).abs() < 1e-12,
4900 "{p_other} != {expected_other}"
4901 );
4902
4903 for chosen in 0..k as u32 {
4905 let p = epsilon_propensity(chosen, greedy, epsilon, k);
4906 assert!(
4907 p > 0.0 && p <= 1.0,
4908 "propensity {p} out of (0,1] for chosen={chosen}"
4909 );
4910 }
4911 }
4912
4913 #[test]
4914 fn epsilon_branch_and_greedy_branch_both_occur_over_many_seeds() {
4915 let epsilon = 0.3_f64;
4917 let mut saw_explore = false;
4918 let mut saw_greedy = false;
4919 for i in 0u64..200 {
4920 let u = u01(i as u128);
4921 if u < epsilon {
4922 saw_explore = true;
4923 } else {
4924 saw_greedy = true;
4925 }
4926 if saw_explore && saw_greedy {
4927 break;
4928 }
4929 }
4930 assert!(
4931 saw_explore,
4932 "epsilon branch must fire with epsilon=0.3 over 200 seeds"
4933 );
4934 assert!(
4935 saw_greedy,
4936 "greedy branch must occur with epsilon=0.3 over 200 seeds"
4937 );
4938 }
4939
4940 #[test]
4941 fn epsilon_propensity_sums_to_one_over_all_rungs() {
4942 let epsilon = 0.15_f64;
4944 let k = 4_usize;
4945 let greedy = 2_u32;
4946 let total: f64 = (0..k as u32)
4947 .map(|r| epsilon_propensity(r, greedy, epsilon, k))
4948 .sum();
4949 assert!(
4950 (total - 1.0).abs() < 1e-12,
4951 "propensities must sum to 1, got {total}"
4952 );
4953 }
4954
4955 #[tokio::test(start_paused = true)]
4958 async fn keepalive_stream_ticks_then_emits_final_frame() {
4959 let (tx, rx) = tokio::sync::oneshot::channel::<Result<Value, ProxyError>>();
4960 let mut ticks = tokio::time::interval(SSE_KEEPALIVE_EVERY);
4961 ticks.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
4962 ticks.reset();
4963 let mut stream = KeepaliveStream {
4964 rx: Some(rx),
4965 ticks,
4966 format_message: anthropic_sse_from_message,
4967 };
4968 async fn next(
4970 stream: &mut KeepaliveStream,
4971 ) -> Option<Option<Result<Bytes, std::convert::Infallible>>> {
4972 std::future::poll_fn(|cx| {
4973 std::task::Poll::Ready(
4974 match futures_core::Stream::poll_next(std::pin::Pin::new(&mut *stream), cx) {
4975 std::task::Poll::Ready(item) => Some(item),
4976 std::task::Poll::Pending => None,
4977 },
4978 )
4979 })
4980 .await
4981 }
4982
4983 assert!(
4985 next(&mut stream).await.is_none(),
4986 "no frame before an interval"
4987 );
4988
4989 tokio::time::advance(SSE_KEEPALIVE_EVERY + Duration::from_millis(1)).await;
4991 let frame = next(&mut stream)
4992 .await
4993 .expect("keepalive due")
4994 .unwrap()
4995 .unwrap();
4996 assert!(
4997 frame.starts_with(b": "),
4998 "keepalive must be an SSE comment (ignored by every conforming parser)"
4999 );
5000
5001 let message = serde_json::json!({
5003 "id": "msg_1", "type": "message", "role": "assistant", "model": "m",
5004 "content": [{ "type": "text", "text": "done" }],
5005 "usage": { "input_tokens": 1, "output_tokens": 1 }
5006 });
5007 tx.send(Ok(message)).unwrap();
5008 let frame = next(&mut stream)
5009 .await
5010 .expect("final frame")
5011 .unwrap()
5012 .unwrap();
5013 let text = String::from_utf8(frame.to_vec()).unwrap();
5014 assert!(text.contains("event: message_start"));
5015 assert!(text.contains("event: message_stop"));
5016 let eos = next(&mut stream)
5017 .await
5018 .expect("stream must end after the final frame");
5019 assert!(eos.is_none(), "end-of-stream after the final frame");
5020 }
5021
5022 #[tokio::test]
5025 async fn streaming_enforce_serves_full_sse_sequence() {
5026 let toml = "[[price]]\nmodel = \"anthropic/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/m\"]\ngates = [\"non-empty\"]\n";
5027 let config = ProxyConfig::from_lookup(|k| match k {
5028 "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
5029 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
5030 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
5031 _ => None,
5032 })
5033 .unwrap();
5034 let mut outs = HashMap::new();
5035 outs.insert(
5036 "anthropic/m".to_owned(),
5037 Ok(model_resp("anthropic/m", "gated answer")),
5038 );
5039 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
5040 map.insert(
5041 "anthropic".to_owned(),
5042 Arc::new(MockProvider::new("anthropic", outs)),
5043 );
5044 let (traces, _rx) = mpsc::channel(64);
5045 let state = AppState {
5046 config: Arc::new(config),
5047 http: reqwest::Client::new(),
5048 providers: ProviderRegistry::from_map(map),
5049 gate_health: Arc::new(GateHealthRegistry::new()),
5050 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
5051 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
5052 traces,
5053 adaptive: None,
5054 bandit: None,
5055 promoter: None,
5056 verified_cache: None,
5057 predictor: None,
5058 tenant_rate_limiter: None,
5059 spill: None,
5060 };
5061 let body = Bytes::from_static(
5062 br#"{"model":"m","stream":true,"messages":[{"role":"user","content":"hi"}]}"#,
5063 );
5064 let resp = messages(
5065 State(state),
5066 Extension(TenantId("default".to_owned())),
5067 HeaderMap::new(),
5068 body,
5069 )
5070 .await;
5071 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5072 assert!(
5073 resp.headers()
5074 .get(axum::http::header::CONTENT_TYPE)
5075 .and_then(|v| v.to_str().ok())
5076 .is_some_and(|ct| ct.starts_with("text/event-stream")),
5077 "streaming client must get SSE"
5078 );
5079 let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
5080 .await
5081 .unwrap();
5082 let text = String::from_utf8(bytes.to_vec()).unwrap();
5083 assert!(text.contains("event: message_start"));
5084 assert!(text.contains("gated answer"));
5085 assert!(text.contains("event: message_stop"));
5086 }
5087
5088 #[test]
5093 fn parse_openai_request_plain_text() {
5094 let body = br#"{"model":"gpt-4o","max_tokens":256,"messages":[{"role":"user","content":"hello"}]}"#;
5096 let req = parse_openai_request(body, false).expect("must parse");
5097 assert_eq!(req.model, "gpt-4o");
5098 assert_eq!(req.max_tokens, 256);
5099 assert_eq!(req.messages.len(), 1);
5100 assert_eq!(req.messages[0].role, "user");
5101 assert_eq!(req.messages[0].content, Value::String("hello".to_owned()));
5102 assert!(req.system.is_none());
5103 assert_eq!(req.raw, Value::Null);
5105 }
5106
5107 #[test]
5108 fn parse_openai_request_system_message() {
5109 let body = br#"{"model":"gpt-4o","messages":[{"role":"system","content":"be concise"},{"role":"user","content":"hi"}]}"#;
5110 let req = parse_openai_request(body, false).expect("must parse");
5111 assert_eq!(req.system.as_deref(), Some("be concise"));
5112 assert_eq!(req.messages.len(), 1);
5113 assert_eq!(req.messages[0].role, "user");
5114 }
5115
5116 #[test]
5117 fn parse_openai_request_tool_calls_translate_to_tool_use() {
5118 let body = br#"{
5119 "model":"gpt-4o",
5120 "messages":[
5121 {"role":"user","content":"what's the weather?"},
5122 {"role":"assistant","content":null,"tool_calls":[
5123 {"id":"call_1","type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Paris\"}"}}
5124 ]},
5125 {"role":"tool","tool_call_id":"call_1","content":"15C, cloudy"}
5126 ]
5127 }"#;
5128 let req = parse_openai_request(body, false).expect("must parse");
5129 assert_eq!(req.messages.len(), 3);
5131 let asst = &req.messages[1];
5133 assert_eq!(asst.role, "assistant");
5134 let blocks = asst.content.as_array().expect("content array");
5135 assert_eq!(blocks[0]["type"], "tool_use");
5136 assert_eq!(blocks[0]["name"], "get_weather");
5137 assert_eq!(blocks[0]["id"], "call_1");
5138 assert_eq!(blocks[0]["input"]["city"], "Paris");
5139 let tool_msg = &req.messages[2];
5141 assert_eq!(tool_msg.role, "user");
5142 let result_blocks = tool_msg.content.as_array().expect("result blocks");
5143 assert_eq!(result_blocks[0]["type"], "tool_result");
5144 assert_eq!(result_blocks[0]["tool_use_id"], "call_1");
5145 }
5146
5147 #[test]
5148 fn parse_openai_request_tools_translate_to_anthropic_format() {
5149 let body = br#"{
5150 "model":"gpt-4o",
5151 "messages":[{"role":"user","content":"use a tool"}],
5152 "tools":[{"type":"function","function":{"name":"search","description":"web search","parameters":{"type":"object","properties":{"q":{"type":"string"}}}}}]
5153 }"#;
5154 let req = parse_openai_request(body, false).expect("must parse");
5155 let tools = req.tools.as_array().expect("tools array");
5156 assert_eq!(tools.len(), 1);
5157 assert_eq!(tools[0]["name"], "search");
5158 assert_eq!(tools[0]["description"], "web search");
5159 assert_eq!(tools[0]["input_schema"]["type"], "object");
5160 }
5161
5162 #[test]
5163 fn parse_openai_request_raw_carry_preserves_full_body() {
5164 let body =
5166 br#"{"model":"gpt-4o","max_tokens":100,"messages":[{"role":"user","content":"hi"}]}"#;
5167 let req = parse_openai_request(body, true).expect("must parse");
5168 assert!(req.raw.is_object(), "raw must be the full JSON object");
5169 assert_eq!(req.raw["model"], "gpt-4o");
5170 assert_eq!(req.raw["max_tokens"], 100);
5171 assert!(req.tools.is_null(), "no tools in this request");
5173 }
5174
5175 #[test]
5176 fn parse_openai_request_http_image_returns_none() {
5177 let body = br#"{"model":"gpt-4o","messages":[{"role":"user","content":[
5179 {"type":"text","text":"describe this"},
5180 {"type":"image_url","image_url":{"url":"https://example.com/cat.png"}}
5181 ]}]}"#;
5182 let result = parse_openai_request(body, false);
5183 assert!(result.is_none(), "http image URL must fail translation");
5184 }
5185
5186 #[test]
5187 fn parse_openai_request_data_url_image_translates_to_anthropic_base64() {
5188 let body = br#"{"model":"gpt-4o","messages":[{"role":"user","content":[
5189 {"type":"text","text":"describe"},
5190 {"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgo="}}
5191 ]}]}"#;
5192 let req = parse_openai_request(body, false).expect("data URL must parse");
5193 let blocks = req.messages[0].content.as_array().expect("blocks");
5194 let img = blocks
5195 .iter()
5196 .find(|b| b["type"] == "image")
5197 .expect("image block");
5198 assert_eq!(img["source"]["type"], "base64");
5199 assert_eq!(img["source"]["media_type"], "image/png");
5200 assert_eq!(img["source"]["data"], "iVBORw0KGgo=");
5201 }
5202
5203 #[test]
5206 fn openai_response_json_renders_text_response() {
5207 let resp = ModelResponse {
5208 model: "gpt-4o".to_owned(),
5209 text: "Hello!".to_owned(),
5210 in_tokens: 10,
5211 cache_write_tokens: 0,
5212 cache_read_tokens: 0,
5213 out_tokens: 5,
5214 raw: serde_json::json!({
5215 "content": [{ "type": "text", "text": "Hello!" }]
5216 }),
5217 };
5218 let json = openai_response_json(&resp);
5219 assert_eq!(json["object"], "chat.completion");
5220 assert_eq!(json["model"], "gpt-4o");
5221 assert_eq!(json["choices"][0]["message"]["role"], "assistant");
5222 assert_eq!(json["choices"][0]["message"]["content"], "Hello!");
5223 assert_eq!(json["choices"][0]["finish_reason"], "stop");
5224 assert_eq!(json["usage"]["prompt_tokens"], 10);
5225 assert_eq!(json["usage"]["completion_tokens"], 5);
5226 }
5227
5228 #[test]
5229 fn openai_response_json_renders_tool_call() {
5230 let resp = ModelResponse {
5231 model: "gpt-4o".to_owned(),
5232 text: String::new(),
5233 in_tokens: 20,
5234 cache_write_tokens: 0,
5235 cache_read_tokens: 0,
5236 out_tokens: 15,
5237 raw: serde_json::json!({
5238 "content": [{
5239 "type": "tool_use",
5240 "id": "call_abc",
5241 "name": "search",
5242 "input": {"q": "Rust async"}
5243 }]
5244 }),
5245 };
5246 let json = openai_response_json(&resp);
5247 assert_eq!(json["choices"][0]["finish_reason"], "tool_calls");
5248 let tc = &json["choices"][0]["message"]["tool_calls"][0];
5249 assert_eq!(tc["id"], "call_abc");
5250 assert_eq!(tc["type"], "function");
5251 assert_eq!(tc["function"]["name"], "search");
5252 assert_eq!(json["choices"][0]["message"]["content"], Value::Null);
5254 }
5255
5256 #[test]
5257 fn openai_sse_from_message_plain_text_has_role_then_content_then_stop() {
5258 let resp = ModelResponse {
5259 model: "gpt-4o".to_owned(),
5260 text: "Hi there!".to_owned(),
5261 in_tokens: 5,
5262 cache_write_tokens: 0,
5263 cache_read_tokens: 0,
5264 out_tokens: 3,
5265 raw: serde_json::json!({
5266 "content": [{ "type": "text", "text": "Hi there!" }]
5267 }),
5268 };
5269 let sse = openai_sse_from_message(&openai_response_json(&resp));
5270 for line in sse.lines() {
5272 assert!(
5273 line.is_empty() || line.starts_with("data: "),
5274 "bad SSE line: {line:?}"
5275 );
5276 }
5277 let frames: Vec<&str> = sse
5278 .lines()
5279 .filter_map(|l| l.strip_prefix("data: "))
5280 .collect();
5281 assert_eq!(*frames.last().unwrap(), "[DONE]");
5283 let role_frame: Value = serde_json::from_str(frames[0]).unwrap();
5285 assert_eq!(role_frame["choices"][0]["delta"]["role"], "assistant");
5286 assert!(frames.iter().any(|f| {
5288 if *f == "[DONE]" {
5289 return false;
5290 }
5291 serde_json::from_str::<Value>(f)
5292 .ok()
5293 .is_some_and(|v| v["choices"][0]["delta"]["content"] == "Hi there!")
5294 }));
5295 assert!(frames.iter().any(|f| {
5297 if *f == "[DONE]" {
5298 return false;
5299 }
5300 serde_json::from_str::<Value>(f)
5301 .ok()
5302 .is_some_and(|v| v["choices"][0]["finish_reason"] == "stop")
5303 }));
5304 }
5305
5306 #[test]
5309 fn detects_openai_tool_calls() {
5310 let with_tool_calls = Bytes::from_static(br#"{"messages":[
5311 {"role":"user","content":"hi"},
5312 {"role":"assistant","content":null,"tool_calls":[{"id":"c1","type":"function","function":{"name":"f","arguments":"{}"}}]}
5313 ]}"#);
5314 let without = Bytes::from_static(br#"{"messages":[{"role":"user","content":"hi"}]}"#);
5315 let with_tool_msg = Bytes::from_static(
5316 br#"{"messages":[
5317 {"role":"tool","tool_call_id":"c1","content":"result"}
5318 ]}"#,
5319 );
5320 assert!(openai_messages_have_tool_calls(&with_tool_calls));
5321 assert!(!openai_messages_have_tool_calls(&without));
5322 assert!(openai_messages_have_tool_calls(&with_tool_msg));
5323 }
5324
5325 #[test]
5326 fn detects_openai_http_images() {
5327 let http_img = Bytes::from_static(
5328 br#"{"messages":[{"role":"user","content":[
5329 {"type":"image_url","image_url":{"url":"https://example.com/img.png"}}
5330 ]}]}"#,
5331 );
5332 let data_img = Bytes::from_static(
5333 br#"{"messages":[{"role":"user","content":[
5334 {"type":"image_url","image_url":{"url":"data:image/png;base64,abc"}}
5335 ]}]}"#,
5336 );
5337 let no_img = Bytes::from_static(br#"{"messages":[{"role":"user","content":"hi"}]}"#);
5338 assert!(openai_has_http_images(&http_img));
5339 assert!(!openai_has_http_images(&data_img));
5340 assert!(!openai_has_http_images(&no_img));
5341 }
5342
5343 #[test]
5344 fn enforce_can_handle_openai_inbound_all_openai_ladder() {
5345 let tools_body = Bytes::from_static(br#"{"model":"gpt-4o","messages":[{"role":"assistant","content":null,"tool_calls":[{"id":"c","type":"function","function":{"name":"f","arguments":"{}"}}]}]}"#);
5347 let f = extract_openai_features(&HeaderMap::new(), &tools_body);
5348 let ladder = vec!["openai/gpt-4o-mini".to_owned(), "openai/gpt-4o".to_owned()];
5349 let providers = crate::provider::ProviderRegistry::new("http://x", "http://x");
5350 assert!(enforce_can_handle(
5351 &f,
5352 &tools_body,
5353 true,
5354 &ladder,
5355 &providers,
5356 Dialect::Openai
5357 ));
5358 }
5359
5360 #[test]
5361 fn enforce_can_handle_openai_inbound_all_anthropic_ladder_no_http_image() {
5362 let tools_body = Bytes::from_static(br#"{"model":"gpt-4o","messages":[{"role":"assistant","content":null,"tool_calls":[{"id":"c","type":"function","function":{"name":"f","arguments":"{}"}}]}]}"#);
5364 let f = extract_openai_features(&HeaderMap::new(), &tools_body);
5365 let ladder = vec!["anthropic/claude-haiku-4-5".to_owned()];
5366 let providers = crate::provider::ProviderRegistry::new("http://x", "http://x");
5367 assert!(enforce_can_handle(
5368 &f,
5369 &tools_body,
5370 true,
5371 &ladder,
5372 &providers,
5373 Dialect::Openai,
5374 ));
5375 }
5376
5377 #[test]
5378 fn enforce_can_handle_openai_inbound_http_image_falls_back() {
5379 let img_body = Bytes::from_static(
5381 br#"{"model":"gpt-4o","messages":[{"role":"user","content":[
5382 {"type":"image_url","image_url":{"url":"https://example.com/img.png"}}
5383 ]}]}"#,
5384 );
5385 let f = extract_openai_features(&HeaderMap::new(), &img_body);
5386 let ladder = vec!["anthropic/claude-haiku-4-5".to_owned()];
5387 let providers = crate::provider::ProviderRegistry::new("http://x", "http://x");
5388 assert!(!enforce_can_handle(
5389 &f,
5390 &img_body,
5391 true,
5392 &ladder,
5393 &providers,
5394 Dialect::Openai,
5395 ));
5396 }
5397
5398 fn openai_enforce_state(mock_resp: ModelResponse) -> AppState {
5402 let toml = "[[price]]\nmodel = \"mock/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"mock/m\"]\ngates = [\"non-empty\"]\n";
5403 let config = ProxyConfig::from_lookup(|k| match k {
5404 "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
5405 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
5406 _ => None,
5407 })
5408 .unwrap();
5409 let mut outs = HashMap::new();
5410 outs.insert("mock/m".to_owned(), Ok(mock_resp));
5411 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
5412 map.insert("mock".to_owned(), Arc::new(MockProvider::new("mock", outs)));
5413 let (traces, _rx) = mpsc::channel(64);
5414 std::mem::forget(_rx);
5415 let tenant_rate_limiter = build_tenant_rate_limiter(&config);
5416 AppState {
5417 config: Arc::new(config),
5418 http: reqwest::Client::new(),
5419 providers: ProviderRegistry::from_map(map),
5420 gate_health: Arc::new(GateHealthRegistry::new()),
5421 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
5422 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
5423 traces,
5424 adaptive: None,
5425 bandit: None,
5426 promoter: None,
5427 verified_cache: None,
5428 predictor: None,
5429 tenant_rate_limiter,
5430 spill: None,
5431 }
5432 }
5433
5434 #[tokio::test]
5435 async fn chat_completions_plain_text_enforce_returns_openai_shape() {
5436 let mock = model_resp("mock/m", "gated answer");
5437 let state = openai_enforce_state(mock);
5438 let body = Bytes::from_static(
5439 br#"{"model":"gpt-4o","messages":[{"role":"user","content":"hello"}]}"#,
5440 );
5441 let resp = chat_completions(
5442 State(state),
5443 Extension(TenantId("default".to_owned())),
5444 HeaderMap::new(),
5445 body,
5446 )
5447 .await;
5448 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5449 let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
5450 .await
5451 .unwrap();
5452 let json: Value = serde_json::from_slice(&bytes).expect("must be JSON");
5453 assert_eq!(json["object"], "chat.completion");
5454 assert_eq!(json["choices"][0]["message"]["role"], "assistant");
5455 assert_eq!(json["choices"][0]["message"]["content"], "gated answer");
5456 assert_eq!(json["choices"][0]["finish_reason"], "stop");
5457 assert!(
5458 json["id"]
5459 .as_str()
5460 .is_some_and(|id| id.starts_with("chatcmpl-"))
5461 );
5462 }
5463
5464 #[tokio::test]
5465 async fn chat_completions_stream_true_returns_sse_with_openai_chunks() {
5466 let mock = model_resp("mock/m", "gated answer");
5467 let state = openai_enforce_state(mock);
5468 let body = Bytes::from_static(
5469 br#"{"model":"gpt-4o","stream":true,"messages":[{"role":"user","content":"hello"}]}"#,
5470 );
5471 let resp = chat_completions(
5472 State(state),
5473 Extension(TenantId("default".to_owned())),
5474 HeaderMap::new(),
5475 body,
5476 )
5477 .await;
5478 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5479 assert!(
5480 resp.headers()
5481 .get(axum::http::header::CONTENT_TYPE)
5482 .and_then(|v| v.to_str().ok())
5483 .is_some_and(|ct| ct.starts_with("text/event-stream")),
5484 "stream:true must return SSE"
5485 );
5486 let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
5487 .await
5488 .unwrap();
5489 let text = String::from_utf8(bytes.to_vec()).unwrap();
5490 assert!(
5492 text.contains("chat.completion.chunk"),
5493 "must have OpenAI chunk frames"
5494 );
5495 assert!(text.contains("[DONE]"), "must end with [DONE]");
5496 assert!(
5497 text.contains("gated answer"),
5498 "content must be in the stream"
5499 );
5500 assert!(
5502 !text.contains("message_start"),
5503 "must not have Anthropic event types"
5504 );
5505 }
5506
5507 fn probe_state(
5511 sample_rate: f64,
5512 k: u32,
5513 outcomes: Vec<(&str, Result<ModelResponse, ProviderError>)>,
5514 ) -> (AppState, mpsc::Receiver<Trace>) {
5515 let toml = format!(
5516 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\nladder = [\"anthropic/claude-haiku-4-5\"]\ngates = [\"non-empty\"]\n\
5517 [escalation.probe]\nk = {k}\nsample_rate = {sample_rate}\n"
5518 );
5519 let config = ProxyConfig::from_lookup(|k_| match k_ {
5520 "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
5521 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
5522 _ => None,
5523 })
5524 .unwrap();
5525 let mut outs = HashMap::new();
5526 for (model, out) in outcomes {
5527 outs.insert(model.to_owned(), out);
5528 }
5529 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
5530 map.insert(
5531 "anthropic".to_owned(),
5532 Arc::new(MockProvider::new("anthropic", outs)),
5533 );
5534 let (traces, rx) = mpsc::channel(64);
5535 let state = AppState {
5536 config: Arc::new(config),
5537 http: reqwest::Client::new(),
5538 providers: ProviderRegistry::from_map(map),
5539 gate_health: Arc::new(GateHealthRegistry::new()),
5540 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
5541 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
5542 traces,
5543 adaptive: None,
5544 bandit: None,
5545 promoter: None,
5546 verified_cache: None,
5547 predictor: None,
5548 tenant_rate_limiter: None,
5549 spill: None,
5550 };
5551 (state, rx)
5552 }
5553
5554 async fn run_enforce_get_trace(state: AppState, mut rx: mpsc::Receiver<Trace>) -> Trace {
5556 let resp = messages(
5557 State(state),
5558 Extension(TenantId("default".to_owned())),
5559 HeaderMap::new(),
5560 user_body(),
5561 )
5562 .await;
5563 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5564 rx.try_recv().expect("trace must be enqueued")
5565 }
5566
5567 #[tokio::test]
5569 async fn probe_off_trace_has_no_probe_field() {
5570 let (state, rx) = enforce_state(
5571 &["anthropic/claude-haiku-4-5"],
5572 &["non-empty"],
5573 vec![(
5574 "anthropic/claude-haiku-4-5",
5575 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
5576 )],
5577 );
5578 assert!(
5579 state
5580 .config
5581 .routing
5582 .as_ref()
5583 .unwrap()
5584 .escalation
5585 .probe
5586 .is_none(),
5587 "probe must default to None"
5588 );
5589 let trace = run_enforce_get_trace(state, rx).await;
5590 assert!(
5591 trace.probe.is_none(),
5592 "probe=None config must not set trace.probe"
5593 );
5594 }
5595
5596 #[tokio::test]
5598 async fn probe_sample_rate_zero_never_fires() {
5599 let (state, rx) = probe_state(
5601 0.0,
5602 5,
5603 vec![(
5604 "anthropic/claude-haiku-4-5",
5605 Ok(model_resp("anthropic/claude-haiku-4-5", "hi")),
5606 )],
5607 );
5608 let trace = run_enforce_get_trace(state, rx).await;
5609 assert!(
5610 trace.probe.is_none(),
5611 "sample_rate=0.0 must never set trace.probe"
5612 );
5613 }
5614
5615 #[tokio::test]
5618 async fn probe_on_all_pass_sets_confident_pass() {
5619 let model = "anthropic/claude-haiku-4-5";
5621 let (state, mut rx) = probe_state(1.0, 3, vec![(model, Ok(model_resp(model, "hello")))]);
5622 let resp = messages(
5623 State(state),
5624 Extension(TenantId("default".to_owned())),
5625 HeaderMap::new(),
5626 user_body(),
5627 )
5628 .await;
5629 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5630 let json = body_json(resp).await;
5632 assert_eq!(
5633 json["content"][0]["text"], "hello",
5634 "served content unchanged"
5635 );
5636
5637 let trace = rx.try_recv().expect("trace enqueued");
5638 let sig = trace.probe.expect("probe must be set when sample_rate=1.0");
5639 assert_eq!(sig.k, 3);
5640 assert_eq!(sig.gate_pass_count, 3, "all 3 samples must pass non-empty");
5641 assert_eq!(
5642 sig.regime,
5643 firstpass_core::ProbeRegime::ConfidentPass,
5644 "all-pass → ConfidentPass"
5645 );
5646 assert!(
5647 sig.probe_cost_usd > 0.0,
5648 "k model calls must cost something"
5649 );
5650 }
5651
5652 #[tokio::test]
5655 async fn probe_on_all_fail_sets_confident_fail() {
5656 let model = "anthropic/claude-haiku-4-5";
5657 let (state, mut rx) = probe_state(1.0, 3, vec![(model, Ok(model_resp(model, "")))]);
5659 let resp = messages(
5661 State(state),
5662 Extension(TenantId("default".to_owned())),
5663 HeaderMap::new(),
5664 user_body(),
5665 )
5666 .await;
5667 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5668
5669 let trace = rx.try_recv().expect("trace enqueued");
5670 let sig = trace.probe.expect("probe must be set when sample_rate=1.0");
5671 assert_eq!(
5672 sig.gate_pass_count, 0,
5673 "empty response fails non-empty: all 0 pass"
5674 );
5675 assert_eq!(
5676 sig.regime,
5677 firstpass_core::ProbeRegime::ConfidentFail,
5678 "0 passes → ConfidentFail"
5679 );
5680 }
5681
5682 #[tokio::test]
5685 async fn probe_on_served_output_identical_to_probe_off() {
5686 let model = "anthropic/claude-haiku-4-5";
5687 let mk = |sample_rate: f64| {
5688 probe_state(
5689 sample_rate,
5690 2,
5691 vec![(model, Ok(model_resp(model, "gated answer")))],
5692 )
5693 };
5694
5695 let (state_off, mut rx_off) = mk(0.0);
5697 let resp_off = messages(
5698 State(state_off),
5699 Extension(TenantId("default".to_owned())),
5700 HeaderMap::new(),
5701 user_body(),
5702 )
5703 .await;
5704 let json_off = body_json(resp_off).await;
5705 let trace_off = rx_off.try_recv().unwrap();
5706
5707 let (state_on, mut rx_on) = mk(1.0);
5709 let resp_on = messages(
5710 State(state_on),
5711 Extension(TenantId("default".to_owned())),
5712 HeaderMap::new(),
5713 user_body(),
5714 )
5715 .await;
5716 let json_on = body_json(resp_on).await;
5717 let trace_on = rx_on.try_recv().unwrap();
5718
5719 assert_eq!(
5721 json_off["content"][0]["text"], json_on["content"][0]["text"],
5722 "served text must be identical regardless of probe"
5723 );
5724 assert!(
5726 (trace_off.final_.total_cost_usd - trace_on.final_.total_cost_usd).abs() < 1e-12,
5727 "total_cost_usd must not include probe cost: off={} on={}",
5728 trace_off.final_.total_cost_usd,
5729 trace_on.final_.total_cost_usd
5730 );
5731 assert!(trace_off.probe.is_none());
5733 assert!(trace_on.probe.is_some());
5734 assert!(
5736 trace_on.probe.as_ref().unwrap().probe_cost_usd > 0.0,
5737 "probe cost must be positive"
5738 );
5739 }
5740
5741 #[tokio::test]
5751 async fn probe_does_not_mutate_gate_health() {
5752 let model = "anthropic/claude-haiku-4-5";
5753 let (mut state, rx) = probe_state(1.0, 2, vec![(model, Ok(model_resp(model, "answer")))]);
5754
5755 let registry = GateHealthRegistry::new().with_budget("non-empty", 2, 0.4);
5758 registry.record("default", "non-empty", true);
5761 assert!(
5762 registry.enabled("default", "non-empty"),
5763 "gate must start enabled (window not full yet)"
5764 );
5765 state.gate_health = Arc::new(registry);
5766
5767 let trace = run_enforce_get_trace(state, rx).await;
5779 let sig = trace.probe.expect("probe must fire with sample_rate=1.0");
5780 assert_eq!(sig.k, 2);
5781 assert!(
5782 sig.gate_pass_count <= 2,
5783 "gate_pass_count must be in [0, k]"
5784 );
5785 }
5786
5787 fn predictor_state(
5789 enabled: bool,
5790 outcome: Result<ModelResponse, ProviderError>,
5791 ) -> (AppState, mpsc::Receiver<Trace>) {
5792 let toml = "[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/claude-haiku-4-5\"]\ngates = [\"non-empty\"]\n";
5793 let config = ProxyConfig::from_lookup(|k_| match k_ {
5794 "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
5795 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
5796 _ => None,
5797 })
5798 .unwrap();
5799 let mut outs = HashMap::new();
5800 outs.insert("anthropic/claude-haiku-4-5".to_owned(), outcome);
5801 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
5802 map.insert(
5803 "anthropic".to_owned(),
5804 Arc::new(MockProvider::new("anthropic", outs)),
5805 );
5806 let (traces, rx) = mpsc::channel(64);
5807 let predictor = enabled.then(|| {
5808 Arc::new(std::sync::Mutex::new(firstpass_core::PassPredictor::new(
5809 0.05, 1e-4,
5810 )))
5811 });
5812 let state = AppState {
5813 config: Arc::new(config),
5814 http: reqwest::Client::new(),
5815 providers: ProviderRegistry::from_map(map),
5816 gate_health: Arc::new(GateHealthRegistry::new()),
5817 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
5818 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
5819 traces,
5820 adaptive: None,
5821 bandit: None,
5822 promoter: None,
5823 verified_cache: None,
5824 predictor,
5825 tenant_rate_limiter: None,
5826 spill: None,
5827 };
5828 (state, rx)
5829 }
5830
5831 #[tokio::test]
5832 async fn predictor_off_leaves_predicted_pass_none() {
5833 let (state, rx) =
5834 predictor_state(false, Ok(model_resp("anthropic/claude-haiku-4-5", "ok")));
5835 let trace = run_enforce_get_trace(state, rx).await;
5836 assert!(
5837 trace.predicted_pass.is_none(),
5838 "predictor off => no field (byte-identical)"
5839 );
5840 let j = serde_json::to_string(&trace).unwrap();
5842 assert!(!j.contains("predicted_pass"), "None must be omitted: {j}");
5843 }
5844
5845 #[tokio::test]
5846 async fn predictor_on_records_shadow_prediction_and_serves_identically() {
5847 let (state_off, rx_off) = predictor_state(
5850 false,
5851 Ok(model_resp("anthropic/claude-haiku-4-5", "served answer")),
5852 );
5853 let off = run_enforce_get_trace(state_off, rx_off).await;
5854
5855 let (state_on, rx_on) = predictor_state(
5856 true,
5857 Ok(model_resp("anthropic/claude-haiku-4-5", "served answer")),
5858 );
5859 let on = run_enforce_get_trace(state_on, rx_on).await;
5860
5861 assert_eq!(
5862 on.final_.served_rung, off.final_.served_rung,
5863 "served rung identical"
5864 );
5865 assert_eq!(on.attempts.len(), off.attempts.len(), "same attempts");
5866 assert_eq!(
5867 on.final_.total_cost_usd, off.final_.total_cost_usd,
5868 "predictor never adds served cost"
5869 );
5870 let p = on
5871 .predicted_pass
5872 .expect("predictor on => predicted_pass recorded");
5873 assert!(p > 0.0 && p < 1.0, "shadow prediction in (0,1): {p}");
5874 }
5875
5876 fn rollout_state(
5878 percent: f64,
5879 key: &str,
5880 outcomes: Vec<(&str, Result<ModelResponse, ProviderError>)>,
5881 ) -> (AppState, mpsc::Receiver<Trace>) {
5882 let toml = format!(
5883 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
5884 ladder = [\"anthropic/claude-haiku-4-5\"]\ngates = [\"non-empty\"]\n\
5885 [route.rollout]\npercent = {percent}\nkey = \"{key}\"\n"
5886 );
5887 let config = ProxyConfig::from_lookup(|k| match k {
5888 "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
5889 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
5890 _ => None,
5891 })
5892 .unwrap();
5893 let mut outs = HashMap::new();
5894 for (model, out) in outcomes {
5895 outs.insert(model.to_owned(), out);
5896 }
5897 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
5898 map.insert(
5899 "anthropic".to_owned(),
5900 Arc::new(MockProvider::new("anthropic", outs)),
5901 );
5902 let (traces, rx) = mpsc::channel(64);
5903 let state = AppState {
5904 config: Arc::new(config),
5905 http: reqwest::Client::new(),
5906 providers: ProviderRegistry::from_map(map),
5907 gate_health: Arc::new(GateHealthRegistry::new()),
5908 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
5909 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
5910 traces,
5911 adaptive: None,
5912 bandit: None,
5913 promoter: None,
5914 verified_cache: None,
5915 predictor: None,
5916 tenant_rate_limiter: None,
5917 spill: None,
5918 };
5919 (state, rx)
5920 }
5921
5922 #[tokio::test]
5926 async fn rollout_at_zero_percent_never_enforces() {
5927 for i in 0..25 {
5928 let (state, mut rx) = rollout_state(
5929 0.0,
5930 "session",
5931 vec![(
5932 "anthropic/claude-haiku-4-5",
5933 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
5934 )],
5935 );
5936 let mut headers = HeaderMap::new();
5937 headers.insert("x-firstpass-session", format!("s{i}").parse().unwrap());
5938 let resp = messages(
5939 State(state),
5940 Extension(TenantId("default".to_owned())),
5941 headers,
5942 user_body(),
5943 )
5944 .await;
5945 let _ = resp;
5948 if let Ok(t) = rx.try_recv() {
5949 assert!(
5950 t.attempts.is_empty(),
5951 "0% rollout produced an enforced decision for session s{i}"
5952 );
5953 }
5954 }
5955 }
5956
5957 #[tokio::test]
5960 async fn rollout_at_hundred_percent_always_enforces() {
5961 for i in 0..25 {
5962 let (state, mut rx) = rollout_state(
5963 100.0,
5964 "session",
5965 vec![(
5966 "anthropic/claude-haiku-4-5",
5967 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
5968 )],
5969 );
5970 let mut headers = HeaderMap::new();
5971 headers.insert("x-firstpass-session", format!("s{i}").parse().unwrap());
5972 let resp = messages(
5973 State(state),
5974 Extension(TenantId("default".to_owned())),
5975 headers,
5976 user_body(),
5977 )
5978 .await;
5979 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5980 let t = rx.try_recv().expect("trace enqueued");
5981 assert!(
5982 !t.attempts.is_empty(),
5983 "100% rollout skipped enforcement for session s{i}"
5984 );
5985 }
5986 }
5987
5988 #[tokio::test]
5996 async fn dispatch_arm_matches_the_predicted_arm_per_session() {
5997 let mut checked_enforced = 0;
5998 let mut checked_control = 0;
5999 for i in 0..24 {
6000 let session = format!("session-{i}");
6001 let (state, mut rx) = rollout_state(
6002 50.0,
6003 "session",
6004 vec![(
6005 "anthropic/claude-haiku-4-5",
6006 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
6007 )],
6008 );
6009 let expected = firstpass_core::rollout::decide(
6012 &state.config.prompt_salt,
6013 &firstpass_core::Rollout {
6014 percent: 50.0,
6015 key: firstpass_core::RolloutKey::Session,
6016 },
6017 &session,
6018 )
6019 .enforced;
6020
6021 let mut headers = HeaderMap::new();
6022 headers.insert("x-firstpass-session", session.parse().unwrap());
6023 let _ = messages(
6024 State(state),
6025 Extension(TenantId("default".to_owned())),
6026 headers,
6027 user_body(),
6028 )
6029 .await;
6030 let observed = rx
6031 .try_recv()
6032 .map(|t| !t.attempts.is_empty())
6033 .unwrap_or(false);
6034
6035 assert_eq!(
6036 observed,
6037 expected,
6038 "{session}: dispatch put it in the {} arm, bucketing predicted the {} arm",
6039 if observed { "enforced" } else { "control" },
6040 if expected { "enforced" } else { "control" }
6041 );
6042 if expected {
6043 checked_enforced += 1;
6044 } else {
6045 checked_control += 1;
6046 }
6047 }
6048 assert!(
6050 checked_enforced > 0 && checked_control > 0,
6051 "test saw only one arm ({checked_enforced} enforced, {checked_control} control)"
6052 );
6053 }
6054}