1use std::sync::Arc;
5use std::time::Instant;
6
7use axum::body::Body;
8use axum::extract::{Request, State};
9use axum::http::HeaderMap;
10use axum::middleware::Next;
11use axum::response::{IntoResponse, Response};
12use axum::routing::{get, post};
13use axum::{Extension, Json, Router};
14use bytes::Bytes;
15use firstpass_core::features::{hour_bucket, token_bucket};
16use firstpass_core::hashchain::sha256_hex;
17use firstpass_core::{
18 Attempt, DeferredVerdict, Dialect, FEATURE_VERSION, Features, FinalOutcome, GENESIS_HASH, Mode,
19 ModelRef, PolicyRef, ProbeRegime, ProbeSignal, RequestInfo, RoutingMode, Score, ServedFrom,
20 TaskKind, Trace, Verdict,
21};
22use serde::Deserialize;
23use serde_json::Value;
24use std::future::Future;
25use std::time::Duration;
26use tokio::sync::mpsc::error::TrySendError;
27use uuid::Uuid;
28
29use crate::config::ProxyConfig;
30use crate::error::ProxyError;
31use crate::gate::{GateHealthRegistry, aggregate_with_policy, resolve_gates};
32use crate::provider::{Auth, ChatMessage, ModelRequest, ModelResponse, ProviderRegistry};
33use crate::router::{EnforceCtx, EngineOutcome, route_enforce};
34use crate::store;
35use crate::tenant_auth::{TenantId, auth_middleware};
36use crate::upstream::{forward_anthropic, forward_anthropic_streaming};
37use firstpass_core::Route;
38use firstpass_core::trace::ShadowSignal;
39
40#[derive(Clone)]
43pub struct AppState {
44 pub config: Arc<ProxyConfig>,
46 pub http: reqwest::Client,
48 pub providers: ProviderRegistry,
50 pub gate_health: Arc<GateHealthRegistry>,
52 pub shadow_ledger: Arc<crate::shadow::ShadowLedger>,
55 pub guardrails: Arc<crate::guard::GuardrailRegistry>,
58 pub traces: store::TraceSender,
60 pub adaptive: Option<Arc<std::sync::Mutex<firstpass_core::conformal::AdaptiveConformal>>>,
64 pub bandit: Option<Arc<std::sync::Mutex<crate::bandit::StartRungBandit>>>,
69 pub promoter: Option<Arc<crate::affinity::SessionPromoter>>,
74 pub verified_cache: Option<Arc<dyn crate::verified_cache::CacheStore>>,
78 pub predictor: Option<Arc<std::sync::Mutex<firstpass_core::PassPredictor>>>,
84 pub tenant_rate_limiter: Option<Arc<governor::DefaultKeyedRateLimiter<String>>>,
88 pub spill: Option<store::SpillHandle>,
92}
93
94#[must_use]
110fn routing_cfg_condense(state: &AppState) -> Option<&firstpass_core::config::CondenseConfig> {
111 state
112 .config
113 .routing
114 .as_ref()
115 .and_then(|r| r.escalation.condense.as_ref())
116}
117
118#[must_use]
123fn mode_now() -> jiff::Timestamp {
125 jiff::Timestamp::now()
126}
127
128fn cache_hit_trace(
136 ctx: &crate::router::EnforceCtx<'_>,
137 entry: &crate::verified_cache::Entry,
138 ts: jiff::Timestamp,
139) -> Trace {
140 Trace {
141 trace_id: uuid::Uuid::now_v7(),
142 prev_hash: firstpass_core::GENESIS_HASH.to_owned(),
143 tenant_id: ctx.tenant_id.clone(),
144 session_id: ctx.session_id.clone(),
145 ts,
146 mode: Mode::Enforce,
147 policy: PolicyRef {
148 id: format!("{}+cache", ctx.policy_id),
149 explore: false,
150 propensity: None,
151 mode_profile: None,
152 },
153 request: RequestInfo {
154 api: ctx.api.clone(),
155 prompt_hash: ctx.prompt_hash.clone(),
156 features: ctx.features.clone(),
157 },
158 attempts: Vec::new(),
159 final_: FinalOutcome {
160 served_rung: None,
161 served_from: ServedFrom::Cache,
162 total_cost_usd: 0.0,
166 gate_cost_usd: 0.0,
167 total_latency_ms: 0,
168 escalations: 0,
169 counterfactual_baseline_usd: entry.original_cost_usd,
170 savings_usd: entry.original_cost_usd,
171 cache_source: Some(entry.source),
172 },
173 deferred: Vec::new(),
174 predicted_pass: None,
175 probe: None,
176 elastic: None,
177 rollout: None,
178 shadow: None,
179 route_ix: None,
180 }
181}
182
183pub async fn build_verified_cache(
184 config: &ProxyConfig,
185) -> Result<Option<Arc<dyn crate::verified_cache::CacheStore>>, String> {
186 let Some(c) = config
187 .routing
188 .as_ref()
189 .and_then(|r| r.escalation.verified_cache.as_ref())
190 else {
191 return Ok(None);
192 };
193 if let Some(url) = c.redis_url.as_deref() {
194 #[cfg(feature = "redis-cache")]
195 {
196 let store = crate::verified_cache::RedisStore::connect(url, c.ttl_secs).await?;
197 tracing::info!("verified cache: shared via redis");
198 return Ok(Some(Arc::new(store)));
199 }
200 #[cfg(not(feature = "redis-cache"))]
201 {
202 let _ = url;
203 return Err(
204 "[escalation.verified_cache] redis_url is set, but this binary was built without \
205 the `redis-cache` feature. Rebuild with `--features redis-cache`, or remove \
206 redis_url to use the in-process cache."
207 .to_owned(),
208 );
209 }
210 }
211 Ok(Some(Arc::new(crate::verified_cache::VerifiedCache::new(
212 std::time::Duration::from_secs(c.ttl_secs),
213 c.max_entries,
214 ))))
215}
216
217pub fn build_promoter(
218 config: &ProxyConfig,
219) -> firstpass_core::Result<Option<Arc<crate::affinity::SessionPromoter>>> {
220 match config
221 .routing
222 .as_ref()
223 .and_then(|r| r.escalation.session_promotion.as_ref())
224 {
225 Some(sp) => Ok(Some(Arc::new(crate::affinity::SessionPromoter::new(
226 sp.clone(),
227 )?))),
228 None => Ok(None),
229 }
230}
231
232#[must_use]
233pub fn build_tenant_rate_limiter(
234 config: &ProxyConfig,
235) -> Option<Arc<governor::DefaultKeyedRateLimiter<String>>> {
236 let per_sec = config.tenant_rate_per_sec?;
237 Some(Arc::new(governor::RateLimiter::keyed(
238 governor::Quota::per_second(per_sec),
239 )))
240}
241
242pub async fn tenant_rate_limit_middleware(
246 State(state): State<AppState>,
247 Extension(tenant): Extension<TenantId>,
248 req: Request,
249 next: Next,
250) -> Response {
251 if let Some(limiter) = &state.tenant_rate_limiter
252 && limiter.check_key(&tenant.0).is_err()
253 {
254 return ProxyError::RateLimited.into_response();
255 }
256 next.run(req).await
257}
258
259impl std::fmt::Debug for AppState {
260 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
261 f.debug_struct("AppState")
262 .field("config", &self.config)
263 .finish_non_exhaustive()
264 }
265}
266
267fn offer_trace(traces: &store::TraceSender, spill: Option<&store::SpillHandle>, trace: Trace) {
284 record_trace_metrics(&trace);
285 match traces.try_send(trace) {
286 Ok(()) => {}
287 Err(TrySendError::Full(t)) => {
288 if let Some(handle) = spill {
289 match store::append_to_spill(handle, &t) {
290 Ok(()) => {
291 metrics::counter!("firstpass_receipts_spilled_total").increment(1);
292 }
293 Err(e) => {
294 tracing::error!(%e, "durable mode: spill write failed; trace lost");
295 metrics::counter!("firstpass_traces_dropped_total").increment(1);
296 }
297 }
298 } else {
299 tracing::warn!("trace channel full; dropping trace (writer behind under load)");
300 metrics::counter!("firstpass_traces_dropped_total").increment(1);
301 }
302 }
303 Err(TrySendError::Closed(_)) => {
304 tracing::warn!("trace writer is gone; dropping trace");
305 }
306 }
307}
308
309fn record_trace_metrics(trace: &Trace) {
313 if trace.mode == Mode::Enforce {
314 metrics::histogram!("firstpass_enforce_latency_ms")
315 .record(trace.final_.total_latency_ms as f64);
316 if trace.final_.escalations > 0 {
317 metrics::counter!("firstpass_escalations_total")
318 .increment(u64::from(trace.final_.escalations));
319 }
320 }
321 let served_from = match trace.final_.served_from {
322 ServedFrom::Attempt => "attempt",
323 ServedFrom::BestAttempt => "best_attempt",
324 ServedFrom::Error => "error",
325 ServedFrom::Cache => "cache",
329 };
330 metrics::counter!("firstpass_served_total", "served_from" => served_from).increment(1);
331 if trace.final_.served_from == ServedFrom::Error {
332 metrics::counter!("firstpass_upstream_failures_total").increment(1);
333 }
334 metrics::gauge!("firstpass_cost_usd_total").increment(trace.final_.total_cost_usd);
338 metrics::gauge!("firstpass_gate_cost_usd_total").increment(trace.final_.gate_cost_usd);
339 metrics::gauge!("firstpass_baseline_usd_total")
340 .increment(trace.final_.counterfactual_baseline_usd);
341 metrics::gauge!("firstpass_savings_usd_total").increment(trace.final_.savings_usd);
342 if let Some(rung) = trace.final_.served_rung {
344 let model = trace
345 .attempts
346 .iter()
347 .find(|a| a.rung == rung)
348 .map(|a| a.model.clone())
349 .unwrap_or_else(|| "unknown".to_owned());
350 metrics::counter!(
351 "firstpass_served_rung_total",
352 "rung" => rung.to_string(),
353 "model" => model
354 )
355 .increment(1);
356 }
357}
358
359const MAX_BODY_BYTES: usize = 16 * 1024 * 1024;
363
364pub(crate) fn u01(seed: u128) -> f64 {
375 let lo = splitmix64_finalise(seed as u64);
376 let hi = splitmix64_finalise((seed >> 64) as u64);
377 ((lo ^ hi) >> 11) as f64 * (1.0_f64 / (1u64 << 53) as f64)
379}
380
381fn splitmix64_finalise(mut z: u64) -> u64 {
382 z = z.wrapping_add(0x9E37_79B9_7F4A_7C15);
383 z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
384 z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
385 z ^ (z >> 31)
386}
387
388#[must_use]
395pub(crate) fn epsilon_propensity(chosen: u32, greedy: u32, epsilon: f64, k: usize) -> f64 {
396 let greedy_term = if chosen == greedy { 1.0 - epsilon } else { 0.0 };
397 greedy_term + epsilon / k as f64
398}
399
400pub fn app(state: AppState) -> Result<Router, ProxyError> {
407 crate::metrics::install()?;
408 let max_concurrency = state.config.max_concurrency;
409
410 let business = Router::new()
419 .route("/v1/messages", post(messages))
420 .route("/v1/chat/completions", post(chat_completions))
421 .route("/v1/responses", post(responses))
422 .route("/v1/feedback", post(feedback))
423 .route("/v1/capabilities", get(capabilities))
424 .route("/v1/models", get(models))
425 .route("/v1/receipts", get(receipts))
428 .layer(axum::middleware::from_fn_with_state(
429 state.clone(),
430 tenant_rate_limit_middleware,
431 ))
432 .layer(axum::middleware::from_fn_with_state(
433 state.clone(),
434 auth_middleware,
435 ));
436
437 Ok(Router::new()
438 .merge(business)
439 .route("/healthz", get(healthz))
440 .route("/metrics", get(crate::metrics::handler))
441 .route("/panel", get(panel))
444 .layer(axum::extract::DefaultBodyLimit::max(MAX_BODY_BYTES))
446 .layer(tower::limit::GlobalConcurrencyLimitLayer::new(
449 max_concurrency,
450 ))
451 .with_state(state))
452}
453
454async fn healthz() -> impl IntoResponse {
456 Json(serde_json::json!({
462 "status": "ok",
463 "service": "firstpass",
464 "version": env!("CARGO_PKG_VERSION"),
465 }))
466}
467
468const PANEL_HTML: &str = include_str!("panel.html");
472
473async fn receipts(
479 State(state): State<AppState>,
480 Extension(TenantId(tenant)): Extension<TenantId>,
481 axum::extract::Query(q): axum::extract::Query<ReceiptsQuery>,
482) -> Response {
483 let limit = q.limit.unwrap_or(20).clamp(1, 200);
485 let db = state.config.db_path.clone();
486 let traces =
487 match tokio::task::spawn_blocking(move || crate::store::load_tenant_traces(&db, &tenant))
488 .await
489 {
490 Ok(Ok(t)) => t,
491 _ => Vec::new(),
494 };
495 let items: Vec<serde_json::Value> = traces
496 .iter()
497 .rev()
498 .take(limit)
499 .map(|t| {
500 let f = &t.final_;
501 serde_json::json!({
502 "trace_id": t.trace_id,
503 "served_rung": f.served_rung,
504 "served_model": f.served_rung
509 .and_then(|r| t.attempts.iter().find(|a| a.rung == r))
510 .map(|a| a.model.clone()),
511 "attempts": t.attempts.iter().map(|a| serde_json::json!({
512 "rung": a.rung,
513 "model": a.model,
514 "verdict": a.verdict,
515 "cost_usd": a.cost_usd,
516 "latency_ms": a.latency_ms,
517 })).collect::<Vec<_>>(),
518 "total_cost_usd": f.total_cost_usd,
519 "baseline_usd": f.counterfactual_baseline_usd,
520 "savings_usd": f.savings_usd,
521 })
522 })
523 .collect();
524 axum::Json(serde_json::json!({ "receipts": items })).into_response()
525}
526
527#[derive(Debug, serde::Deserialize)]
529struct ReceiptsQuery {
530 limit: Option<usize>,
532}
533
534async fn panel() -> impl IntoResponse {
540 (
541 [(axum::http::header::CONTENT_TYPE, "text/html; charset=utf-8")],
542 PANEL_HTML,
543 )
544}
545
546async fn capabilities(State(state): State<AppState>) -> impl IntoResponse {
547 let (ladder, gates) = state
550 .config
551 .routing
552 .as_ref()
553 .and_then(|c| c.routes.iter().find(|r| r.mode == Mode::Enforce))
554 .map(|r| (r.ladder.clone(), r.gates.clone()))
555 .unwrap_or_default();
556 let routing_modes: Vec<serde_json::Value> = RoutingMode::ALL
557 .iter()
558 .map(|m| {
559 let p = m.preset();
560 serde_json::json!({
561 "name": m.as_str(),
562 "description": p.description,
563 "tradeoff": p.tradeoff,
564 })
565 })
566 .collect();
567 Json(serde_json::json!({
568 "service": "firstpass",
569 "version": env!("CARGO_PKG_VERSION"),
570 "feature_version": FEATURE_VERSION,
571 "modes": ["observe", "enforce"],
572 "routing_modes": routing_modes,
573 "wire_apis": ["anthropic.messages", "openai.chat_completions", "openai.responses"],
574 "ladder": ladder,
575 "gates": gates,
576 "feedback_api": "POST /v1/feedback",
577 "offboarding": "unset ANTHROPIC_BASE_URL (or OPENAI_BASE_URL for OpenAI clients)",
578 }))
579}
580
581async fn models(State(state): State<AppState>) -> impl IntoResponse {
593 Json(model_list(state.config.routing.as_ref()))
594}
595
596fn model_list(routing: Option<&firstpass_core::config::Config>) -> serde_json::Value {
599 let prices = routing.map(|c| c.price_table()).unwrap_or_default();
600 let mut seen = std::collections::HashSet::new();
603 let data: Vec<serde_json::Value> = routing
604 .map(|c| c.routes.as_slice())
605 .unwrap_or_default()
606 .iter()
607 .flat_map(|r| r.ladder.iter())
608 .filter(|id| seen.insert(id.as_str()))
609 .map(|id| {
610 let price = prices.get(id);
611 serde_json::json!({
612 "id": id,
613 "object": "model",
614 "created": 0,
617 "owned_by": id.split_once('/').map_or("firstpass", |(p, _)| p),
618 "firstpass": {
619 "input_per_mtok": price.map(|p| p.input_per_mtok),
620 "output_per_mtok": price.map(|p| p.output_per_mtok),
621 },
622 })
623 })
624 .collect();
625 serde_json::json!({ "object": "list", "data": data })
626}
627
628#[derive(Debug, Deserialize)]
630struct FeedbackRequest {
631 trace_id: String,
633 gate_id: String,
635 verdict: String,
637 #[serde(default)]
639 score: Option<f64>,
640 reporter: String,
642}
643
644async fn feedback(
649 State(state): State<AppState>,
650 Extension(TenantId(tenant)): Extension<TenantId>,
651 body: Bytes,
652) -> Response {
653 let req: FeedbackRequest = match serde_json::from_slice(&body) {
654 Ok(r) => r,
655 Err(e) => {
656 return ProxyError::BadRequest(format!("invalid feedback body: {e}")).into_response();
657 }
658 };
659 let verdict = match req.verdict.as_str() {
660 "pass" => Verdict::Pass,
661 "fail" => Verdict::Fail,
662 "abstain" => Verdict::Abstain,
663 other => {
664 return ProxyError::BadRequest(format!("unknown verdict {other:?}")).into_response();
665 }
666 };
667 let score = match req.score {
668 Some(s) => match Score::new(s) {
669 Ok(sc) => Some(sc),
670 Err(_) => {
671 return ProxyError::BadRequest(format!("score {s} out of range [0,1]"))
672 .into_response();
673 }
674 },
675 None => None,
676 };
677
678 let db = state.config.db_path.clone();
679
680 let (db_check, tenant_check, tid_check) = (db.clone(), tenant.clone(), req.trace_id.clone());
685 match tokio::task::spawn_blocking(move || {
686 store::trace_exists(&db_check, &tenant_check, &tid_check)
687 })
688 .await
689 {
690 Ok(Ok(true)) => {}
691 Ok(Ok(false)) => {
692 return ProxyError::NotFound(format!("unknown trace_id {:?}", req.trace_id))
693 .into_response();
694 }
695 Ok(Err(e)) => {
696 tracing::error!(%e, "feedback: trace_exists check failed");
697 return ProxyError::Internal(e.to_string()).into_response();
698 }
699 Err(e) => {
700 tracing::error!(%e, "feedback: trace_exists task panicked");
701 return ProxyError::Internal(e.to_string()).into_response();
702 }
703 }
704
705 if verdict == Verdict::Fail
710 && let Some(cache) = state.verified_cache.as_ref()
711 {
712 let reported = req.trace_id.parse::<uuid::Uuid>().ok();
717 let origin = {
718 let (d, t, i) = (db.clone(), tenant.clone(), req.trace_id.clone());
719 tokio::task::spawn_blocking(move || store::trace_cache_source(&d, &t, &i))
720 .await
721 .ok()
722 .and_then(Result::ok)
723 .flatten()
724 .flatten()
725 };
726 let mut dropped = 0usize;
727 for id in reported.into_iter().chain(origin) {
728 dropped += cache.retract(id).await;
729 }
730 if dropped > 0 {
731 tracing::info!(
732 trace = %req.trace_id,
733 ?origin,
734 dropped,
735 "verified cache: retracted a disproven answer"
736 );
737 metrics::counter!("firstpass_cache_retractions_total").increment(dropped as u64);
738 }
739 }
740
741 let feedback_signal = match verdict {
743 Verdict::Pass => Some(true),
744 Verdict::Fail => Some(false),
745 Verdict::Abstain => None,
746 };
747 let dv = DeferredVerdict {
748 gate_id: req.gate_id,
749 verdict,
750 score,
751 reported_at: jiff::Timestamp::now(),
752 reporter: req.reporter,
753 };
754 let trace_id = req.trace_id.clone();
755 match tokio::task::spawn_blocking(move || store::append_deferred(&db, &req.trace_id, &dv)).await
756 {
757 Ok(Ok(())) => {
758 if let (Some(a), Some(correct)) = (state.adaptive.as_ref(), feedback_signal)
760 && let Ok(mut g) = a.lock()
761 {
762 g.observe_served(correct);
763 metrics::gauge!("firstpass_serve_threshold").set(g.threshold());
764 metrics::gauge!("firstpass_realized_served_failure")
765 .set(g.realized_served_failure());
766 }
767
768 let attributed_route = tokio::task::spawn_blocking({
773 let (db, ten, tid) = (
774 state.config.db_path.clone(),
775 tenant.clone(),
776 trace_id.clone(),
777 );
778 move || store::trace_route(&db, &ten, &tid)
779 })
780 .await
781 .ok()
782 .and_then(Result::ok)
783 .flatten()
784 .flatten()
785 .unwrap_or(0) as usize;
786
787 if let (Some(cfg), Some(correct)) = (
792 state.config.routing.as_ref().and_then(|r| r.guardrail),
793 feedback_signal,
794 ) {
795 let cooldown = state
796 .config
797 .routing
798 .as_ref()
799 .map_or(3_600, |r| r.guardrail_cooldown_secs);
800 let reaction = state.guardrails.record(
801 &tenant,
802 attributed_route,
803 &cfg,
804 correct,
805 jiff::Timestamp::now().as_second(),
806 cooldown,
807 );
808 match &reaction {
809 crate::guard::Reaction::Demoted(v) => {
810 metrics::counter!("firstpass_guardrail_demotions_total").increment(1);
811 tracing::error!(
812 tenant = %tenant,
813 n = v.n,
814 rate = v.rate,
815 bound = v.bound,
816 alpha = cfg.alpha,
817 "GUARDRAIL: served-failure bound exceeded target — route demoted to observe; traffic now serves as it would without Firstpass"
818 );
819 }
820 crate::guard::Reaction::Alarmed(v) => {
821 metrics::counter!("firstpass_guardrail_breaches_total").increment(1);
822 tracing::error!(
823 tenant = %tenant,
824 n = v.n,
825 rate = v.rate,
826 bound = v.bound,
827 alpha = cfg.alpha,
828 "GUARDRAIL: served-failure bound exceeded target (alarm only — routing unchanged)"
829 );
830 }
831 crate::guard::Reaction::None => {}
832 }
833 }
834 (
835 axum::http::StatusCode::ACCEPTED,
836 Json(serde_json::json!({ "status": "recorded", "trace_id": trace_id })),
837 )
838 .into_response()
839 }
840 Ok(Err(e)) => {
841 tracing::error!(%e, "feedback: append_deferred failed");
842 ProxyError::Internal(e.to_string()).into_response()
843 }
844 Err(e) => {
845 tracing::error!(%e, "feedback: append_deferred task panicked");
846 ProxyError::Internal(e.to_string()).into_response()
847 }
848 }
849}
850
851const SESSION_HEADER: &str = "x-firstpass-session";
854
855const AGENT_HEADER: &str = "x-firstpass-agent";
857const SUBAGENT_HEADER: &str = "x-firstpass-subagent";
859#[expect(
865 clippy::too_many_arguments,
866 reason = "mirrors the enforce context it evaluates; a wrapper struct would only move the list"
867)]
868fn spawn_shadow(
869 state: &AppState,
870 route: &firstpass_core::Route,
871 route_ix: usize,
872 body: Bytes,
873 auth: Auth,
874 features: Features,
875 tenant: String,
876 session_id: String,
877 api: &str,
878) {
879 let Some(shadow) = route.shadow else {
880 return;
881 };
882 if !shadow.sampled(&state.config.prompt_salt, &session_id) {
885 return;
886 }
887 let (state, route, api) = (state.clone(), route.clone(), api.to_owned());
888 tokio::spawn(async move {
889 let signal = evaluate_shadow(
890 &state, &route, shadow, route_ix, &body, auth, &features, tenant, session_id, &api, 0.0,
891 )
892 .await;
893 tracing::debug!(
895 would_pass = signal.would_pass,
896 projected_usd = signal.projected_cost_usd,
897 skipped = ?signal.skipped,
898 "shadow evaluation complete"
899 );
900 });
901}
902
903#[expect(
910 clippy::too_many_arguments,
911 reason = "mirrors the enforce context; grouping them into a struct would only move the list"
912)]
913async fn evaluate_shadow(
914 state: &AppState,
915 route: &firstpass_core::Route,
916 shadow: firstpass_core::rollout::Shadow,
917 route_ix: usize,
918 body: &Bytes,
919 auth: Auth,
920 features: &Features,
921 tenant: String,
922 session_id: String,
923 api: &str,
924 actual_cost_usd: f64,
925) -> ShadowSignal {
926 let skipped = |why: &str| ShadowSignal {
927 would_serve_rung: None,
928 would_pass: false,
929 projected_cost_usd: 0.0,
930 actual_cost_usd,
931 skipped: Some(why.to_owned()),
932 };
933
934 let now = jiff::Timestamp::now();
935 if !state
936 .shadow_ledger
937 .may_spend(&tenant, route_ix, shadow.max_usd_per_day, now)
938 {
939 return skipped("budget_exhausted");
942 }
943
944 let Some(base_request) = parse_model_request(body) else {
945 return skipped("unparseable_request");
946 };
947 let gate_defs = state
948 .config
949 .routing
950 .as_ref()
951 .map_or(&[][..], |cfg| &cfg.gate_defs);
952 let gates = resolve_gates(
953 &route.gates,
954 gate_defs,
955 &state.providers,
956 &auth,
957 &state.config.prices,
958 );
959 let (budget, max_rungs) = state
960 .config
961 .routing
962 .as_ref()
963 .map_or((None, u32::MAX), |cfg| {
964 (
965 cfg.budget.per_request_usd,
966 cfg.escalation.max_rungs_per_request,
967 )
968 });
969
970 let mut base_request = base_request;
971 base_request.cache_prefix = state
975 .config
976 .routing
977 .as_ref()
978 .is_some_and(|r| r.escalation.prompt_cache);
979 let ctx = EnforceCtx {
980 condense: routing_cfg_condense(state),
981 ladder: &route.ladder,
982 gates: &gates,
983 health: &state.gate_health,
984 base_request: &base_request,
985 providers: &state.providers,
986 auth: &auth,
987 prices: &state.config.prices,
988 budget_per_request_usd: budget,
989 max_rungs,
990 speculation: 0,
993 serve_threshold: None,
994 elastic: None,
995 features: features.clone(),
996 start_rung: 0,
997 tenant_id: tenant.clone(),
998 session_id,
999 prompt_hash: prompt_hash(&state.config.prompt_salt, body),
1000 api: api.to_owned(),
1001 policy_id: "shadow".to_owned(),
1002 };
1003
1004 let (outcome, trace) = route_enforce(ctx).await;
1005 let spent = trace.final_.total_cost_usd;
1006 state.shadow_ledger.debit(&tenant, route_ix, spent, now);
1007
1008 let would_pass = matches!(outcome, EngineOutcome::Served(_));
1011 let would_serve_rung = if would_pass {
1012 trace.final_.served_rung
1013 } else {
1014 None
1015 };
1016 ShadowSignal {
1017 would_serve_rung,
1018 would_pass,
1019 projected_cost_usd: spent,
1020 actual_cost_usd,
1021 skipped: None,
1022 }
1023}
1024
1025const MODE_PROFILE_HEADER: &str = "x-firstpass-mode";
1028
1029fn resolve_mode(headers: &HeaderMap, route: &Route, config: &ProxyConfig) -> RoutingMode {
1038 if let Some(val) = header_str(headers, MODE_PROFILE_HEADER) {
1040 match val.trim().to_ascii_lowercase().as_str() {
1041 "observe" => return RoutingMode::Observe,
1042 "cost" => return RoutingMode::Cost,
1043 "balanced" => return RoutingMode::Balanced,
1044 "quality" => return RoutingMode::Quality,
1045 "latency" => return RoutingMode::Latency,
1046 "max" => return RoutingMode::Max,
1047 other => {
1048 tracing::warn!(
1049 value = other,
1050 "unknown x-firstpass-mode value; ignoring \
1051 (valid: observe|cost|balanced|quality|latency|max)"
1052 );
1053 }
1054 }
1055 }
1056 if let Some(m) = route.routing_mode {
1058 return m;
1059 }
1060 config.default_routing_mode
1062}
1063
1064async fn messages(
1069 State(state): State<AppState>,
1070 Extension(TenantId(tenant)): Extension<TenantId>,
1071 headers: HeaderMap,
1072 body: Bytes,
1073) -> Response {
1074 let session_header = header_str(&headers, SESSION_HEADER);
1075
1076 if let Some(routing) = state.config.routing.as_ref() {
1079 let features = extract_features(&headers, &body);
1080 if let Some(route) = routing
1081 .route_for(&features)
1082 .filter(|r| r.mode == Mode::Enforce && !r.ladder.is_empty())
1083 {
1084 let route_ix = routing
1087 .routes
1088 .iter()
1089 .position(|r| std::ptr::eq(r, route))
1090 .unwrap_or(0);
1091 let route = route.clone();
1094 let routing_mode = resolve_mode(&headers, &route, &state.config);
1096 if routing_mode == RoutingMode::Observe {
1098 return observe_passthrough(state, headers, body, session_header, tenant).await;
1099 }
1100 if state
1104 .guardrails
1105 .is_demoted(&tenant, route_ix, jiff::Timestamp::now().as_second())
1106 {
1107 return observe_passthrough(state, headers, body, session_header, tenant).await;
1108 }
1109 if let Some(rollout) = route.rollout.as_ref() {
1114 let key_value = match rollout.key {
1115 firstpass_core::RolloutKey::Session => session_header
1116 .clone()
1117 .unwrap_or_else(|| firstpass_core::rollout::request_identity(&body)),
1118 firstpass_core::RolloutKey::Request => {
1124 firstpass_core::rollout::request_identity(&body)
1125 }
1126 firstpass_core::RolloutKey::Tenant => tenant.to_string(),
1127 };
1128 let decision =
1129 firstpass_core::rollout::decide(&state.config.prompt_salt, rollout, &key_value);
1130 if !decision.enforced {
1131 let shadow_ctx = route.shadow.map(|_| {
1133 (
1134 body.clone(),
1135 Auth::from_headers(&headers),
1136 features.clone(),
1137 tenant.clone(),
1138 session_header
1139 .clone()
1140 .unwrap_or_else(|| Uuid::now_v7().to_string()),
1141 )
1142 });
1143 let resp =
1144 observe_passthrough(state.clone(), headers, body, session_header, tenant)
1145 .await;
1146 if let Some((b, auth, feats, ten, sess)) = shadow_ctx {
1149 spawn_shadow(
1150 &state,
1151 &route,
1152 route_ix,
1153 b,
1154 auth,
1155 feats,
1156 ten,
1157 sess,
1158 "anthropic",
1159 );
1160 }
1161 return resp;
1162 }
1163 }
1164 if enforce_can_handle(
1165 &features,
1166 &body,
1167 routing.escalation.enforce_structured,
1168 &route.ladder,
1169 &state.providers,
1170 Dialect::Anthropic,
1171 ) {
1172 return handle_enforce(
1173 &state,
1174 &headers,
1175 &body,
1176 features,
1177 &route,
1178 route_ix,
1179 session_header,
1180 tenant,
1181 routing_mode,
1182 )
1183 .await;
1184 }
1185 tracing::info!(
1189 "enforce route matched but structured request can't be routed faithfully (flag/ladder); serving via observe passthrough"
1190 );
1191 }
1192 }
1193 observe_passthrough(state, headers, body, session_header, tenant).await
1194}
1195
1196fn enforce_can_handle(
1210 features: &Features,
1211 body: &[u8],
1212 enforce_structured: bool,
1213 ladder: &[String],
1214 providers: &crate::provider::ProviderRegistry,
1215 inbound: Dialect,
1216) -> bool {
1217 let structured = features.tool_count > 0
1218 || features.has_images
1219 || match inbound {
1220 Dialect::Anthropic => messages_have_tool_blocks(body),
1221 Dialect::Openai => openai_messages_have_tool_calls(body),
1222 Dialect::Gemini => false,
1223 };
1224 if !structured {
1225 return true;
1226 }
1227 if !enforce_structured {
1228 return false;
1229 }
1230 let all_verbatim = ladder.iter().all(|rung| {
1232 let provider_id = rung.split('/').next().unwrap_or_default();
1233 providers
1234 .get(provider_id)
1235 .is_some_and(|p| p.carries_structured_verbatim(inbound))
1236 });
1237 if all_verbatim {
1238 return true;
1239 }
1240 if inbound == Dialect::Openai && !openai_has_http_images(body) {
1244 let all_anthropic = ladder.iter().all(|rung| {
1245 let pid = rung.split('/').next().unwrap_or_default();
1246 providers
1247 .get(pid)
1248 .is_some_and(|p| p.carries_structured_verbatim(Dialect::Anthropic))
1249 });
1250 if all_anthropic {
1251 return true;
1252 }
1253 }
1254 false
1255}
1256
1257fn messages_have_tool_blocks(body: &[u8]) -> bool {
1260 serde_json::from_slice::<Value>(body)
1261 .ok()
1262 .and_then(|json| {
1263 json.get("messages")
1264 .and_then(Value::as_array)
1265 .map(|messages| messages.iter().any(message_has_tool_block))
1266 })
1267 .unwrap_or(false)
1268}
1269
1270fn message_has_tool_block(message: &Value) -> bool {
1272 message
1273 .get("content")
1274 .and_then(Value::as_array)
1275 .is_some_and(|blocks| {
1276 blocks.iter().any(|block| {
1277 matches!(
1278 block.get("type").and_then(Value::as_str),
1279 Some("tool_use" | "tool_result")
1280 )
1281 })
1282 })
1283}
1284
1285fn is_stream_request(body: &[u8]) -> bool {
1287 serde_json::from_slice::<Value>(body)
1288 .ok()
1289 .and_then(|json| json.get("stream").and_then(Value::as_bool))
1290 .unwrap_or(false)
1291}
1292
1293fn header_str(headers: &HeaderMap, name: &str) -> Option<String> {
1295 headers
1296 .get(name)
1297 .and_then(|v| v.to_str().ok())
1298 .map(str::to_owned)
1299}
1300
1301fn extract_features(headers: &HeaderMap, body: &[u8]) -> Features {
1304 let (_model, tool_count, has_images) = request_features(body);
1305 let mut f = Features::new(TaskKind::Other);
1306 f.agent = header_str(headers, AGENT_HEADER);
1307 f.subagent = header_str(headers, SUBAGENT_HEADER);
1308 f.tool_count = tool_count;
1309 f.has_images = has_images;
1310 f.prompt_token_bucket = token_bucket(body.len() as u64);
1313 f.hour_bucket = hour_bucket(jiff::Timestamp::now());
1314 f
1315}
1316
1317#[allow(clippy::too_many_arguments)]
1320async fn handle_enforce(
1321 state: &AppState,
1322 headers: &HeaderMap,
1323 body: &Bytes,
1324 features: Features,
1325 route: &Route,
1326 route_ix: usize,
1329 session_header: Option<String>,
1330 tenant: String,
1331 routing_mode: RoutingMode,
1332) -> Response {
1333 if is_stream_request(body) {
1341 let (tx, rx) = tokio::sync::oneshot::channel::<Result<Value, ProxyError>>();
1342 let (state_c, headers_c, body_c, route_c) =
1343 (state.clone(), headers.clone(), body.clone(), route.clone());
1344 tokio::spawn(async move {
1345 let out = enforce_pipeline(
1346 &state_c,
1347 &headers_c,
1348 &body_c,
1349 features,
1350 &route_c,
1351 route_ix,
1352 session_header,
1353 tenant,
1354 routing_mode,
1355 )
1356 .await;
1357 let _ = tx.send(out);
1358 });
1359 return sse_keepalive_response(rx, anthropic_sse_from_message);
1360 }
1361 match enforce_pipeline(
1362 state,
1363 headers,
1364 body,
1365 features,
1366 route,
1367 route_ix,
1368 session_header,
1369 tenant,
1370 routing_mode,
1371 )
1372 .await
1373 {
1374 Ok(message) => (axum::http::StatusCode::OK, Json(message)).into_response(),
1375 Err(e) => e.into_response(),
1376 }
1377}
1378
1379#[allow(clippy::too_many_arguments)] async fn enforce_pipeline_inner(
1384 state: &AppState,
1385 body: &Bytes,
1386 base_request: ModelRequest,
1387 auth: Auth,
1388 features: Features,
1389 route: &Route,
1390 session_header: Option<String>,
1391 tenant: String,
1392 api: &str,
1393 routing_mode: RoutingMode,
1394 route_ix: usize,
1395) -> Result<ModelResponse, ProxyError> {
1396 let gate_defs = state
1397 .config
1398 .routing
1399 .as_ref()
1400 .map_or(&[][..], |cfg| &cfg.gate_defs);
1401 let gates = resolve_gates(
1402 &route.gates,
1403 gate_defs,
1404 &state.providers,
1405 &auth,
1406 &state.config.prices,
1407 );
1408 let session_id = session_header.unwrap_or_else(|| Uuid::now_v7().to_string());
1409 let (budget, max_rungs, speculation, serve_threshold, elastic) =
1410 match state.config.routing.as_ref() {
1411 Some(cfg) => (
1412 cfg.budget.per_request_usd,
1413 cfg.escalation.max_rungs_per_request,
1414 cfg.escalation.speculation,
1415 cfg.escalation.serve_threshold,
1416 cfg.escalation.elastic.as_ref(),
1417 ),
1418 None => (None, 3, 0, None, None),
1419 };
1420 let serve_threshold = state
1423 .adaptive
1424 .as_ref()
1425 .and_then(|a| a.lock().ok().map(|g| g.threshold()))
1426 .or(serve_threshold);
1427
1428 let preset = routing_mode.preset();
1431 let max_rungs = if let Some(delta) = preset.max_rungs_delta {
1432 (max_rungs as i32 + delta).max(1) as u32
1433 } else {
1434 max_rungs
1435 };
1436 let speculation = preset.speculation.unwrap_or(speculation);
1437
1438 let bandit_ctx = crate::bandit::ContextBucket::from_features(&features);
1442
1443 let (greedy_rung, base_policy_id, ts_propensity) = {
1447 let (chosen, ts_p) = state
1448 .bandit
1449 .as_ref()
1450 .and_then(|b| b.lock().ok())
1451 .map(|mut b| {
1452 b.choose_start_with_propensity(&bandit_ctx, &route.ladder, &state.config.prices)
1453 })
1454 .unwrap_or((0, None));
1455 let policy = if ts_p.is_some() {
1456 "bandit@v2-ts".to_owned()
1457 } else if chosen > 0 {
1458 "bandit@v1".to_owned()
1459 } else {
1460 "static-ladder@v0".to_owned()
1461 };
1462 (chosen, policy, ts_p)
1463 };
1464
1465 let exploration_epsilon = state
1470 .config
1471 .routing
1472 .as_ref()
1473 .and_then(|cfg| cfg.escalation.exploration.as_ref())
1474 .map(|e| e.epsilon);
1475
1476 let (start_rung, policy_id, explore_flag, propensity) = if ts_propensity.is_some() {
1477 if exploration_epsilon.is_some() {
1480 tracing::warn!(
1481 "bandit.algorithm = thompson already logs propensities; \
1482 [escalation.exploration] epsilon is ignored"
1483 );
1484 }
1485 (greedy_rung, base_policy_id, false, ts_propensity)
1486 } else if let Some(epsilon) = exploration_epsilon {
1487 let k = route.ladder.len().max(1);
1488 let u = u01(Uuid::now_v7().as_u128());
1490 let (chosen, eps_branch) = if u < epsilon {
1491 let idx = ((u / epsilon) * k as f64) as u32;
1493 (idx.min(k as u32 - 1), true)
1494 } else {
1495 (greedy_rung, false)
1496 };
1497 let p = epsilon_propensity(chosen, greedy_rung, epsilon, k);
1498 (chosen, format!("{base_policy_id}+eps"), eps_branch, Some(p))
1499 } else {
1500 (greedy_rung, base_policy_id, false, None)
1501 };
1502
1503 let promotion = state
1508 .promoter
1509 .as_ref()
1510 .map(|p| p.decide(&tenant, &session_id, std::time::Instant::now()))
1511 .unwrap_or(crate::affinity::Decision::Cold);
1512 let start_rung = start_rung.max(promotion.start_rung());
1513 let promo_key = state
1516 .promoter
1517 .as_ref()
1518 .map(|_| (tenant.clone(), session_id.clone()));
1519
1520 let start_rung = if preset.start_at_top {
1523 route.ladder.len().saturating_sub(1) as u32
1524 } else {
1525 start_rung
1526 };
1527
1528 let speculation = match state
1534 .config
1535 .routing
1536 .as_ref()
1537 .and_then(|cfg| cfg.escalation.speculation_band)
1538 {
1539 Some([lo, hi]) if speculation > 0 => {
1540 let estimate = state
1541 .bandit
1542 .as_ref()
1543 .and_then(|b| b.lock().ok())
1544 .and_then(|b| b.pass_estimate(&bandit_ctx, start_rung));
1545 match estimate {
1546 Some(p) if p < lo || p > hi => {
1547 metrics::counter!("firstpass_speculation_skipped_total").increment(1);
1548 0
1549 }
1550 _ => speculation,
1551 }
1552 }
1553 _ => speculation,
1554 };
1555
1556 if state.bandit.is_some() {
1558 metrics::counter!(
1559 "firstpass_bandit_start_rung",
1560 "rung" => start_rung.to_string()
1561 )
1562 .increment(1);
1563 }
1564
1565 let mut base_request = base_request;
1566 base_request.cache_prefix = state
1570 .config
1571 .routing
1572 .as_ref()
1573 .is_some_and(|r| r.escalation.prompt_cache);
1574 let ctx = EnforceCtx {
1575 condense: routing_cfg_condense(state),
1576 ladder: &route.ladder,
1577 gates: &gates,
1578 health: &state.gate_health,
1579 base_request: &base_request,
1580 providers: &state.providers,
1581 auth: &auth,
1582 prices: &state.config.prices,
1583 budget_per_request_usd: budget,
1584 max_rungs,
1585 speculation,
1586 serve_threshold,
1587 elastic,
1588 features,
1589 start_rung,
1590 tenant_id: tenant,
1593 session_id,
1594 prompt_hash: prompt_hash(&state.config.prompt_salt, body),
1595 api: api.to_owned(),
1596 policy_id,
1597 };
1598
1599 if let Some(cache) = state.verified_cache.as_ref()
1604 && let Some(entry) = cache
1605 .get(
1606 &crate::verified_cache::VerifiedCache::compose_key(
1607 &ctx.tenant_id,
1608 &ctx.prompt_hash,
1609 &route.ladder,
1610 ),
1611 std::time::Instant::now(),
1612 )
1613 .await
1614 {
1615 metrics::counter!("firstpass_cache_hits_total").increment(1);
1616 let trace = cache_hit_trace(&ctx, &entry, mode_now());
1617 offer_trace(&state.traces, state.spill.as_ref(), trace);
1618 return Ok(entry.response);
1619 }
1620
1621 let (outcome, mut trace) = route_enforce(ctx).await;
1622
1623 trace.route_ix = Some(u32::try_from(route_ix).unwrap_or(0));
1627 trace.policy.explore = explore_flag;
1630 trace.policy.propensity = propensity;
1631 if routing_mode != RoutingMode::Balanced {
1634 trace.policy.mode_profile = Some(routing_mode.as_str().to_owned());
1635 }
1636
1637 if let Some(bandit) = state.bandit.as_ref()
1641 && let Ok(mut b) = bandit.lock()
1642 {
1643 for attempt in &trace.attempts {
1644 b.observe(&bandit_ctx, attempt.rung, attempt.verdict);
1645 }
1646 }
1647
1648 if let Some(promoter) = state.promoter.as_ref()
1653 && let Some((tenant_key, session_key)) = promo_key.as_ref()
1654 && let Some(served) = trace.final_.served_rung
1655 {
1656 promoter.record(
1657 tenant_key,
1658 session_key,
1659 served,
1660 served > start_rung,
1661 std::time::Instant::now(),
1662 );
1663 }
1664
1665 if let Some(predictor) = state.predictor.as_ref()
1671 && let Ok(mut p) = predictor.lock()
1672 {
1673 let predicted = p.predict(&trace.request.features, start_rung);
1675 for attempt in &trace.attempts {
1676 match attempt.verdict {
1677 Verdict::Pass => p.update(&trace.request.features, attempt.rung, true),
1678 Verdict::Fail => p.update(&trace.request.features, attempt.rung, false),
1679 Verdict::Abstain => {} }
1681 }
1682 trace.predicted_pass = Some(predicted);
1683 metrics::histogram!("firstpass_predictor_pass_prob").record(predicted);
1684 }
1685
1686 if let Some(probe_cfg) = state
1693 .config
1694 .routing
1695 .as_ref()
1696 .and_then(|c| c.escalation.probe)
1697 && u01(Uuid::now_v7().as_u128()) < probe_cfg.sample_rate
1698 {
1699 let probe_rung = (start_rung as usize).min(route.ladder.len().saturating_sub(1));
1701 if let Some(probe_model_str) = route.ladder.get(probe_rung).cloned() {
1702 let probe_provider = ModelRef::parse(&probe_model_str)
1703 .ok()
1704 .and_then(|m| state.providers.get(&m.provider));
1705
1706 if let Some(probe_provider) = probe_provider {
1707 let mut join_set = tokio::task::JoinSet::new();
1710 for _ in 0..probe_cfg.k {
1711 let mut probe_req = base_request.clone();
1712 probe_req.model = probe_model_str.clone();
1713 let probe_auth = auth.clone();
1714 let prov = probe_provider.clone();
1715 join_set.spawn(async move { prov.complete(&probe_req, &probe_auth).await });
1716 }
1717
1718 let fail_closed_owned: std::collections::HashSet<String> = gates
1721 .iter()
1722 .filter(|g| g.abstain_fails_closed())
1723 .map(|g| g.id().to_owned())
1724 .collect();
1725
1726 let mut gate_pass_count = 0u32;
1727 let mut probe_cost_usd = 0.0f64;
1728
1729 while let Some(task_result) = join_set.join_next().await {
1730 let Ok(Ok(probe_resp)) = task_result else {
1731 continue; };
1733 probe_cost_usd += state
1736 .config
1737 .prices
1738 .cost_usd_with_cache(
1739 &probe_model_str,
1740 probe_resp.in_tokens,
1741 probe_resp.cache_write_tokens,
1742 probe_resp.cache_read_tokens,
1743 probe_resp.out_tokens,
1744 )
1745 .unwrap_or(0.0);
1746
1747 let mut probe_gate_req = base_request.clone();
1748 probe_gate_req.model = probe_model_str.clone();
1749
1750 let mut probe_gate_results = Vec::with_capacity(gates.len());
1753 for g in &gates {
1754 if !state.gate_health.enabled(&trace.tenant_id, g.id()) {
1756 continue;
1757 }
1758 let r = g.evaluate(&probe_gate_req, &probe_resp).await;
1759 probe_gate_results.push(r);
1761 }
1762
1763 let fail_closed_refs: std::collections::HashSet<&str> =
1764 fail_closed_owned.iter().map(|s| s.as_str()).collect();
1765 let verdict = aggregate_with_policy(&probe_gate_results, &fail_closed_refs);
1766 let passes = match serve_threshold {
1769 None => verdict == Verdict::Pass,
1770 Some(t) => crate::calibrate::gate_score(&probe_gate_results, verdict) >= t,
1771 };
1772 if passes {
1773 gate_pass_count += 1;
1774 }
1775 }
1776
1777 let regime = ProbeRegime::classify(gate_pass_count, probe_cfg.k);
1778 let regime_label = match regime {
1779 ProbeRegime::ConfidentPass => "confident_pass",
1780 ProbeRegime::ConfidentFail => "confident_fail",
1781 ProbeRegime::Ambiguous => "ambiguous",
1782 };
1783 metrics::counter!(
1784 "firstpass_probe_regime_total",
1785 "regime" => regime_label
1786 )
1787 .increment(1);
1788 metrics::gauge!("firstpass_probe_cost_usd_total").increment(probe_cost_usd);
1789 trace.probe = Some(ProbeSignal {
1790 k: probe_cfg.k,
1791 gate_pass_count,
1792 regime,
1793 probe_cost_usd,
1794 });
1795 }
1796 }
1797 }
1798 if let Some(cache) = state.verified_cache.as_ref()
1807 && let EngineOutcome::Served(resp) = &outcome
1808 && crate::verified_cache::is_cacheable(&trace)
1809 {
1810 let now = std::time::Instant::now();
1811 cache
1812 .put(
1813 &crate::verified_cache::VerifiedCache::compose_key(
1814 &trace.tenant_id,
1815 &trace.request.prompt_hash,
1816 &route.ladder,
1817 ),
1818 trace.trace_id,
1819 crate::verified_cache::Entry::new(resp.clone(), &trace, now),
1820 now,
1821 )
1822 .await;
1823 metrics::counter!("firstpass_cache_stores_total").increment(1);
1824 }
1825
1826 offer_trace(&state.traces, state.spill.as_ref(), trace);
1828
1829 match outcome {
1830 EngineOutcome::Served(resp) => Ok(resp),
1831 EngineOutcome::Failed(msg) => Err(ProxyError::Engine(msg)),
1832 }
1833}
1834
1835#[allow(clippy::too_many_arguments)]
1838async fn enforce_pipeline(
1839 state: &AppState,
1840 headers: &HeaderMap,
1841 body: &Bytes,
1842 features: Features,
1843 route: &Route,
1844 route_ix: usize,
1846 session_header: Option<String>,
1847 tenant: String,
1848 routing_mode: RoutingMode,
1849) -> Result<Value, ProxyError> {
1850 let Some(base_request) = parse_model_request(body) else {
1851 return Err(ProxyError::BadRequest(
1852 "request body is not a valid Anthropic Messages request".to_owned(),
1853 ));
1854 };
1855 let auth = Auth::from_headers(headers);
1856 let resp = enforce_pipeline_inner(
1857 state,
1858 body,
1859 base_request,
1860 auth,
1861 features,
1862 route,
1863 session_header,
1864 tenant,
1865 "anthropic.messages",
1866 routing_mode,
1867 route_ix,
1868 )
1869 .await?;
1870 Ok(anthropic_response_json(&resp))
1871}
1872
1873#[allow(clippy::too_many_arguments)]
1876async fn enforce_pipeline_openai(
1877 state: &AppState,
1878 headers: &HeaderMap,
1879 body: &Bytes,
1880 features: Features,
1881 route: &Route,
1882 route_ix: usize,
1884 session_header: Option<String>,
1885 tenant: String,
1886 routing_mode: RoutingMode,
1887) -> Result<Value, ProxyError> {
1888 enforce_pipeline_openai_as(
1889 state,
1890 headers,
1891 body,
1892 body,
1893 "openai.chat_completions",
1894 features,
1895 route,
1896 route_ix,
1897 session_header,
1898 tenant,
1899 routing_mode,
1900 )
1901 .await
1902}
1903
1904#[allow(clippy::too_many_arguments)]
1913async fn enforce_pipeline_openai_as(
1914 state: &AppState,
1915 headers: &HeaderMap,
1916 body: &Bytes,
1917 receipt_body: &Bytes,
1918 api: &str,
1919 features: Features,
1920 route: &Route,
1921 route_ix: usize,
1922 session_header: Option<String>,
1923 tenant: String,
1924 routing_mode: RoutingMode,
1925) -> Result<Value, ProxyError> {
1926 let providers = &state.providers;
1928 let all_openai = route.ladder.iter().all(|rung| {
1929 let pid = rung.split('/').next().unwrap_or_default();
1930 providers
1931 .get(pid)
1932 .is_some_and(|p| p.carries_structured_verbatim(Dialect::Openai))
1933 });
1934 let Some(base_request) = parse_openai_request(body, all_openai) else {
1935 return Err(ProxyError::BadRequest(
1936 "request body is not a valid OpenAI Chat Completions request".to_owned(),
1937 ));
1938 };
1939 let auth = Auth::from_headers(headers);
1940 let resp = enforce_pipeline_inner(
1941 state,
1942 receipt_body,
1943 base_request,
1944 auth,
1945 features,
1946 route,
1947 session_header,
1948 tenant,
1949 api,
1950 routing_mode,
1951 route_ix,
1952 )
1953 .await?;
1954 Ok(openai_response_json(&resp))
1955}
1956
1957const SSE_KEEPALIVE_EVERY: Duration = Duration::from_secs(5);
1959
1960fn sse_keepalive_response(
1969 rx: tokio::sync::oneshot::Receiver<Result<Value, ProxyError>>,
1970 format_message: fn(&Value) -> String,
1971) -> Response {
1972 let mut ticks = tokio::time::interval(SSE_KEEPALIVE_EVERY);
1973 ticks.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
1974 ticks.reset(); let stream = KeepaliveStream {
1976 rx: Some(rx),
1977 ticks,
1978 format_message,
1979 };
1980 (
1981 axum::http::StatusCode::OK,
1982 [(
1983 axum::http::header::CONTENT_TYPE,
1984 "text/event-stream; charset=utf-8",
1985 )],
1986 axum::body::Body::from_stream(stream),
1987 )
1988 .into_response()
1989}
1990
1991struct KeepaliveStream {
1994 rx: Option<tokio::sync::oneshot::Receiver<Result<Value, ProxyError>>>,
1996 ticks: tokio::time::Interval,
1997 format_message: fn(&Value) -> String,
1999}
2000
2001impl futures_core::Stream for KeepaliveStream {
2002 type Item = Result<Bytes, std::convert::Infallible>;
2003
2004 fn poll_next(
2005 mut self: std::pin::Pin<&mut Self>,
2006 cx: &mut std::task::Context<'_>,
2007 ) -> std::task::Poll<Option<Self::Item>> {
2008 use std::task::Poll;
2009 let Some(rx) = self.rx.as_mut() else {
2010 return Poll::Ready(None); };
2012 if let Poll::Ready(out) = std::pin::Pin::new(rx).poll(cx) {
2013 let fmt = self.format_message;
2014 let frame = match out {
2015 Ok(Ok(message)) => fmt(&message),
2016 Ok(Err(e)) => sse_error_event(&e),
2017 Err(_) => sse_error_event(&ProxyError::Internal(
2018 "enforce pipeline task dropped".to_owned(),
2019 )),
2020 };
2021 self.rx = None;
2022 return Poll::Ready(Some(Ok(Bytes::from(frame))));
2023 }
2024 if self.ticks.poll_tick(cx).is_ready() {
2025 return Poll::Ready(Some(Ok(Bytes::from_static(b": firstpass routing\n\n"))));
2026 }
2027 Poll::Pending
2028 }
2029}
2030
2031fn sse_error_event(e: &ProxyError) -> String {
2034 let mut out = String::new();
2035 sse_event(
2036 &mut out,
2037 "error",
2038 &serde_json::json!({
2039 "type": "error",
2040 "error": { "type": "api_error", "message": e.client_message() }
2041 }),
2042 );
2043 out
2044}
2045
2046fn parse_model_request(body: &[u8]) -> Option<ModelRequest> {
2055 let json: Value = serde_json::from_slice(body).ok()?;
2056 let raw = json.clone();
2057 let messages_json = json.get("messages")?.as_array()?;
2058 let messages = messages_json
2059 .iter()
2060 .map(|m| ChatMessage {
2061 role: m
2062 .get("role")
2063 .and_then(Value::as_str)
2064 .unwrap_or("user")
2065 .to_owned(),
2066 content: m
2067 .get("content")
2068 .cloned()
2069 .unwrap_or_else(|| Value::String(String::new())),
2070 })
2071 .collect();
2072 let system = json
2073 .get("system")
2074 .and_then(Value::as_str)
2075 .map(str::to_owned);
2076 let max_tokens = json
2077 .get("max_tokens")
2078 .and_then(Value::as_u64)
2079 .and_then(|n| u32::try_from(n).ok())
2080 .unwrap_or(1024);
2081 let tools = json.get("tools").cloned().unwrap_or(Value::Null);
2082 Some(ModelRequest {
2083 model: json
2084 .get("model")
2085 .and_then(Value::as_str)
2086 .unwrap_or_default()
2087 .to_owned(),
2088 system,
2089 messages,
2090 max_tokens,
2091 tools,
2092 raw,
2093 cache_prefix: false,
2094 })
2095}
2096
2097fn anthropic_response_json(resp: &ModelResponse) -> Value {
2107 let content = resp
2108 .raw
2109 .get("content")
2110 .filter(|c| c.is_array())
2111 .cloned()
2112 .unwrap_or_else(|| serde_json::json!([{ "type": "text", "text": resp.text }]));
2113 serde_json::json!({
2114 "id": format!("msg_{}", Uuid::now_v7()),
2115 "type": "message",
2116 "role": "assistant",
2117 "model": resp.model,
2118 "content": content,
2119 "usage": { "input_tokens": resp.in_tokens, "output_tokens": resp.out_tokens },
2120 })
2121}
2122
2123fn sse_event(out: &mut String, event: &str, data: &Value) {
2125 out.push_str("event: ");
2126 out.push_str(event);
2127 out.push_str("\ndata: ");
2128 out.push_str(&data.to_string());
2129 out.push_str("\n\n");
2130}
2131
2132fn anthropic_sse_from_message(message: &Value) -> String {
2139 let mut out = String::new();
2140
2141 let mut start_msg = message.clone();
2143 start_msg["content"] = Value::Array(Vec::new());
2144 sse_event(
2145 &mut out,
2146 "message_start",
2147 &serde_json::json!({ "type": "message_start", "message": start_msg }),
2148 );
2149
2150 let empty = Vec::new();
2151 let blocks = message
2152 .get("content")
2153 .and_then(Value::as_array)
2154 .unwrap_or(&empty);
2155 for (i, block) in blocks.iter().enumerate() {
2156 match block.get("type").and_then(Value::as_str) {
2157 Some("tool_use") => {
2158 let mut shell = block.clone();
2160 shell["input"] = serde_json::json!({});
2161 sse_event(
2162 &mut out,
2163 "content_block_start",
2164 &serde_json::json!({ "type": "content_block_start", "index": i, "content_block": shell }),
2165 );
2166 let input_json = block
2167 .get("input")
2168 .map_or_else(|| "{}".to_owned(), std::string::ToString::to_string);
2169 sse_event(
2170 &mut out,
2171 "content_block_delta",
2172 &serde_json::json!({ "type": "content_block_delta", "index": i,
2173 "delta": { "type": "input_json_delta", "partial_json": input_json } }),
2174 );
2175 }
2176 _ => {
2177 let text = block.get("text").and_then(Value::as_str).unwrap_or("");
2179 sse_event(
2180 &mut out,
2181 "content_block_start",
2182 &serde_json::json!({ "type": "content_block_start", "index": i,
2183 "content_block": { "type": "text", "text": "" } }),
2184 );
2185 sse_event(
2186 &mut out,
2187 "content_block_delta",
2188 &serde_json::json!({ "type": "content_block_delta", "index": i,
2189 "delta": { "type": "text_delta", "text": text } }),
2190 );
2191 }
2192 }
2193 sse_event(
2194 &mut out,
2195 "content_block_stop",
2196 &serde_json::json!({ "type": "content_block_stop", "index": i }),
2197 );
2198 }
2199
2200 let out_tokens = message
2201 .pointer("/usage/output_tokens")
2202 .cloned()
2203 .unwrap_or_else(|| Value::from(0));
2204 sse_event(
2205 &mut out,
2206 "message_delta",
2207 &serde_json::json!({ "type": "message_delta", "delta": { "stop_reason": "end_turn" },
2208 "usage": { "output_tokens": out_tokens } }),
2209 );
2210 sse_event(
2211 &mut out,
2212 "message_stop",
2213 &serde_json::json!({ "type": "message_stop" }),
2214 );
2215 out
2216}
2217
2218async fn observe_passthrough(
2220 state: AppState,
2221 headers: HeaderMap,
2222 body: Bytes,
2223 session_header: Option<String>,
2224 tenant: String,
2225) -> Response {
2226 if is_stream_request(&body) {
2228 return observe_stream(state, headers, body, session_header, tenant).await;
2229 }
2230 let start = Instant::now();
2231 let result = forward_anthropic(
2232 &state.http,
2233 &state.config.upstream_anthropic,
2234 &headers,
2235 body.clone(),
2236 )
2237 .await;
2238 let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
2239
2240 match result {
2241 Ok((status, resp_headers, resp_body)) => {
2242 spawn_trace(
2246 &state,
2247 body,
2248 Some(resp_body.clone()),
2249 latency_ms,
2250 session_header,
2251 tenant,
2252 );
2253 (status, resp_headers, resp_body).into_response()
2254 }
2255 Err(err) => {
2256 spawn_trace(&state, body, None, latency_ms, session_header, tenant);
2257 err.into_response()
2258 }
2259 }
2260}
2261
2262async fn observe_stream(
2271 state: AppState,
2272 headers: HeaderMap,
2273 body: Bytes,
2274 session_header: Option<String>,
2275 tenant: String,
2276) -> Response {
2277 let start = Instant::now();
2278 let result = forward_anthropic_streaming(
2279 &state.http,
2280 &state.config.upstream_anthropic,
2281 &headers,
2282 body.clone(),
2283 )
2284 .await;
2285 let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
2286
2287 match result {
2288 Ok((status, resp_headers, response)) => {
2289 spawn_stream_trace(&state, body, latency_ms, session_header, tenant);
2290 let stream_body = Body::from_stream(response.bytes_stream());
2291 (status, resp_headers, stream_body).into_response()
2292 }
2293 Err(err) => {
2294 spawn_trace(&state, body, None, latency_ms, session_header, tenant);
2295 err.into_response()
2296 }
2297 }
2298}
2299
2300fn spawn_stream_trace(
2302 state: &AppState,
2303 req_body: Bytes,
2304 latency_ms: u64,
2305 session_header: Option<String>,
2306 tenant: String,
2307) {
2308 let config = state.config.clone();
2309 let traces = state.traces.clone();
2310 let spill = state.spill.clone();
2311 tokio::spawn(async move {
2312 let mut trace =
2313 build_stream_trace(&config, &req_body, latency_ms, session_header.as_deref());
2314 trace.tenant_id = tenant;
2316 offer_trace(&traces, spill.as_ref(), trace);
2317 });
2318}
2319
2320fn spawn_trace(
2325 state: &AppState,
2326 req_body: Bytes,
2327 resp_body: Option<Bytes>,
2328 latency_ms: u64,
2329 session_header: Option<String>,
2330 tenant: String,
2331) {
2332 let config = state.config.clone();
2333 let traces = state.traces.clone();
2334 let spill = state.spill.clone();
2335 tokio::spawn(async move {
2336 let mut trace = match resp_body {
2337 Some(resp) => build_trace(
2338 &config,
2339 &req_body,
2340 &resp,
2341 latency_ms,
2342 session_header.as_deref(),
2343 ),
2344 None => build_error_trace(&config, &req_body, latency_ms, session_header.as_deref()),
2345 };
2346 trace.tenant_id = tenant;
2348 offer_trace(&traces, spill.as_ref(), trace);
2349 });
2350}
2351
2352fn session_id(session_header: Option<&str>, trace_id: Uuid) -> String {
2354 session_header
2355 .map(str::to_owned)
2356 .unwrap_or_else(|| trace_id.to_string())
2357}
2358
2359fn prompt_hash(salt: &str, body: &[u8]) -> String {
2362 let mut salted = Vec::with_capacity(salt.len() + body.len());
2363 salted.extend_from_slice(salt.as_bytes());
2364 salted.extend_from_slice(body);
2365 sha256_hex(&salted)
2366}
2367
2368fn request_features(body: &[u8]) -> (Option<String>, u32, bool) {
2372 let Ok(json) = serde_json::from_slice::<Value>(body) else {
2373 return (None, 0, false);
2374 };
2375 let model = json.get("model").and_then(Value::as_str).map(str::to_owned);
2376 let tool_count = json
2377 .get("tools")
2378 .and_then(Value::as_array)
2379 .map_or(0, |tools| u32::try_from(tools.len()).unwrap_or(u32::MAX));
2380 let has_images = json
2381 .get("messages")
2382 .and_then(Value::as_array)
2383 .is_some_and(|messages| messages.iter().any(message_has_image));
2384 (model, tool_count, has_images)
2385}
2386
2387fn message_has_image(message: &Value) -> bool {
2389 message
2390 .get("content")
2391 .and_then(Value::as_array)
2392 .is_some_and(|blocks| {
2393 blocks
2394 .iter()
2395 .any(|block| block.get("type").and_then(Value::as_str) == Some("image"))
2396 })
2397}
2398
2399#[derive(Debug, Default, Clone, PartialEq, Eq)]
2401struct ResponseUsage {
2402 model: Option<String>,
2403 in_tokens: u64,
2404 cache_write_tokens: u64,
2405 cache_read_tokens: u64,
2406 out_tokens: u64,
2407}
2408
2409fn response_usage(body: &[u8]) -> ResponseUsage {
2417 let Ok(json) = serde_json::from_slice::<Value>(body) else {
2418 return ResponseUsage::default();
2419 };
2420 let u = |ptr: &str| json.pointer(ptr).and_then(Value::as_u64).unwrap_or(0);
2421 ResponseUsage {
2422 model: json.get("model").and_then(Value::as_str).map(str::to_owned),
2423 in_tokens: u("/usage/input_tokens"),
2424 cache_write_tokens: u("/usage/cache_creation_input_tokens"),
2425 cache_read_tokens: u("/usage/cache_read_input_tokens"),
2426 out_tokens: u("/usage/output_tokens"),
2427 }
2428}
2429
2430fn build_trace(
2432 config: &ProxyConfig,
2433 req_body: &Bytes,
2434 resp_body: &Bytes,
2435 latency_ms: u64,
2436 session_header: Option<&str>,
2437) -> Trace {
2438 let (req_model, tool_count, has_images) = request_features(req_body);
2439 let usage = response_usage(resp_body);
2440 let (resp_model, in_tokens, out_tokens) = (usage.model, usage.in_tokens, usage.out_tokens);
2441 let (cache_write_tokens, cache_read_tokens) =
2442 (usage.cache_write_tokens, usage.cache_read_tokens);
2443 let model = resp_model
2444 .or(req_model)
2445 .unwrap_or_else(|| "unknown".to_owned());
2446
2447 let cost_usd = config
2448 .prices
2449 .cost_usd_with_cache(
2450 &format!("anthropic/{model}"),
2451 in_tokens,
2452 cache_write_tokens,
2453 cache_read_tokens,
2454 out_tokens,
2455 )
2456 .unwrap_or(0.0);
2457
2458 let attempt = Attempt {
2459 rung: 0,
2460 model,
2461 provider: "anthropic".to_owned(),
2462 in_tokens,
2463 cache_write_tokens,
2464 cache_read_tokens,
2465 out_tokens,
2466 cost_usd,
2467 latency_ms,
2468 gates: Vec::new(),
2469 verdict: Verdict::Pass,
2470 };
2471
2472 let mut trace = base_trace(config, req_body, latency_ms, session_header);
2473 trace.request.features.prompt_token_bucket = token_bucket(in_tokens);
2474 trace.request.features.tool_count = tool_count;
2475 trace.request.features.has_images = has_images;
2476 trace.attempts.push(attempt);
2477 trace.final_ = FinalOutcome {
2478 served_rung: Some(0),
2479 served_from: ServedFrom::Attempt,
2480 total_cost_usd: cost_usd,
2481 gate_cost_usd: 0.0,
2482 total_latency_ms: latency_ms,
2483 escalations: 0,
2484 counterfactual_baseline_usd: cost_usd,
2485 savings_usd: 0.0,
2486 cache_source: None,
2487 };
2488 trace.recompute_savings();
2489 trace
2490}
2491
2492fn build_stream_trace(
2496 config: &ProxyConfig,
2497 req_body: &Bytes,
2498 latency_ms: u64,
2499 session_header: Option<&str>,
2500) -> Trace {
2501 let (req_model, tool_count, has_images) = request_features(req_body);
2502 let model = req_model.unwrap_or_else(|| "unknown".to_owned());
2503
2504 let attempt = Attempt {
2505 rung: 0,
2506 model,
2507 provider: "anthropic".to_owned(),
2508 in_tokens: 0,
2509 cache_write_tokens: 0,
2511 cache_read_tokens: 0,
2512 out_tokens: 0,
2513 cost_usd: 0.0,
2514 latency_ms,
2515 gates: Vec::new(),
2516 verdict: Verdict::Pass,
2517 };
2518
2519 let mut trace = base_trace(config, req_body, latency_ms, session_header);
2520 trace.request.features.tool_count = tool_count;
2521 trace.request.features.has_images = has_images;
2522 trace.attempts.push(attempt);
2523 trace.final_ = FinalOutcome {
2524 served_rung: Some(0),
2525 served_from: ServedFrom::Attempt,
2526 total_cost_usd: 0.0,
2527 gate_cost_usd: 0.0,
2528 total_latency_ms: latency_ms,
2529 escalations: 0,
2530 counterfactual_baseline_usd: 0.0,
2531 savings_usd: 0.0,
2532 cache_source: None,
2533 };
2534 trace.recompute_savings();
2535 trace
2536}
2537
2538fn build_error_trace(
2542 config: &ProxyConfig,
2543 req_body: &Bytes,
2544 latency_ms: u64,
2545 session_header: Option<&str>,
2546) -> Trace {
2547 let (_, tool_count, has_images) = request_features(req_body);
2548 let mut trace = base_trace(config, req_body, latency_ms, session_header);
2549 trace.request.features.tool_count = tool_count;
2550 trace.request.features.has_images = has_images;
2551 trace.final_ = FinalOutcome {
2552 served_rung: None,
2553 served_from: ServedFrom::Error,
2554 total_cost_usd: 0.0,
2555 gate_cost_usd: 0.0,
2556 total_latency_ms: latency_ms,
2557 escalations: 0,
2558 counterfactual_baseline_usd: 0.0,
2559 savings_usd: 0.0,
2560 cache_source: None,
2561 };
2562 trace.recompute_savings();
2563 trace
2564}
2565
2566fn base_trace(
2569 config: &ProxyConfig,
2570 req_body: &Bytes,
2571 latency_ms: u64,
2572 session_header: Option<&str>,
2573) -> Trace {
2574 let trace_id = Uuid::now_v7();
2575 let mut features = Features::new(TaskKind::Other);
2576 features.hour_bucket = hour_bucket(jiff::Timestamp::now());
2577
2578 Trace {
2579 trace_id,
2580 prev_hash: GENESIS_HASH.to_owned(),
2581 tenant_id: config.tenant_id.clone(),
2582 session_id: session_id(session_header, trace_id),
2583 ts: jiff::Timestamp::now(),
2584 mode: Mode::Observe,
2585 policy: PolicyRef {
2586 id: "observe-passthrough@v0".to_owned(),
2587 explore: false,
2588 propensity: None,
2589 mode_profile: None,
2590 },
2591 request: RequestInfo {
2592 api: "anthropic.messages".to_owned(),
2593 prompt_hash: prompt_hash(&config.prompt_salt, req_body),
2594 features,
2595 },
2596 attempts: Vec::new(),
2597 deferred: Vec::new(),
2598 final_: FinalOutcome {
2599 served_rung: None,
2600 served_from: ServedFrom::Error,
2601 total_cost_usd: 0.0,
2602 gate_cost_usd: 0.0,
2603 total_latency_ms: latency_ms,
2604 escalations: 0,
2605 counterfactual_baseline_usd: 0.0,
2606 savings_usd: 0.0,
2607 cache_source: None,
2608 },
2609 probe: None,
2610 rollout: None,
2611 shadow: None,
2612 route_ix: None,
2613 predicted_pass: None,
2614 elastic: None,
2615 }
2616}
2617
2618fn openai_messages_have_tool_calls(body: &[u8]) -> bool {
2623 serde_json::from_slice::<Value>(body)
2624 .ok()
2625 .and_then(|json| {
2626 json.get("messages").and_then(Value::as_array).map(|msgs| {
2627 msgs.iter().any(|m| {
2628 m.get("tool_calls").is_some()
2629 || m.get("role").and_then(Value::as_str) == Some("tool")
2630 })
2631 })
2632 })
2633 .unwrap_or(false)
2634}
2635
2636fn openai_has_http_images(body: &[u8]) -> bool {
2640 serde_json::from_slice::<Value>(body)
2641 .ok()
2642 .and_then(|json| {
2643 json.get("messages").and_then(Value::as_array).map(|msgs| {
2644 msgs.iter().any(|m| {
2645 m.get("content")
2646 .and_then(Value::as_array)
2647 .is_some_and(|parts| {
2648 parts.iter().any(|p| {
2649 p.get("type").and_then(Value::as_str) == Some("image_url")
2650 && p.pointer("/image_url/url")
2651 .and_then(Value::as_str)
2652 .is_some_and(|u| {
2653 u.starts_with("http://") || u.starts_with("https://")
2654 })
2655 })
2656 })
2657 })
2658 })
2659 })
2660 .unwrap_or(false)
2661}
2662
2663fn openai_messages_have_images(body: &[u8]) -> bool {
2666 serde_json::from_slice::<Value>(body)
2667 .ok()
2668 .and_then(|json| {
2669 json.get("messages").and_then(Value::as_array).map(|msgs| {
2670 msgs.iter().any(|m| {
2671 m.get("content")
2672 .and_then(Value::as_array)
2673 .is_some_and(|parts| {
2674 parts
2675 .iter()
2676 .any(|p| p.get("type").and_then(Value::as_str) == Some("image_url"))
2677 })
2678 })
2679 })
2680 })
2681 .unwrap_or(false)
2682}
2683
2684fn extract_openai_features(headers: &HeaderMap, body: &[u8]) -> Features {
2687 let Ok(json) = serde_json::from_slice::<Value>(body) else {
2688 let mut f = Features::new(TaskKind::Other);
2689 f.hour_bucket = hour_bucket(jiff::Timestamp::now());
2690 return f;
2691 };
2692 let tool_count = json
2693 .get("tools")
2694 .and_then(Value::as_array)
2695 .map_or(0, |tools| u32::try_from(tools.len()).unwrap_or(u32::MAX));
2696 let has_images = openai_messages_have_images(body);
2697 let mut f = Features::new(TaskKind::Other);
2698 f.agent = header_str(headers, AGENT_HEADER);
2699 f.subagent = header_str(headers, SUBAGENT_HEADER);
2700 f.tool_count = tool_count;
2701 f.has_images = has_images;
2702 f.prompt_token_bucket = token_bucket(body.len() as u64);
2703 f.hour_bucket = hour_bucket(jiff::Timestamp::now());
2704 f
2705}
2706
2707fn parse_data_url(url: &str) -> Option<(&str, &str)> {
2711 let rest = url.strip_prefix("data:")?;
2712 let (meta, data) = rest.split_once(',')?;
2713 let media_type = meta.strip_suffix(";base64")?;
2714 Some((media_type, data))
2715}
2716
2717fn translate_openai_user_content(content: &Value) -> Option<Value> {
2720 match content {
2721 Value::String(_) => Some(content.clone()),
2723 Value::Array(parts) => {
2724 let mut blocks: Vec<Value> = Vec::with_capacity(parts.len());
2725 for part in parts {
2726 match part.get("type").and_then(Value::as_str) {
2727 Some("text") => {
2728 let text = part.get("text").and_then(Value::as_str).unwrap_or("");
2729 blocks.push(serde_json::json!({ "type": "text", "text": text }));
2730 }
2731 Some("image_url") => {
2732 let url = part.pointer("/image_url/url").and_then(Value::as_str)?;
2733 if url.starts_with("http://") || url.starts_with("https://") {
2734 return None; }
2736 let (media_type, data) = parse_data_url(url)?;
2738 blocks.push(serde_json::json!({
2739 "type": "image",
2740 "source": { "type": "base64", "media_type": media_type, "data": data }
2741 }));
2742 }
2743 _ => {} }
2745 }
2746 Some(Value::Array(blocks))
2747 }
2748 _ => Some(Value::String(String::new())),
2749 }
2750}
2751
2752fn translate_openai_tools(tools: &Value) -> Value {
2756 let Some(arr) = tools.as_array() else {
2757 return Value::Null;
2758 };
2759 let converted: Vec<Value> = arr
2760 .iter()
2761 .map(|tool| {
2762 let func = tool.get("function").unwrap_or(&Value::Null);
2763 let mut out = serde_json::json!({
2764 "name": func.get("name").cloned().unwrap_or(Value::String(String::new())),
2765 "input_schema": func.get("parameters").cloned()
2766 .unwrap_or_else(|| serde_json::json!({ "type": "object" })),
2767 });
2768 if let Some(desc) = func.get("description") {
2769 out["description"] = desc.clone();
2770 }
2771 out
2772 })
2773 .collect();
2774 Value::Array(converted)
2775}
2776
2777fn translate_openai_tool_choice(tc: &Value) -> Value {
2779 match tc {
2780 Value::String(s) => match s.as_str() {
2781 "auto" => serde_json::json!({ "type": "auto" }),
2782 "required" => serde_json::json!({ "type": "any" }),
2783 _ => serde_json::json!({ "type": "auto" }),
2785 },
2786 Value::Object(_) => {
2787 if tc.get("type").and_then(Value::as_str) == Some("function") {
2789 let name = tc.pointer("/function/name").cloned().unwrap_or(Value::Null);
2790 serde_json::json!({ "type": "tool", "name": name })
2791 } else {
2792 serde_json::json!({ "type": "auto" })
2793 }
2794 }
2795 _ => serde_json::json!({ "type": "auto" }),
2796 }
2797}
2798
2799pub fn parse_openai_request(body: &[u8], carry_raw: bool) -> Option<ModelRequest> {
2812 let json: Value = serde_json::from_slice(body).ok()?;
2813 let raw = if carry_raw { json.clone() } else { Value::Null };
2814
2815 let messages_json = json.get("messages")?.as_array()?;
2816
2817 let mut system: Option<String> = None;
2818 let mut messages: Vec<ChatMessage> = Vec::with_capacity(messages_json.len());
2819 let mut tools = Value::Null;
2820 let mut tool_choice_override: Option<Value> = None;
2821
2822 for msg in messages_json {
2823 let role = msg.get("role").and_then(Value::as_str).unwrap_or("user");
2824 match role {
2825 "system" => {
2826 if let Some(s) = msg.get("content").and_then(Value::as_str) {
2831 system = Some(s.to_owned());
2832 }
2833 }
2834 "user" => {
2835 let content_val = msg.get("content").unwrap_or(&Value::Null);
2836 let translated = translate_openai_user_content(content_val)?;
2837 messages.push(ChatMessage {
2838 role: "user".to_owned(),
2839 content: translated,
2840 });
2841 }
2842 "assistant" => {
2843 if let Some(tc_arr) = msg.get("tool_calls").and_then(Value::as_array) {
2844 let mut blocks: Vec<Value> = Vec::new();
2846 if let Some(text) = msg.get("content").and_then(Value::as_str)
2848 && !text.is_empty()
2849 {
2850 blocks.push(serde_json::json!({ "type": "text", "text": text }));
2851 }
2852 for tc in tc_arr {
2853 let id = tc.get("id").and_then(Value::as_str).unwrap_or("");
2854 let func = tc.get("function").unwrap_or(&Value::Null);
2855 let name = func.get("name").and_then(Value::as_str).unwrap_or("");
2856 let args_str = func
2857 .get("arguments")
2858 .and_then(Value::as_str)
2859 .unwrap_or("{}");
2860 let input: Value = serde_json::from_str(args_str)
2861 .unwrap_or_else(|_| serde_json::json!({}));
2862 blocks.push(serde_json::json!({
2863 "type": "tool_use",
2864 "id": id,
2865 "name": name,
2866 "input": input,
2867 }));
2868 }
2869 messages.push(ChatMessage {
2870 role: "assistant".to_owned(),
2871 content: Value::Array(blocks),
2872 });
2873 } else {
2874 let content = msg
2876 .get("content")
2877 .cloned()
2878 .unwrap_or_else(|| Value::String(String::new()));
2879 messages.push(ChatMessage {
2880 role: "assistant".to_owned(),
2881 content,
2882 });
2883 }
2884 }
2885 "tool" => {
2886 let tool_call_id = msg
2888 .get("tool_call_id")
2889 .and_then(Value::as_str)
2890 .unwrap_or("");
2891 let content = msg
2892 .get("content")
2893 .cloned()
2894 .unwrap_or_else(|| Value::String(String::new()));
2895 let result_block = serde_json::json!({
2896 "type": "tool_result",
2897 "tool_use_id": tool_call_id,
2898 "content": content,
2899 });
2900 messages.push(ChatMessage {
2901 role: "user".to_owned(),
2902 content: Value::Array(vec![result_block]),
2903 });
2904 }
2905 _ => {} }
2907 }
2908
2909 if !carry_raw {
2911 if let Some(t) = json.get("tools") {
2912 tools = translate_openai_tools(t);
2913 }
2914 if let Some(tc) = json.get("tool_choice") {
2915 tool_choice_override = Some(translate_openai_tool_choice(tc));
2916 }
2917 } else {
2918 tools = json.get("tools").cloned().unwrap_or(Value::Null);
2919 }
2920
2921 let max_tokens = json
2922 .get("max_tokens")
2923 .and_then(Value::as_u64)
2924 .and_then(|n| u32::try_from(n).ok())
2925 .unwrap_or(1024);
2926
2927 let model = json
2928 .get("model")
2929 .and_then(Value::as_str)
2930 .unwrap_or_default()
2931 .to_owned();
2932
2933 let _ = tool_choice_override; Some(ModelRequest {
2944 model,
2945 system,
2946 messages,
2947 max_tokens,
2948 tools,
2949 raw,
2950 cache_prefix: false,
2951 })
2952}
2953
2954fn extract_openai_content_and_tools(raw: &Value, text: &str) -> (Value, Option<Value>) {
2961 if let Some(blocks) = raw.get("content").and_then(Value::as_array) {
2963 let mut text_parts: Vec<&str> = Vec::new();
2964 let mut tool_calls: Vec<Value> = Vec::new();
2965 for block in blocks {
2966 match block.get("type").and_then(Value::as_str) {
2967 Some("text") => {
2968 if let Some(t) = block.get("text").and_then(Value::as_str) {
2969 text_parts.push(t);
2970 }
2971 }
2972 Some("tool_use") => {
2973 let id = block.get("id").and_then(Value::as_str).unwrap_or("");
2974 let name = block.get("name").and_then(Value::as_str).unwrap_or("");
2975 let input_str = block
2976 .get("input")
2977 .map_or_else(|| "{}".to_owned(), std::string::ToString::to_string);
2978 tool_calls.push(serde_json::json!({
2979 "id": id,
2980 "type": "function",
2981 "function": { "name": name, "arguments": input_str },
2982 }));
2983 }
2984 _ => {}
2985 }
2986 }
2987 let content_text = if tool_calls.is_empty() || !text_parts.is_empty() {
2988 Value::String(text_parts.join(""))
2989 } else {
2990 Value::Null };
2992 let tc = if tool_calls.is_empty() {
2993 None
2994 } else {
2995 Some(Value::Array(tool_calls))
2996 };
2997 return (content_text, tc);
2998 }
2999
3000 if let Some(msg) = raw.pointer("/choices/0/message") {
3002 let content = msg
3003 .get("content")
3004 .cloned()
3005 .unwrap_or(Value::String(text.to_owned()));
3006 let tc = msg.get("tool_calls").cloned();
3007 return (content, tc);
3008 }
3009
3010 (Value::String(text.to_owned()), None)
3012}
3013
3014fn openai_response_json(resp: &ModelResponse) -> Value {
3017 let (content_text, tool_calls) = extract_openai_content_and_tools(&resp.raw, &resp.text);
3018 let finish_reason = if tool_calls.is_some() {
3019 "tool_calls"
3020 } else {
3021 "stop"
3022 };
3023 let mut message = serde_json::json!({
3024 "role": "assistant",
3025 "content": content_text,
3026 });
3027 if let Some(tc) = tool_calls {
3028 message["tool_calls"] = tc;
3029 }
3030 serde_json::json!({
3031 "id": format!("chatcmpl-{}", Uuid::now_v7()),
3032 "object": "chat.completion",
3033 "created": jiff::Timestamp::now().as_second(),
3034 "model": resp.model,
3035 "choices": [{
3036 "index": 0,
3037 "message": message,
3038 "finish_reason": finish_reason,
3039 }],
3040 "usage": {
3041 "prompt_tokens": resp.in_tokens,
3042 "completion_tokens": resp.out_tokens,
3043 "total_tokens": resp.in_tokens + resp.out_tokens,
3044 }
3045 })
3046}
3047
3048fn openai_sse_from_message(message: &Value) -> String {
3052 let id = message
3053 .get("id")
3054 .and_then(Value::as_str)
3055 .unwrap_or("chatcmpl-unknown")
3056 .to_owned();
3057 let created = message.get("created").cloned().unwrap_or(Value::from(0));
3058 let model = message
3059 .get("model")
3060 .and_then(Value::as_str)
3061 .unwrap_or("unknown");
3062
3063 let choices = message.get("choices").and_then(Value::as_array);
3064 let msg = choices
3065 .and_then(|c| c.first())
3066 .and_then(|c| c.get("message"));
3067 let content = msg
3068 .and_then(|m| m.get("content"))
3069 .cloned()
3070 .unwrap_or(Value::Null);
3071 let tool_calls = msg.and_then(|m| m.get("tool_calls")).cloned();
3072 let finish_reason = choices
3073 .and_then(|c| c.first())
3074 .and_then(|c| c.get("finish_reason"))
3075 .cloned()
3076 .unwrap_or_else(|| Value::String("stop".to_owned()));
3077
3078 let mut out = String::new();
3079 let chunk = |delta: Value| {
3080 serde_json::json!({
3081 "id": id,
3082 "object": "chat.completion.chunk",
3083 "created": created,
3084 "model": model,
3085 "choices": [{ "index": 0, "delta": delta, "finish_reason": Value::Null }]
3086 })
3087 };
3088
3089 let role_chunk = chunk(serde_json::json!({ "role": "assistant", "content": "" }));
3091 out.push_str("data: ");
3092 out.push_str(&role_chunk.to_string());
3093 out.push_str("\n\n");
3094
3095 if let Value::String(text) = &content
3097 && !text.is_empty()
3098 {
3099 let content_chunk = chunk(serde_json::json!({ "content": text }));
3100 out.push_str("data: ");
3101 out.push_str(&content_chunk.to_string());
3102 out.push_str("\n\n");
3103 }
3104
3105 if let Some(tc) = tool_calls {
3107 let tc_chunk = chunk(serde_json::json!({ "tool_calls": tc }));
3108 out.push_str("data: ");
3109 out.push_str(&tc_chunk.to_string());
3110 out.push_str("\n\n");
3111 }
3112
3113 let finish_chunk = serde_json::json!({
3115 "id": id,
3116 "object": "chat.completion.chunk",
3117 "created": created,
3118 "model": model,
3119 "choices": [{ "index": 0, "delta": {}, "finish_reason": finish_reason }]
3120 });
3121 out.push_str("data: ");
3122 out.push_str(&finish_chunk.to_string());
3123 out.push_str("\n\n");
3124
3125 out.push_str("data: [DONE]\n\n");
3126 out
3127}
3128
3129#[allow(clippy::too_many_arguments)]
3134async fn handle_enforce_openai(
3135 state: &AppState,
3136 headers: &HeaderMap,
3137 body: &Bytes,
3138 features: Features,
3139 route: &Route,
3140 route_ix: usize,
3143 session_header: Option<String>,
3144 tenant: String,
3145 routing_mode: RoutingMode,
3146) -> Response {
3147 if is_stream_request(body) {
3148 let (tx, rx) = tokio::sync::oneshot::channel::<Result<Value, ProxyError>>();
3149 let (state_c, headers_c, body_c, route_c) =
3150 (state.clone(), headers.clone(), body.clone(), route.clone());
3151 tokio::spawn(async move {
3152 let out = enforce_pipeline_openai(
3153 &state_c,
3154 &headers_c,
3155 &body_c,
3156 features,
3157 &route_c,
3158 route_ix,
3159 session_header,
3160 tenant,
3161 routing_mode,
3162 )
3163 .await;
3164 let _ = tx.send(out);
3165 });
3166 return sse_keepalive_response(rx, openai_sse_from_message);
3167 }
3168 match enforce_pipeline_openai(
3169 state,
3170 headers,
3171 body,
3172 features,
3173 route,
3174 route_ix,
3175 session_header,
3176 tenant,
3177 routing_mode,
3178 )
3179 .await
3180 {
3181 Ok(message) => (axum::http::StatusCode::OK, Json(message)).into_response(),
3182 Err(e) => e.into_response(),
3183 }
3184}
3185
3186async fn observe_passthrough_openai(
3192 state: AppState,
3193 headers: HeaderMap,
3194 body: Bytes,
3195 session_header: Option<String>,
3196 tenant: String,
3197) -> Response {
3198 observe_passthrough_openai_path(
3199 state,
3200 headers,
3201 body,
3202 session_header,
3203 tenant,
3204 crate::upstream::OPENAI_CHAT_PATH,
3205 )
3206 .await
3207}
3208
3209async fn observe_passthrough_openai_path(
3215 state: AppState,
3216 headers: HeaderMap,
3217 body: Bytes,
3218 session_header: Option<String>,
3219 tenant: String,
3220 path: &'static str,
3221) -> Response {
3222 if is_stream_request(&body) {
3223 return observe_stream_openai_path(state, headers, body, session_header, tenant, path)
3224 .await;
3225 }
3226 let start = Instant::now();
3227 let result = crate::upstream::forward_openai_path(
3228 &state.http,
3229 &state.config.upstream_openai,
3230 path,
3231 &headers,
3232 body.clone(),
3233 )
3234 .await;
3235 let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
3236 match result {
3237 Ok((status, resp_headers, resp_body)) => {
3238 spawn_trace(
3239 &state,
3240 body,
3241 Some(resp_body.clone()),
3242 latency_ms,
3243 session_header,
3244 tenant,
3245 );
3246 (status, resp_headers, resp_body).into_response()
3247 }
3248 Err(err) => {
3249 spawn_trace(&state, body, None, latency_ms, session_header, tenant);
3250 err.into_response()
3251 }
3252 }
3253}
3254
3255async fn observe_stream_openai_path(
3259 state: AppState,
3260 headers: HeaderMap,
3261 body: Bytes,
3262 session_header: Option<String>,
3263 tenant: String,
3264 path: &'static str,
3265) -> Response {
3266 let start = Instant::now();
3267 let result = crate::upstream::forward_openai_streaming_path(
3268 &state.http,
3269 &state.config.upstream_openai,
3270 path,
3271 &headers,
3272 body.clone(),
3273 )
3274 .await;
3275 let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
3276 match result {
3277 Ok((status, resp_headers, response)) => {
3278 spawn_stream_trace(&state, body, latency_ms, session_header, tenant);
3279 let stream_body = Body::from_stream(response.bytes_stream());
3280 (status, resp_headers, stream_body).into_response()
3281 }
3282 Err(err) => {
3283 spawn_trace(&state, body, None, latency_ms, session_header, tenant);
3284 err.into_response()
3285 }
3286 }
3287}
3288
3289async fn chat_completions(
3294 State(state): State<AppState>,
3295 Extension(TenantId(tenant)): Extension<TenantId>,
3296 headers: HeaderMap,
3297 body: Bytes,
3298) -> Response {
3299 let session_header = header_str(&headers, SESSION_HEADER);
3300
3301 if let Some(routing) = state.config.routing.as_ref() {
3302 let features = extract_openai_features(&headers, &body);
3303 if let Some(route) = routing
3304 .route_for(&features)
3305 .filter(|r| r.mode == Mode::Enforce && !r.ladder.is_empty())
3306 {
3307 let route_ix = routing
3309 .routes
3310 .iter()
3311 .position(|r| std::ptr::eq(r, route))
3312 .unwrap_or(0);
3313 let route = route.clone();
3314 let routing_mode = resolve_mode(&headers, &route, &state.config);
3316 if routing_mode == RoutingMode::Observe {
3318 return observe_passthrough_openai(state, headers, body, session_header, tenant)
3319 .await;
3320 }
3321 if enforce_can_handle(
3322 &features,
3323 &body,
3324 routing.escalation.enforce_structured,
3325 &route.ladder,
3326 &state.providers,
3327 Dialect::Openai,
3328 ) {
3329 return handle_enforce_openai(
3330 &state,
3331 &headers,
3332 &body,
3333 features,
3334 &route,
3335 route_ix,
3336 session_header,
3337 tenant,
3338 routing_mode,
3339 )
3340 .await;
3341 }
3342 tracing::info!(
3343 "enforce route matched but OpenAI structured request can't be routed faithfully (flag/ladder); serving via observe passthrough"
3344 );
3345 }
3346 }
3347 observe_passthrough_openai(state, headers, body, session_header, tenant).await
3348}
3349
3350async fn responses(
3359 State(state): State<AppState>,
3360 Extension(TenantId(tenant)): Extension<TenantId>,
3361 headers: HeaderMap,
3362 body: Bytes,
3363) -> Response {
3364 let session_header = header_str(&headers, SESSION_HEADER);
3365
3366 if is_stream_request(&body) {
3370 return observe_passthrough_openai_path(
3371 state,
3372 headers,
3373 body,
3374 session_header,
3375 tenant,
3376 crate::upstream::OPENAI_RESPONSES_PATH,
3377 )
3378 .await;
3379 }
3380
3381 let Ok(parsed) = serde_json::from_slice::<Value>(&body) else {
3382 return observe_passthrough_openai_path(
3383 state,
3384 headers,
3385 body,
3386 session_header,
3387 tenant,
3388 crate::upstream::OPENAI_RESPONSES_PATH,
3389 )
3390 .await;
3391 };
3392 if crate::responses::has_untranslatable_content(&parsed) {
3396 return observe_passthrough_openai_path(
3397 state,
3398 headers,
3399 body,
3400 session_header,
3401 tenant,
3402 crate::upstream::OPENAI_RESPONSES_PATH,
3403 )
3404 .await;
3405 }
3406 let chat_body = Bytes::from(crate::responses::request_to_chat(&parsed).to_string());
3407
3408 if let Some(routing) = state.config.routing.as_ref() {
3409 let features = extract_openai_features(&headers, &chat_body);
3410 if let Some(route) = routing
3411 .route_for(&features)
3412 .filter(|r| r.mode == Mode::Enforce && !r.ladder.is_empty())
3413 {
3414 let route_ix = routing
3415 .routes
3416 .iter()
3417 .position(|r| std::ptr::eq(r, route))
3418 .unwrap_or(0);
3419 let route = route.clone();
3420 let routing_mode = resolve_mode(&headers, &route, &state.config);
3421 if routing_mode != RoutingMode::Observe
3422 && enforce_can_handle(
3423 &features,
3424 &chat_body,
3425 routing.escalation.enforce_structured,
3426 &route.ladder,
3427 &state.providers,
3428 Dialect::Openai,
3429 )
3430 {
3431 return match enforce_pipeline_openai_as(
3432 &state,
3433 &headers,
3434 &chat_body,
3435 &body,
3436 "openai.responses",
3437 features,
3438 &route,
3439 route_ix,
3440 session_header,
3441 tenant,
3442 routing_mode,
3443 )
3444 .await
3445 {
3446 Ok(chat) => Json(crate::responses::response_from_chat(&chat)).into_response(),
3447 Err(e) => e.into_response(),
3448 };
3449 }
3450 }
3451 }
3452 observe_passthrough_openai_path(
3453 state,
3454 headers,
3455 body,
3456 session_header,
3457 tenant,
3458 crate::upstream::OPENAI_RESPONSES_PATH,
3459 )
3460 .await
3461}
3462
3463#[cfg(test)]
3464mod tests {
3465 use bytes::Bytes;
3466
3467 use super::*;
3468
3469 fn test_config() -> ProxyConfig {
3470 ProxyConfig::from_lookup(|_| None).unwrap()
3471 }
3472
3473 #[test]
3474 fn build_trace_maps_request_and_response_fields() {
3475 let config = test_config();
3476 let req = Bytes::from_static(
3477 br#"{"model":"claude-haiku-4-5","tools":[{"name":"a"}],"messages":[]}"#,
3478 );
3479 let resp = Bytes::from_static(
3480 br#"{"model":"claude-haiku-4-5","usage":{"input_tokens":1200,"output_tokens":300}}"#,
3481 );
3482
3483 let trace = build_trace(&config, &req, &resp, 42, Some("sess-1"));
3484
3485 assert_eq!(trace.request.api, "anthropic.messages");
3486 assert_eq!(trace.session_id, "sess-1");
3487 assert_eq!(trace.attempts.len(), 1);
3488 let attempt = &trace.attempts[0];
3489 assert_eq!(attempt.model, "claude-haiku-4-5");
3490 assert_eq!(attempt.provider, "anthropic");
3491 assert_eq!(attempt.in_tokens, 1200);
3492 assert_eq!(attempt.out_tokens, 300);
3493 assert!(attempt.cost_usd > 0.0);
3494 assert_eq!(trace.request.features.tool_count, 1);
3495 assert!(!trace.request.features.has_images);
3496 assert_eq!(trace.final_.served_rung, Some(0));
3497 }
3498
3499 #[test]
3500 fn build_trace_falls_back_to_trace_id_session_when_header_absent() {
3501 let config = test_config();
3502 let req = Bytes::from_static(b"{}");
3503 let resp = Bytes::from_static(b"{}");
3504
3505 let trace = build_trace(&config, &req, &resp, 1, None);
3506
3507 assert_eq!(trace.session_id, trace.trace_id.to_string());
3508 }
3509
3510 #[test]
3511 fn build_error_trace_has_no_attempts_and_served_from_error() {
3512 let config = test_config();
3513 let req = Bytes::from_static(br#"{"model":"claude-haiku-4-5"}"#);
3514
3515 let trace = build_error_trace(&config, &req, 7, None);
3516
3517 assert!(trace.attempts.is_empty());
3518 assert_eq!(trace.final_.served_from, ServedFrom::Error);
3519 assert_eq!(trace.final_.served_rung, None);
3520 }
3521
3522 #[test]
3523 fn message_with_image_block_sets_has_images() {
3524 let req = br#"{"messages":[{"role":"user","content":[{"type":"image"}]}]}"#;
3525 let (_, _, has_images) = request_features(req);
3526 assert!(has_images);
3527 }
3528
3529 #[test]
3530 fn prompt_hash_never_contains_raw_prompt_text() {
3531 let hash = prompt_hash("salt", b"super secret prompt");
3532 assert!(!hash.contains("secret"));
3533 assert_eq!(hash.len(), 64);
3534 }
3535
3536 #[test]
3537 fn parse_model_request_preserves_content_verbatim_and_projects_text() {
3538 let body = br#"{"model":"m","system":"sys","max_tokens":50,
3539 "messages":[{"role":"user","content":[{"type":"text","text":"a"},{"type":"text","text":"b"}]},
3540 {"role":"assistant","content":"c"}]}"#;
3541 let req = parse_model_request(body).unwrap();
3542 assert_eq!(req.system.as_deref(), Some("sys"));
3543 assert_eq!(req.max_tokens, 50);
3544 assert_eq!(req.messages.len(), 2);
3545 assert_eq!(
3547 req.messages[0].content,
3548 serde_json::json!([{"type":"text","text":"a"},{"type":"text","text":"b"}])
3549 );
3550 assert_eq!(req.messages[1].content, Value::String("c".to_owned()));
3552 assert_eq!(req.messages[0].text_view(), "a\nb");
3554 assert_eq!(req.messages[1].text_view(), "c");
3555 }
3556
3557 #[test]
3558 fn tool_and_image_blocks_survive_the_request_round_trip() {
3559 let body = br#"{"model":"m","max_tokens":50,"messages":[
3561 {"role":"assistant","content":[{"type":"tool_use","id":"t1","name":"calc","input":{"x":1}}]},
3562 {"role":"user","content":[
3563 {"type":"tool_result","tool_use_id":"t1","content":"2"},
3564 {"type":"image","source":{"type":"base64","media_type":"image/png","data":"AA=="}}
3565 ]}]}"#;
3566 let req = parse_model_request(body).unwrap();
3567 let round_tripped = serde_json::to_value(&req.messages).unwrap();
3568 assert_eq!(
3569 round_tripped,
3570 serde_json::json!([
3571 {"role":"assistant","content":[{"type":"tool_use","id":"t1","name":"calc","input":{"x":1}}]},
3572 {"role":"user","content":[
3573 {"type":"tool_result","tool_use_id":"t1","content":"2"},
3574 {"type":"image","source":{"type":"base64","media_type":"image/png","data":"AA=="}}
3575 ]}
3576 ])
3577 );
3578 }
3579
3580 #[test]
3581 fn text_message_serializes_byte_identical_to_a_plain_string() {
3582 let m = ChatMessage::text("user", "hello");
3584 assert_eq!(
3585 serde_json::to_string(&m).unwrap(),
3586 r#"{"role":"user","content":"hello"}"#
3587 );
3588 }
3589
3590 #[test]
3591 fn parse_model_request_rejects_non_message_bodies() {
3592 assert!(parse_model_request(b"not json").is_none());
3593 assert!(parse_model_request(br#"{"no":"messages"}"#).is_none());
3594 }
3595
3596 use crate::provider::{MockProvider, ModelResponse, Provider, ProviderError, ProviderRegistry};
3599 use axum::extract::State;
3600 use std::collections::HashMap;
3601 use std::sync::Arc;
3602 use tokio::sync::mpsc;
3603
3604 fn model_resp(model: &str, text: &str) -> ModelResponse {
3605 ModelResponse {
3606 model: model.to_owned(),
3607 text: text.to_owned(),
3608 in_tokens: 1000,
3609 cache_write_tokens: 0,
3610 cache_read_tokens: 0,
3611 out_tokens: 400,
3612 raw: serde_json::Value::Null,
3613 }
3614 }
3615
3616 fn enforce_state(
3619 ladder: &[&str],
3620 gates: &[&str],
3621 outcomes: Vec<(&str, Result<ModelResponse, ProviderError>)>,
3622 ) -> (AppState, mpsc::Receiver<Trace>) {
3623 let toml = format!(
3624 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\nladder = [{}]\ngates = [{}]\n",
3625 ladder
3626 .iter()
3627 .map(|m| format!("\"{m}\""))
3628 .collect::<Vec<_>>()
3629 .join(", "),
3630 gates
3631 .iter()
3632 .map(|g| format!("\"{g}\""))
3633 .collect::<Vec<_>>()
3634 .join(", "),
3635 );
3636 let config = ProxyConfig::from_lookup(|k| match k {
3637 "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
3638 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
3639 _ => None,
3640 })
3641 .unwrap();
3642
3643 let mut outs = HashMap::new();
3644 for (model, out) in outcomes {
3645 outs.insert(model.to_owned(), out);
3646 }
3647 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
3648 map.insert(
3649 "anthropic".to_owned(),
3650 Arc::new(MockProvider::new("anthropic", outs)),
3651 );
3652 let providers = ProviderRegistry::from_map(map);
3653
3654 let (traces, rx) = mpsc::channel(64);
3655 let state = AppState {
3656 config: Arc::new(config),
3657 http: reqwest::Client::new(),
3658 providers,
3659 gate_health: Arc::new(GateHealthRegistry::new()),
3660 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
3661 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
3662 traces,
3663 adaptive: None,
3664 bandit: None,
3665 promoter: None,
3666 verified_cache: None,
3667 predictor: None,
3668 tenant_rate_limiter: None,
3669 spill: None,
3670 };
3671 (state, rx)
3672 }
3673
3674 fn user_body() -> Bytes {
3675 Bytes::from_static(
3676 br#"{"model":"ignored","max_tokens":64,"messages":[{"role":"user","content":"hi"}]}"#,
3677 )
3678 }
3679
3680 async fn body_json(resp: Response) -> Value {
3681 let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
3682 .await
3683 .unwrap();
3684 serde_json::from_slice(&bytes).unwrap()
3685 }
3686
3687 #[tokio::test]
3688 async fn mode_profile_stamped_on_trace_when_non_balanced() {
3689 let (state, mut rx) = enforce_state(
3690 &["anthropic/claude-haiku-4-5"],
3691 &["non-empty"],
3692 vec![(
3693 "anthropic/claude-haiku-4-5",
3694 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
3695 )],
3696 );
3697 let mut headers = HeaderMap::new();
3698 headers.insert("x-firstpass-mode", "quality".parse().unwrap());
3699 let resp = messages(
3700 State(state),
3701 Extension(TenantId("default".to_owned())),
3702 headers,
3703 user_body(),
3704 )
3705 .await;
3706 assert_eq!(resp.status(), axum::http::StatusCode::OK);
3707 let trace = rx.try_recv().expect("trace enqueued");
3708 assert_eq!(
3709 trace.policy.mode_profile.as_deref(),
3710 Some("quality"),
3711 "mode_profile must be stamped when quality mode is active"
3712 );
3713 }
3714
3715 #[tokio::test]
3716 async fn mode_profile_absent_from_trace_when_balanced() {
3717 let (state, mut rx) = enforce_state(
3718 &["anthropic/claude-haiku-4-5"],
3719 &["non-empty"],
3720 vec![(
3721 "anthropic/claude-haiku-4-5",
3722 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
3723 )],
3724 );
3725 let resp = messages(
3727 State(state),
3728 Extension(TenantId("default".to_owned())),
3729 HeaderMap::new(),
3730 user_body(),
3731 )
3732 .await;
3733 assert_eq!(resp.status(), axum::http::StatusCode::OK);
3734 let trace = rx.try_recv().expect("trace enqueued");
3735 assert!(
3736 trace.policy.mode_profile.is_none(),
3737 "mode_profile must be absent when Balanced (byte-identical invariant)"
3738 );
3739 }
3740
3741 #[tokio::test]
3742 async fn enforce_serves_first_pass_and_returns_anthropic_shape() {
3743 let (state, mut rx) = enforce_state(
3744 &["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"],
3745 &["non-empty"],
3746 vec![(
3747 "anthropic/claude-haiku-4-5",
3748 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
3749 )],
3750 );
3751 let resp = messages(
3752 State(state),
3753 Extension(TenantId("default".to_owned())),
3754 HeaderMap::new(),
3755 user_body(),
3756 )
3757 .await;
3758 assert_eq!(resp.status(), axum::http::StatusCode::OK);
3759 let json = body_json(resp).await;
3760 assert_eq!(json["type"], "message");
3761 assert_eq!(json["content"][0]["text"], "hello");
3762 assert_eq!(json["model"], "anthropic/claude-haiku-4-5");
3763
3764 let trace = rx.try_recv().expect("a trace was enqueued");
3765 assert_eq!(trace.mode, Mode::Enforce);
3766 assert_eq!(trace.final_.served_rung, Some(0));
3767 assert_eq!(trace.attempts.len(), 1);
3768 }
3769
3770 #[tokio::test]
3771 async fn enforce_escalates_then_serves_and_traces_two_attempts() {
3772 let (state, mut rx) = enforce_state(
3773 &["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"],
3774 &["non-empty"],
3775 vec![
3776 (
3777 "anthropic/claude-haiku-4-5",
3778 Ok(model_resp("anthropic/claude-haiku-4-5", " ")),
3779 ), (
3781 "anthropic/claude-sonnet-5",
3782 Ok(model_resp("anthropic/claude-sonnet-5", "answer")),
3783 ),
3784 ],
3785 );
3786 let resp = messages(
3787 State(state),
3788 Extension(TenantId("default".to_owned())),
3789 HeaderMap::new(),
3790 user_body(),
3791 )
3792 .await;
3793 let json = body_json(resp).await;
3794 assert_eq!(json["content"][0]["text"], "answer");
3795
3796 let trace = rx.try_recv().expect("trace enqueued");
3797 assert_eq!(trace.attempts.len(), 2);
3798 assert_eq!(trace.final_.escalations, 1);
3799 assert_eq!(trace.final_.served_rung, Some(1));
3800 }
3801
3802 #[tokio::test]
3803 async fn enforce_all_rungs_error_returns_502() {
3804 let (state, mut rx) = enforce_state(
3805 &["anthropic/claude-haiku-4-5"],
3806 &["non-empty"],
3807 vec![(
3808 "anthropic/claude-haiku-4-5",
3809 Err(ProviderError::Transport("down".into())),
3810 )],
3811 );
3812 let resp = messages(
3813 State(state),
3814 Extension(TenantId("default".to_owned())),
3815 HeaderMap::new(),
3816 user_body(),
3817 )
3818 .await;
3819 assert_eq!(resp.status(), axum::http::StatusCode::BAD_GATEWAY);
3820 assert!(rx.try_recv().is_ok());
3822 }
3823
3824 #[tokio::test]
3825 async fn no_routing_config_falls_through_to_observe_not_enforce() {
3826 let config = ProxyConfig::from_lookup(|k| match k {
3830 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
3831 _ => None,
3832 })
3833 .unwrap();
3834 let (traces, _rx) = mpsc::channel(64);
3835 let state = AppState {
3836 config: Arc::new(config),
3837 http: reqwest::Client::new(),
3838 providers: ProviderRegistry::new("http://127.0.0.1:1", "http://127.0.0.1:1"),
3839 gate_health: Arc::new(GateHealthRegistry::new()),
3840 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
3841 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
3842 traces,
3843 adaptive: None,
3844 bandit: None,
3845 promoter: None,
3846 verified_cache: None,
3847 predictor: None,
3848 tenant_rate_limiter: None,
3849 spill: None,
3850 };
3851 let resp = messages(
3852 State(state),
3853 Extension(TenantId("default".to_owned())),
3854 HeaderMap::new(),
3855 user_body(),
3856 )
3857 .await;
3858 assert_ne!(resp.status(), axum::http::StatusCode::OK);
3860 }
3861
3862 fn bare_enforce_route() -> Route {
3865 use firstpass_core::config::{Match, Mode};
3866 Route {
3867 match_: Match::default(),
3868 mode: Mode::Enforce,
3869 ladder: vec!["anthropic/claude-haiku-4-5".to_owned()],
3870 gates: vec![],
3871 deferred_gates: vec![],
3872 routing_mode: None,
3873 rollout: None,
3874 shadow: None,
3875 }
3876 }
3877
3878 #[test]
3879 fn balanced_preset_application_is_noop() {
3880 let base_max_rungs = 3u32;
3882 let base_speculation = 2u32;
3883 let preset = RoutingMode::Balanced.preset();
3884 let max_rungs = if let Some(d) = preset.max_rungs_delta {
3885 (base_max_rungs as i32 + d).max(1) as u32
3886 } else {
3887 base_max_rungs
3888 };
3889 let speculation = preset.speculation.unwrap_or(base_speculation);
3890 let start_at_top = preset.start_at_top;
3891 assert_eq!(max_rungs, 3, "Balanced must not change max_rungs");
3892 assert_eq!(speculation, 2, "Balanced must not change speculation");
3893 assert!(!start_at_top, "Balanced must not set start_at_top");
3894 }
3895
3896 #[test]
3897 fn resolve_mode_header_wins_over_route_and_global() {
3898 let mut headers = HeaderMap::new();
3899 headers.insert("x-firstpass-mode", "cost".parse().unwrap());
3900 let mut route = bare_enforce_route();
3901 route.routing_mode = Some(RoutingMode::Quality); let mut config = test_config();
3903 config.default_routing_mode = RoutingMode::Max; assert_eq!(
3905 resolve_mode(&headers, &route, &config),
3906 RoutingMode::Cost,
3907 "header must win"
3908 );
3909 }
3910
3911 #[test]
3912 fn resolve_mode_route_wins_over_global_when_no_header() {
3913 let mut route = bare_enforce_route();
3914 route.routing_mode = Some(RoutingMode::Latency);
3915 let mut config = test_config();
3916 config.default_routing_mode = RoutingMode::Max;
3917 assert_eq!(
3918 resolve_mode(&HeaderMap::new(), &route, &config),
3919 RoutingMode::Latency,
3920 "route must beat global default"
3921 );
3922 }
3923
3924 #[test]
3925 fn resolve_mode_global_when_header_and_route_absent() {
3926 let mut config = test_config();
3927 config.default_routing_mode = RoutingMode::Quality;
3928 assert_eq!(
3929 resolve_mode(&HeaderMap::new(), &bare_enforce_route(), &config),
3930 RoutingMode::Quality
3931 );
3932 }
3933
3934 #[test]
3935 fn resolve_mode_unknown_header_falls_through_to_route() {
3936 let mut headers = HeaderMap::new();
3937 headers.insert("x-firstpass-mode", "turbo-mode".parse().unwrap());
3939 let mut route = bare_enforce_route();
3940 route.routing_mode = Some(RoutingMode::Cost);
3941 let config = test_config();
3942 assert_eq!(
3943 resolve_mode(&headers, &route, &config),
3944 RoutingMode::Cost,
3945 "unknown header value must fall through to route"
3946 );
3947 }
3948
3949 #[test]
3950 fn resolve_mode_header_case_insensitive() {
3951 let mut headers = HeaderMap::new();
3952 headers.insert("x-firstpass-mode", "QUALITY".parse().unwrap());
3953 assert_eq!(
3954 resolve_mode(&headers, &bare_enforce_route(), &test_config()),
3955 RoutingMode::Quality
3956 );
3957 }
3958
3959 #[test]
3960 fn resolve_mode_no_mode_set_returns_balanced() {
3961 assert_eq!(
3963 resolve_mode(&HeaderMap::new(), &bare_enforce_route(), &test_config()),
3964 RoutingMode::Balanced
3965 );
3966 }
3967
3968 #[test]
3969 fn capabilities_json_includes_routing_modes() {
3970 let modes: Vec<&'static str> = RoutingMode::ALL.iter().map(|m| m.as_str()).collect();
3971 assert!(modes.contains(&"balanced"));
3972 assert!(modes.contains(&"cost"));
3973 assert!(modes.contains(&"quality"));
3974 assert!(modes.contains(&"latency"));
3975 assert!(modes.contains(&"max"));
3976 assert!(modes.contains(&"observe"));
3977 }
3978
3979 #[test]
3980 fn model_list_dedups_across_ladders_keeping_cheapest_first_order() {
3981 let config = firstpass_core::config::Config::parse(
3984 r#"
3985 [[route]]
3986 mode = "enforce"
3987 ladder = ["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"]
3988 [[route]]
3989 mode = "enforce"
3990 ladder = ["anthropic/claude-sonnet-5", "anthropic/claude-opus-4-8"]
3991 "#,
3992 )
3993 .expect("test config parses");
3994
3995 let out = model_list(Some(&config));
3996
3997 let ids: Vec<&str> = out["data"]
3998 .as_array()
3999 .expect("data is an array")
4000 .iter()
4001 .map(|m| m["id"].as_str().expect("id is a string"))
4002 .collect();
4003 assert_eq!(
4004 ids,
4005 [
4006 "anthropic/claude-haiku-4-5",
4007 "anthropic/claude-sonnet-5",
4008 "anthropic/claude-opus-4-8",
4009 ],
4010 "sonnet appears once, and ladder order is preserved"
4011 );
4012 assert_eq!(out["object"], "list");
4013 assert_eq!(out["data"][0]["owned_by"], "anthropic");
4014 let haiku_in = out["data"][0]["firstpass"]["input_per_mtok"]
4017 .as_f64()
4018 .expect("a first-party rung has a price");
4019 assert!(haiku_in > 0.0, "expected a real price, got {haiku_in}");
4020 let opus_in = out["data"][2]["firstpass"]["input_per_mtok"]
4022 .as_f64()
4023 .expect("a first-party rung has a price");
4024 assert!(haiku_in < opus_in, "{haiku_in} should undercut {opus_in}");
4025 }
4026
4027 #[tokio::test]
4028 async fn healthz_names_the_service_so_a_client_can_tell_us_from_a_stranger() {
4029 let body = healthz().await.into_response();
4033 let bytes = axum::body::to_bytes(body.into_body(), 64 * 1024)
4034 .await
4035 .expect("healthz body");
4036 let json: serde_json::Value = serde_json::from_slice(&bytes).expect("healthz is json");
4037 assert_eq!(json["status"], "ok");
4038 assert_eq!(json["service"], "firstpass");
4039 }
4040
4041 #[test]
4042 fn model_list_is_empty_without_routing_config() {
4043 let out = model_list(None);
4044 assert_eq!(out["object"], "list");
4045 assert_eq!(
4046 out["data"].as_array().expect("data is an array").len(),
4047 0,
4048 "an observe-only deployment advertises no models"
4049 );
4050 }
4051
4052 #[test]
4053 fn detects_stream_requests() {
4054 assert!(is_stream_request(br#"{"stream": true}"#));
4055 assert!(!is_stream_request(br#"{"stream": false}"#));
4056 assert!(!is_stream_request(br#"{"model":"m"}"#));
4057 assert!(!is_stream_request(b"not json"));
4058 }
4059
4060 #[test]
4061 fn detects_tool_blocks_in_messages() {
4062 let with =
4063 br#"{"messages":[{"role":"user","content":[{"type":"tool_result","content":"42"}]}]}"#;
4064 let without = br#"{"messages":[{"role":"user","content":"hi"}]}"#;
4065 assert!(messages_have_tool_blocks(with));
4066 assert!(!messages_have_tool_blocks(without));
4067 }
4068
4069 #[test]
4070 fn enforce_only_handles_plain_text() {
4071 let plain =
4072 Bytes::from_static(br#"{"model":"m","messages":[{"role":"user","content":"hi"}]}"#);
4073 let tools = Bytes::from_static(
4074 br#"{"model":"m","tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
4075 );
4076 let f_plain = extract_features(&HeaderMap::new(), &plain);
4077 let f_tools = extract_features(&HeaderMap::new(), &tools);
4078 let anthropic_ladder = vec!["anthropic/claude-haiku-4-5".to_owned()];
4079 let providers = test_registry();
4080 assert!(enforce_can_handle(
4082 &f_plain,
4083 &plain,
4084 false,
4085 &anthropic_ladder,
4086 &providers,
4087 Dialect::Anthropic,
4088 ));
4089 assert!(!enforce_can_handle(
4090 &f_tools,
4091 &tools,
4092 false,
4093 &anthropic_ladder,
4094 &providers,
4095 Dialect::Anthropic,
4096 ));
4097 }
4098
4099 #[test]
4100 fn structured_enforce_routes_tools_and_streaming() {
4101 let tools = Bytes::from_static(
4104 br#"{"model":"m","tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
4105 );
4106 let streaming_tools = Bytes::from_static(
4107 br#"{"model":"m","stream":true,"tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
4108 );
4109 let f = extract_features(&HeaderMap::new(), &tools);
4110 let anthropic_ladder = vec![
4111 "anthropic/claude-haiku-4-5".to_owned(),
4112 "anthropic/claude-sonnet-5".to_owned(),
4113 ];
4114 let providers = test_registry();
4115 assert!(enforce_can_handle(
4116 &f,
4117 &tools,
4118 true,
4119 &anthropic_ladder,
4120 &providers,
4121 Dialect::Anthropic,
4122 ));
4123 assert!(enforce_can_handle(
4124 &f,
4125 &streaming_tools,
4126 true,
4127 &anthropic_ladder,
4128 &providers,
4129 Dialect::Anthropic,
4130 ));
4131 }
4132
4133 fn test_registry() -> crate::provider::ProviderRegistry {
4135 crate::provider::ProviderRegistry::new("http://localhost", "http://localhost")
4136 }
4137
4138 #[test]
4139 fn fidelity_guard_blocks_structured_on_non_verbatim_ladder() {
4140 let tools = Bytes::from_static(
4144 br#"{"model":"m","tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
4145 );
4146 let plain =
4147 Bytes::from_static(br#"{"model":"m","messages":[{"role":"user","content":"hi"}]}"#);
4148 let f_tools = extract_features(&HeaderMap::new(), &tools);
4149 let f_plain = extract_features(&HeaderMap::new(), &plain);
4150 let providers = test_registry();
4151 let mixed_ladder = vec![
4152 "openai/gpt-4.1-mini".to_owned(),
4153 "anthropic/claude-sonnet-5".to_owned(),
4154 ];
4155 assert!(!enforce_can_handle(
4156 &f_tools,
4157 &tools,
4158 true,
4159 &mixed_ladder,
4160 &providers,
4161 Dialect::Anthropic,
4162 ));
4163 assert!(enforce_can_handle(
4164 &f_plain,
4165 &plain,
4166 true,
4167 &mixed_ladder,
4168 &providers,
4169 Dialect::Anthropic,
4170 ));
4171 }
4172
4173 #[test]
4174 fn enforce_sse_reemission_preserves_text_and_tool_use() {
4175 let resp = ModelResponse {
4178 model: "anthropic/claude-haiku-4-5".to_owned(),
4179 text: "let me check".to_owned(),
4180 in_tokens: 5,
4181 cache_write_tokens: 0,
4182 cache_read_tokens: 0,
4183 out_tokens: 7,
4184 raw: serde_json::json!({
4185 "content": [
4186 { "type": "text", "text": "let me check" },
4187 { "type": "tool_use", "id": "tu_1", "name": "get_weather", "input": { "city": "Paris" } }
4188 ]
4189 }),
4190 };
4191 let sse = anthropic_sse_from_message(&anthropic_response_json(&resp));
4192
4193 let frames: Vec<Value> = sse
4195 .lines()
4196 .filter_map(|l| l.strip_prefix("data: "))
4197 .map(|d| serde_json::from_str::<Value>(d).expect("each SSE data frame is valid JSON"))
4198 .collect();
4199
4200 assert_eq!(frames.first().unwrap()["type"], "message_start");
4202 assert_eq!(frames.last().unwrap()["type"], "message_stop");
4203 assert!(frames.iter().any(|f| f["delta"]["type"] == "text_delta"
4205 && f["delta"]["text"] == "let me check"));
4206 assert!(
4209 frames
4210 .iter()
4211 .any(|f| f["content_block"]["type"] == "tool_use"
4212 && f["content_block"]["name"] == "get_weather"
4213 && f["content_block"]["id"] == "tu_1")
4214 );
4215 assert!(
4216 frames
4217 .iter()
4218 .any(|f| f["delta"]["type"] == "input_json_delta"
4219 && f["delta"]["partial_json"] == r#"{"city":"Paris"}"#)
4220 );
4221 }
4222
4223 #[tokio::test]
4228 async fn enforce_falls_back_to_observe_for_tool_requests() {
4229 let toml = "[[price]]\nmodel = \"anthropic/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/m\"]\ngates = [\"non-empty\"]\n";
4230 let config = ProxyConfig::from_lookup(|k| match k {
4231 "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
4232 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
4233 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
4234 _ => None,
4235 })
4236 .unwrap();
4237 let mut outs = HashMap::new();
4238 outs.insert(
4239 "anthropic/m".to_owned(),
4240 Ok(model_resp("anthropic/m", "hello")),
4241 );
4242 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
4243 map.insert(
4244 "anthropic".to_owned(),
4245 Arc::new(MockProvider::new("anthropic", outs)),
4246 );
4247 let (traces, _rx) = mpsc::channel(64);
4248 let state = AppState {
4249 config: Arc::new(config),
4250 http: reqwest::Client::new(),
4251 providers: ProviderRegistry::from_map(map),
4252 gate_health: Arc::new(GateHealthRegistry::new()),
4253 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
4254 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
4255 traces,
4256 adaptive: None,
4257 bandit: None,
4258 promoter: None,
4259 verified_cache: None,
4260 predictor: None,
4261 tenant_rate_limiter: None,
4262 spill: None,
4263 };
4264
4265 let plain =
4267 Bytes::from_static(br#"{"model":"m","messages":[{"role":"user","content":"hi"}]}"#);
4268 let resp = messages(
4269 State(state.clone()),
4270 Extension(TenantId("default".to_owned())),
4271 HeaderMap::new(),
4272 plain,
4273 )
4274 .await;
4275 assert_eq!(
4276 resp.status(),
4277 axum::http::StatusCode::OK,
4278 "plain text should enforce"
4279 );
4280
4281 let tools = Bytes::from_static(
4284 br#"{"model":"m","tools":[{"name":"get_weather"}],"messages":[{"role":"user","content":"hi"}]}"#,
4285 );
4286 let resp = messages(
4287 State(state.clone()),
4288 Extension(TenantId("default".to_owned())),
4289 HeaderMap::new(),
4290 tools.clone(),
4291 )
4292 .await;
4293 assert_eq!(
4294 resp.status(),
4295 axum::http::StatusCode::OK,
4296 "tool request must route through enforce by default (ADR 0005 default-on)"
4297 );
4298
4299 let toml_off = "[[price]]\nmodel = \"anthropic/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/m\"]\ngates = [\"non-empty\"]\n[escalation]\nenforce_structured = false\n";
4302 let config_off = ProxyConfig::from_lookup(|k| match k {
4303 "FIRSTPASS_CONFIG_TOML" => Some(toml_off.to_owned()),
4304 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
4305 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
4306 _ => None,
4307 })
4308 .unwrap();
4309 let state_off = AppState {
4310 config: Arc::new(config_off),
4311 ..state.clone()
4312 };
4313 let resp = messages(
4314 State(state_off),
4315 Extension(TenantId("default".to_owned())),
4316 HeaderMap::new(),
4317 tools,
4318 )
4319 .await;
4320 assert_ne!(
4321 resp.status(),
4322 axum::http::StatusCode::OK,
4323 "with enforce_structured = false a tool request must fall back to observe"
4324 );
4325
4326 let toolres = Bytes::from_static(
4328 br#"{"model":"m","messages":[{"role":"user","content":[{"type":"tool_result","tool_use_id":"x","content":"42"}]}]}"#,
4329 );
4330 let resp = messages(
4331 State(state),
4332 Extension(TenantId("default".to_owned())),
4333 HeaderMap::new(),
4334 toolres,
4335 )
4336 .await;
4337 assert_eq!(
4338 resp.status(),
4339 axum::http::StatusCode::OK,
4340 "tool_result blocks route through enforce by default too (verbatim carry)"
4341 );
4342 }
4343
4344 #[tokio::test]
4348 async fn panel_is_public_markup_but_receipts_stay_behind_auth() {
4349 use tower::ServiceExt;
4350 let (state, _rx) = enforce_state(
4351 &["anthropic/claude-haiku-4-5"],
4352 &["non-empty"],
4353 vec![(
4354 "anthropic/claude-haiku-4-5",
4355 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
4356 )],
4357 );
4358 let router = app(state).expect("prometheus recorder installs");
4359
4360 let page = router
4361 .clone()
4362 .oneshot(
4363 axum::http::Request::builder()
4364 .uri("/panel")
4365 .body(Body::empty())
4366 .unwrap(),
4367 )
4368 .await
4369 .unwrap();
4370 assert_eq!(page.status(), axum::http::StatusCode::OK);
4371 let html = axum::body::to_bytes(page.into_body(), 1 << 20)
4372 .await
4373 .unwrap();
4374 let html = String::from_utf8_lossy(&html);
4375 assert!(
4376 html.contains("firstpass") && html.contains("/v1/receipts"),
4377 "the page must fetch its data rather than embed it"
4378 );
4379 assert!(
4382 !html.contains("http://") && !html.contains("https://"),
4383 "the panel must not reference any external origin"
4384 );
4385 assert!(html.contains("const esc ="), "the escaper must exist");
4390 for raw in [
4391 "${a.model}",
4392 "${a.rung}",
4393 "${a.verdict}",
4394 "${r.trace_id}",
4395 "${r.served_rung}",
4396 ] {
4397 assert!(
4398 !html.contains(raw),
4399 "receipt field interpolated unescaped into innerHTML: {raw}"
4400 );
4401 }
4402
4403 let data = router
4404 .oneshot(
4405 axum::http::Request::builder()
4406 .uri("/v1/receipts?limit=5")
4407 .body(Body::empty())
4408 .unwrap(),
4409 )
4410 .await
4411 .unwrap();
4412 assert_eq!(data.status(), axum::http::StatusCode::OK);
4416 let body = axum::body::to_bytes(data.into_body(), 1 << 20)
4417 .await
4418 .unwrap();
4419 let v: Value = serde_json::from_slice(&body).unwrap();
4420 assert!(v.get("receipts").and_then(Value::as_array).is_some());
4421 }
4422
4423 async fn feedback_state() -> (AppState, std::path::PathBuf, String) {
4427 let db = std::env::temp_dir().join(format!("firstpass-feedback-{}.db", Uuid::now_v7()));
4428 let (tx, handle) = crate::store::open(&db).unwrap();
4429
4430 let mut trace = build_error_trace(
4431 &ProxyConfig::from_lookup(|_| None).unwrap(),
4432 &Bytes::from_static(b"{}"),
4433 5,
4434 Some("sess-fb"),
4435 );
4436 trace.attempts.push(Attempt {
4437 rung: 0,
4438 model: "anthropic/claude-haiku-4-5".into(),
4439 provider: "anthropic".into(),
4440 in_tokens: 10,
4441 cache_write_tokens: 0,
4442 cache_read_tokens: 0,
4443 out_tokens: 5,
4444 cost_usd: 0.001,
4445 latency_ms: 5,
4446 gates: vec![],
4447 verdict: Verdict::Pass,
4448 });
4449 let trace_id = trace.trace_id.to_string();
4450 tx.try_send(trace).unwrap();
4451 drop(tx);
4452 handle.await.unwrap();
4453
4454 let db_str = db.to_string_lossy().into_owned();
4455 let config = ProxyConfig::from_lookup(move |k| match k {
4456 "FIRSTPASS_DB" => Some(db_str.clone()),
4457 _ => None,
4458 })
4459 .unwrap();
4460 let (traces, _rx) = mpsc::channel(64);
4461 let state = AppState {
4462 config: Arc::new(config),
4463 http: reqwest::Client::new(),
4464 providers: ProviderRegistry::new("http://127.0.0.1:1", "http://127.0.0.1:1"),
4465 gate_health: Arc::new(GateHealthRegistry::new()),
4466 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
4467 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
4468 traces,
4469 adaptive: None,
4470 bandit: None,
4471 promoter: None,
4472 verified_cache: None,
4473 predictor: None,
4474 tenant_rate_limiter: None,
4475 spill: None,
4476 };
4477 (state, db, trace_id)
4478 }
4479
4480 #[tokio::test]
4481 async fn feedback_nudges_the_adaptive_threshold() {
4482 use firstpass_core::conformal::AdaptiveConformal;
4483 let (mut state, _db, trace_id) = feedback_state().await;
4484 let aci = Arc::new(std::sync::Mutex::new(AdaptiveConformal::new(0.1, 0.2, 0.5)));
4485 state.adaptive = Some(aci.clone());
4486 let before = aci.lock().unwrap().threshold();
4487
4488 let fail = Bytes::from(
4490 serde_json::json!({ "trace_id": trace_id, "gate_id": "tests", "verdict": "fail", "reporter": "ci" })
4491 .to_string(),
4492 );
4493 assert_eq!(
4494 feedback(
4495 State(state.clone()),
4496 Extension(TenantId("default".to_owned())),
4497 fail
4498 )
4499 .await
4500 .status(),
4501 axum::http::StatusCode::ACCEPTED
4502 );
4503 let after_fail = aci.lock().unwrap().threshold();
4504 assert!(
4505 after_fail > before,
4506 "served fail should raise the live threshold: {before} -> {after_fail}"
4507 );
4508
4509 let pass = Bytes::from(
4511 serde_json::json!({ "trace_id": trace_id, "gate_id": "tests", "verdict": "pass", "reporter": "ci" })
4512 .to_string(),
4513 );
4514 let _ = feedback(
4515 State(state),
4516 Extension(TenantId("default".to_owned())),
4517 pass,
4518 )
4519 .await;
4520 assert!(aci.lock().unwrap().threshold() < after_fail);
4521 }
4522
4523 #[tokio::test]
4524 async fn feedback_records_a_deferred_verdict_without_breaking_the_chain() {
4525 let (state, db, trace_id) = feedback_state().await;
4526 let body = Bytes::from(
4527 serde_json::json!({
4528 "trace_id": trace_id,
4529 "gate_id": "tests",
4530 "verdict": "pass",
4531 "score": 1.0,
4532 "reporter": "ci",
4533 })
4534 .to_string(),
4535 );
4536 let resp = feedback(
4537 State(state),
4538 Extension(TenantId("default".to_owned())),
4539 body,
4540 )
4541 .await;
4542 assert_eq!(resp.status(), axum::http::StatusCode::ACCEPTED);
4543
4544 let view = crate::store::load_trace_view(&db, "default", &trace_id)
4546 .unwrap()
4547 .unwrap();
4548 assert_eq!(view.deferred.len(), 1);
4549 assert_eq!(view.deferred[0].gate_id, "tests");
4550 let traces = crate::store::load_all_traces(&db).unwrap();
4552 firstpass_core::verify_chain(&traces, GENESIS_HASH).unwrap();
4553
4554 let _ = std::fs::remove_file(&db);
4555 }
4556
4557 #[tokio::test]
4558 async fn feedback_for_unknown_trace_is_404() {
4559 let (state, db, _trace_id) = feedback_state().await;
4560 let body = Bytes::from(
4561 serde_json::json!({
4562 "trace_id": "does-not-exist",
4563 "gate_id": "tests",
4564 "verdict": "pass",
4565 "reporter": "ci",
4566 })
4567 .to_string(),
4568 );
4569 let resp = feedback(
4570 State(state),
4571 Extension(TenantId("default".to_owned())),
4572 body,
4573 )
4574 .await;
4575 assert_eq!(resp.status(), axum::http::StatusCode::NOT_FOUND);
4576 let _ = std::fs::remove_file(&db);
4577 }
4578
4579 #[tokio::test]
4582 async fn feedback_across_tenants_is_404_not_403() {
4583 let (state, db, trace_id) = feedback_state().await;
4584 let body = Bytes::from(
4585 serde_json::json!({
4586 "trace_id": trace_id,
4587 "gate_id": "tests",
4588 "verdict": "pass",
4589 "score": 1.0,
4590 "reporter": "attacker",
4591 })
4592 .to_string(),
4593 );
4594 let resp = feedback(
4596 State(state),
4597 Extension(TenantId("tenant-b".to_owned())),
4598 body,
4599 )
4600 .await;
4601 assert_eq!(
4602 resp.status(),
4603 axum::http::StatusCode::NOT_FOUND,
4604 "cross-tenant feedback must look exactly like a missing trace"
4605 );
4606 let _ = std::fs::remove_file(&db);
4607 }
4608
4609 #[tokio::test]
4610 async fn feedback_rejects_bad_verdict_and_score() {
4611 let (state, db, trace_id) = feedback_state().await;
4612 let bad_verdict = Bytes::from(
4613 serde_json::json!({ "trace_id": trace_id, "gate_id": "g", "verdict": "maybe", "reporter": "x" })
4614 .to_string(),
4615 );
4616 assert_eq!(
4617 feedback(
4618 State(state.clone()),
4619 Extension(TenantId("default".to_owned())),
4620 bad_verdict
4621 )
4622 .await
4623 .status(),
4624 axum::http::StatusCode::BAD_REQUEST
4625 );
4626 let bad_score = Bytes::from(
4627 serde_json::json!({ "trace_id": trace_id, "gate_id": "g", "verdict": "pass", "score": 9.0, "reporter": "x" })
4628 .to_string(),
4629 );
4630 assert_eq!(
4631 feedback(
4632 State(state),
4633 Extension(TenantId("default".to_owned())),
4634 bad_score
4635 )
4636 .await
4637 .status(),
4638 axum::http::StatusCode::BAD_REQUEST
4639 );
4640 let _ = std::fs::remove_file(&db);
4641 }
4642
4643 #[tokio::test]
4644 async fn metrics_endpoint_renders_after_a_real_request() {
4645 use tower::ServiceExt;
4646
4647 let (state, mut rx) = enforce_state(
4648 &["anthropic/claude-haiku-4-5"],
4649 &["non-empty"],
4650 vec![(
4651 "anthropic/claude-haiku-4-5",
4652 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
4653 )],
4654 );
4655 let router = app(state).expect("prometheus recorder installs");
4656
4657 let req = axum::http::Request::builder()
4658 .method("POST")
4659 .uri("/v1/messages")
4660 .header("content-type", "application/json")
4661 .body(Body::from(user_body()))
4662 .unwrap();
4663 let resp = router.clone().oneshot(req).await.unwrap();
4664 assert_eq!(resp.status(), axum::http::StatusCode::OK);
4665 rx.try_recv().expect("a trace was enqueued");
4666
4667 let metrics_req = axum::http::Request::builder()
4668 .method("GET")
4669 .uri("/metrics")
4670 .body(Body::empty())
4671 .unwrap();
4672 let metrics_resp = router.oneshot(metrics_req).await.unwrap();
4673 assert_eq!(metrics_resp.status(), axum::http::StatusCode::OK);
4674 let bytes = axum::body::to_bytes(metrics_resp.into_body(), 1 << 20)
4675 .await
4676 .unwrap();
4677 let body = String::from_utf8(bytes.to_vec()).unwrap();
4678 assert!(
4679 body.contains("firstpass_enforce_latency_ms"),
4680 "metrics body missing enforce latency histogram: {body}"
4681 );
4682 assert!(
4683 body.contains("firstpass_served_total"),
4684 "metrics body missing served counter: {body}"
4685 );
4686 }
4687
4688 fn auth_state(require_auth: bool, keys_json: Option<String>) -> AppState {
4692 auth_state_rated(require_auth, keys_json, None)
4693 }
4694
4695 fn auth_state_rated(
4698 require_auth: bool,
4699 keys_json: Option<String>,
4700 rate_per_sec: Option<u32>,
4701 ) -> AppState {
4702 let config = ProxyConfig::from_lookup(|k| match k {
4703 "FIRSTPASS_REQUIRE_AUTH" => require_auth.then(|| "true".to_owned()),
4704 "FIRSTPASS_TENANT_KEYS_JSON" => keys_json.clone(),
4705 "FIRSTPASS_TENANT_RATE_PER_SEC" => rate_per_sec.map(|n| n.to_string()),
4706 _ => None,
4707 })
4708 .unwrap();
4709 let (traces, _rx) = mpsc::channel(64);
4710 std::mem::forget(_rx);
4713 let providers: HashMap<String, Arc<dyn Provider>> = HashMap::new();
4714 let tenant_rate_limiter = build_tenant_rate_limiter(&config);
4715 AppState {
4716 config: Arc::new(config),
4717 http: reqwest::Client::new(),
4718 providers: ProviderRegistry::from_map(providers),
4719 gate_health: Arc::new(GateHealthRegistry::new()),
4720 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
4721 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
4722 traces,
4723 adaptive: None,
4724 bandit: None,
4725 promoter: None,
4726 verified_cache: None,
4727 predictor: None,
4728 tenant_rate_limiter,
4729 spill: None,
4730 }
4731 }
4732
4733 fn cap_request(auth_header: Option<&str>) -> axum::http::Request<Body> {
4734 let mut b = axum::http::Request::builder()
4735 .method("GET")
4736 .uri("/v1/capabilities");
4737 if let Some(h) = auth_header {
4738 b = b.header("authorization", h);
4739 }
4740 b.body(Body::empty()).unwrap()
4741 }
4742
4743 #[tokio::test]
4744 async fn auth_off_allows_unauthenticated_request() {
4745 use tower::ServiceExt;
4746 let router = app(auth_state(false, None)).expect("router");
4747 let resp = router.oneshot(cap_request(None)).await.unwrap();
4748 assert_eq!(resp.status(), axum::http::StatusCode::OK);
4750 }
4751
4752 #[tokio::test]
4753 async fn auth_on_missing_key_is_401_opaque() {
4754 use tower::ServiceExt;
4755 let hash = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
4756 let keys = format!("{{\"tenant-a\": {hash:?}}}");
4757 let router = app(auth_state(true, Some(keys))).expect("router");
4758
4759 let resp = router.oneshot(cap_request(None)).await.unwrap();
4760 assert_eq!(resp.status(), axum::http::StatusCode::UNAUTHORIZED);
4761 let json = body_json(resp).await;
4762 assert_eq!(json["error"]["type"], "unauthorized");
4763 let msg = json["error"]["message"].as_str().unwrap();
4765 assert!(!msg.contains("tenant"), "no tenant oracle in body: {msg}");
4766 }
4767
4768 #[tokio::test]
4769 async fn auth_on_invalid_key_is_401() {
4770 use tower::ServiceExt;
4771 let hash = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
4772 let keys = format!("{{\"tenant-a\": {hash:?}}}");
4773 let router = app(auth_state(true, Some(keys))).expect("router");
4774
4775 let resp = router
4776 .oneshot(cap_request(Some("Bearer wrong-key")))
4777 .await
4778 .unwrap();
4779 assert_eq!(resp.status(), axum::http::StatusCode::UNAUTHORIZED);
4780 }
4781
4782 #[tokio::test]
4783 async fn auth_on_valid_key_proceeds() {
4784 use tower::ServiceExt;
4785 let hash = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
4786 let keys = format!("{{\"tenant-a\": {hash:?}}}");
4787 let router = app(auth_state(true, Some(keys))).expect("router");
4788
4789 let resp = router
4791 .oneshot(cap_request(Some("Bearer tenant-a.key-a")))
4792 .await
4793 .unwrap();
4794 assert_eq!(resp.status(), axum::http::StatusCode::OK);
4796 }
4797
4798 fn two_tenant_state(rate_per_sec: Option<u32>) -> AppState {
4800 let hash_a = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
4801 let hash_b = crate::tenant_auth::TenantKeys::hash_key("key-b").unwrap();
4802 let keys = format!("{{\"tenant-a\": {hash_a:?}, \"tenant-b\": {hash_b:?}}}");
4803 auth_state_rated(true, Some(keys), rate_per_sec)
4804 }
4805
4806 #[tokio::test]
4807 async fn tenant_exceeding_rate_limit_gets_429_opaque() {
4808 use tower::ServiceExt;
4809 let router = app(two_tenant_state(Some(1))).expect("router");
4815
4816 let (r1, r2, r3, r4) = tokio::join!(
4817 router
4818 .clone()
4819 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
4820 router
4821 .clone()
4822 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
4823 router
4824 .clone()
4825 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
4826 router
4827 .clone()
4828 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
4829 );
4830 let responses = [r1.unwrap(), r2.unwrap(), r3.unwrap(), r4.unwrap()];
4831 let ok = responses
4832 .iter()
4833 .filter(|r| r.status() == axum::http::StatusCode::OK)
4834 .count();
4835 assert!(ok >= 1, "the burst's first request must pass");
4836 let limited: Vec<_> = responses
4837 .into_iter()
4838 .filter(|r| r.status() == axum::http::StatusCode::TOO_MANY_REQUESTS)
4839 .collect();
4840 assert!(
4841 !limited.is_empty(),
4842 "a 4-request burst against 1 req/sec must trip the limiter"
4843 );
4844
4845 let json = body_json(limited.into_iter().next().unwrap()).await;
4846 assert_eq!(json["error"]["type"], "rate_limited");
4847 let msg = json["error"]["message"].as_str().unwrap();
4849 assert!(!msg.contains('1'), "no limit value in body: {msg}");
4850 }
4851
4852 #[tokio::test]
4853 async fn rate_limit_buckets_are_independent_per_tenant() {
4854 use tower::ServiceExt;
4855 let router = app(two_tenant_state(Some(1))).expect("router");
4856
4857 let (a1, a2, a3) = tokio::join!(
4860 router
4861 .clone()
4862 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
4863 router
4864 .clone()
4865 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
4866 router
4867 .clone()
4868 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
4869 );
4870 let a_limited = [a1.unwrap(), a2.unwrap(), a3.unwrap()]
4871 .iter()
4872 .filter(|r| r.status() == axum::http::StatusCode::TOO_MANY_REQUESTS)
4873 .count();
4874 assert!(a_limited >= 1, "tenant A's burst must trip its limiter");
4875
4876 let b1 = router
4878 .clone()
4879 .oneshot(cap_request(Some("Bearer tenant-b.key-b")))
4880 .await
4881 .unwrap();
4882 assert_eq!(b1.status(), axum::http::StatusCode::OK);
4883 }
4884
4885 #[tokio::test]
4886 async fn rate_limit_unset_never_429s() {
4887 use tower::ServiceExt;
4888 let router = app(two_tenant_state(None)).expect("router");
4891 for _ in 0..20 {
4892 let resp = router
4893 .clone()
4894 .oneshot(cap_request(Some("Bearer tenant-a.key-a")))
4895 .await
4896 .unwrap();
4897 assert_eq!(resp.status(), axum::http::StatusCode::OK);
4898 }
4899 }
4900
4901 #[test]
4904 fn u01_is_deterministic_and_in_range() {
4905 let s1 = u01(0xDEAD_BEEF_CAFE_1234_u128);
4906 let s2 = u01(0xDEAD_BEEF_CAFE_1234_u128);
4907 assert_eq!(s1, s2, "u01 must be deterministic for the same seed");
4908 assert!((0.0..1.0).contains(&s1), "u01 must return [0, 1), got {s1}");
4909
4910 let s3 = u01(0x1234_5678_9ABC_DEF0_u128);
4912 assert_ne!(s1, s3, "different seeds should give different values");
4913
4914 for i in 0u64..256 {
4916 let v = u01(i as u128);
4917 assert!((0.0..1.0).contains(&v), "seed {i}: u01={v} out of [0,1)");
4918 }
4919 }
4920
4921 #[test]
4922 fn epsilon_propensity_formula() {
4923 let epsilon = 0.2_f64;
4924 let k = 3_usize;
4925 let greedy = 1_u32;
4926
4927 let p_greedy = epsilon_propensity(greedy, greedy, epsilon, k);
4929 let expected_greedy = (1.0 - epsilon) + epsilon / k as f64;
4930 assert!(
4931 (p_greedy - expected_greedy).abs() < 1e-12,
4932 "{p_greedy} != {expected_greedy}"
4933 );
4934
4935 let p_other = epsilon_propensity(0, greedy, epsilon, k);
4937 let expected_other = epsilon / k as f64;
4938 assert!(
4939 (p_other - expected_other).abs() < 1e-12,
4940 "{p_other} != {expected_other}"
4941 );
4942
4943 for chosen in 0..k as u32 {
4945 let p = epsilon_propensity(chosen, greedy, epsilon, k);
4946 assert!(
4947 p > 0.0 && p <= 1.0,
4948 "propensity {p} out of (0,1] for chosen={chosen}"
4949 );
4950 }
4951 }
4952
4953 #[test]
4954 fn epsilon_branch_and_greedy_branch_both_occur_over_many_seeds() {
4955 let epsilon = 0.3_f64;
4957 let mut saw_explore = false;
4958 let mut saw_greedy = false;
4959 for i in 0u64..200 {
4960 let u = u01(i as u128);
4961 if u < epsilon {
4962 saw_explore = true;
4963 } else {
4964 saw_greedy = true;
4965 }
4966 if saw_explore && saw_greedy {
4967 break;
4968 }
4969 }
4970 assert!(
4971 saw_explore,
4972 "epsilon branch must fire with epsilon=0.3 over 200 seeds"
4973 );
4974 assert!(
4975 saw_greedy,
4976 "greedy branch must occur with epsilon=0.3 over 200 seeds"
4977 );
4978 }
4979
4980 #[test]
4981 fn epsilon_propensity_sums_to_one_over_all_rungs() {
4982 let epsilon = 0.15_f64;
4984 let k = 4_usize;
4985 let greedy = 2_u32;
4986 let total: f64 = (0..k as u32)
4987 .map(|r| epsilon_propensity(r, greedy, epsilon, k))
4988 .sum();
4989 assert!(
4990 (total - 1.0).abs() < 1e-12,
4991 "propensities must sum to 1, got {total}"
4992 );
4993 }
4994
4995 #[tokio::test(start_paused = true)]
4998 async fn keepalive_stream_ticks_then_emits_final_frame() {
4999 let (tx, rx) = tokio::sync::oneshot::channel::<Result<Value, ProxyError>>();
5000 let mut ticks = tokio::time::interval(SSE_KEEPALIVE_EVERY);
5001 ticks.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
5002 ticks.reset();
5003 let mut stream = KeepaliveStream {
5004 rx: Some(rx),
5005 ticks,
5006 format_message: anthropic_sse_from_message,
5007 };
5008 async fn next(
5010 stream: &mut KeepaliveStream,
5011 ) -> Option<Option<Result<Bytes, std::convert::Infallible>>> {
5012 std::future::poll_fn(|cx| {
5013 std::task::Poll::Ready(
5014 match futures_core::Stream::poll_next(std::pin::Pin::new(&mut *stream), cx) {
5015 std::task::Poll::Ready(item) => Some(item),
5016 std::task::Poll::Pending => None,
5017 },
5018 )
5019 })
5020 .await
5021 }
5022
5023 assert!(
5025 next(&mut stream).await.is_none(),
5026 "no frame before an interval"
5027 );
5028
5029 tokio::time::advance(SSE_KEEPALIVE_EVERY + Duration::from_millis(1)).await;
5031 let frame = next(&mut stream)
5032 .await
5033 .expect("keepalive due")
5034 .unwrap()
5035 .unwrap();
5036 assert!(
5037 frame.starts_with(b": "),
5038 "keepalive must be an SSE comment (ignored by every conforming parser)"
5039 );
5040
5041 let message = serde_json::json!({
5043 "id": "msg_1", "type": "message", "role": "assistant", "model": "m",
5044 "content": [{ "type": "text", "text": "done" }],
5045 "usage": { "input_tokens": 1, "output_tokens": 1 }
5046 });
5047 tx.send(Ok(message)).unwrap();
5048 let frame = next(&mut stream)
5049 .await
5050 .expect("final frame")
5051 .unwrap()
5052 .unwrap();
5053 let text = String::from_utf8(frame.to_vec()).unwrap();
5054 assert!(text.contains("event: message_start"));
5055 assert!(text.contains("event: message_stop"));
5056 let eos = next(&mut stream)
5057 .await
5058 .expect("stream must end after the final frame");
5059 assert!(eos.is_none(), "end-of-stream after the final frame");
5060 }
5061
5062 #[tokio::test]
5065 async fn streaming_enforce_serves_full_sse_sequence() {
5066 let toml = "[[price]]\nmodel = \"anthropic/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/m\"]\ngates = [\"non-empty\"]\n";
5067 let config = ProxyConfig::from_lookup(|k| match k {
5068 "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
5069 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
5070 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
5071 _ => None,
5072 })
5073 .unwrap();
5074 let mut outs = HashMap::new();
5075 outs.insert(
5076 "anthropic/m".to_owned(),
5077 Ok(model_resp("anthropic/m", "gated answer")),
5078 );
5079 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
5080 map.insert(
5081 "anthropic".to_owned(),
5082 Arc::new(MockProvider::new("anthropic", outs)),
5083 );
5084 let (traces, _rx) = mpsc::channel(64);
5085 let state = AppState {
5086 config: Arc::new(config),
5087 http: reqwest::Client::new(),
5088 providers: ProviderRegistry::from_map(map),
5089 gate_health: Arc::new(GateHealthRegistry::new()),
5090 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
5091 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
5092 traces,
5093 adaptive: None,
5094 bandit: None,
5095 promoter: None,
5096 verified_cache: None,
5097 predictor: None,
5098 tenant_rate_limiter: None,
5099 spill: None,
5100 };
5101 let body = Bytes::from_static(
5102 br#"{"model":"m","stream":true,"messages":[{"role":"user","content":"hi"}]}"#,
5103 );
5104 let resp = messages(
5105 State(state),
5106 Extension(TenantId("default".to_owned())),
5107 HeaderMap::new(),
5108 body,
5109 )
5110 .await;
5111 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5112 assert!(
5113 resp.headers()
5114 .get(axum::http::header::CONTENT_TYPE)
5115 .and_then(|v| v.to_str().ok())
5116 .is_some_and(|ct| ct.starts_with("text/event-stream")),
5117 "streaming client must get SSE"
5118 );
5119 let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
5120 .await
5121 .unwrap();
5122 let text = String::from_utf8(bytes.to_vec()).unwrap();
5123 assert!(text.contains("event: message_start"));
5124 assert!(text.contains("gated answer"));
5125 assert!(text.contains("event: message_stop"));
5126 }
5127
5128 #[test]
5133 fn parse_openai_request_plain_text() {
5134 let body = br#"{"model":"gpt-4o","max_tokens":256,"messages":[{"role":"user","content":"hello"}]}"#;
5136 let req = parse_openai_request(body, false).expect("must parse");
5137 assert_eq!(req.model, "gpt-4o");
5138 assert_eq!(req.max_tokens, 256);
5139 assert_eq!(req.messages.len(), 1);
5140 assert_eq!(req.messages[0].role, "user");
5141 assert_eq!(req.messages[0].content, Value::String("hello".to_owned()));
5142 assert!(req.system.is_none());
5143 assert_eq!(req.raw, Value::Null);
5145 }
5146
5147 #[test]
5148 fn parse_openai_request_system_message() {
5149 let body = br#"{"model":"gpt-4o","messages":[{"role":"system","content":"be concise"},{"role":"user","content":"hi"}]}"#;
5150 let req = parse_openai_request(body, false).expect("must parse");
5151 assert_eq!(req.system.as_deref(), Some("be concise"));
5152 assert_eq!(req.messages.len(), 1);
5153 assert_eq!(req.messages[0].role, "user");
5154 }
5155
5156 #[test]
5157 fn parse_openai_request_tool_calls_translate_to_tool_use() {
5158 let body = br#"{
5159 "model":"gpt-4o",
5160 "messages":[
5161 {"role":"user","content":"what's the weather?"},
5162 {"role":"assistant","content":null,"tool_calls":[
5163 {"id":"call_1","type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Paris\"}"}}
5164 ]},
5165 {"role":"tool","tool_call_id":"call_1","content":"15C, cloudy"}
5166 ]
5167 }"#;
5168 let req = parse_openai_request(body, false).expect("must parse");
5169 assert_eq!(req.messages.len(), 3);
5171 let asst = &req.messages[1];
5173 assert_eq!(asst.role, "assistant");
5174 let blocks = asst.content.as_array().expect("content array");
5175 assert_eq!(blocks[0]["type"], "tool_use");
5176 assert_eq!(blocks[0]["name"], "get_weather");
5177 assert_eq!(blocks[0]["id"], "call_1");
5178 assert_eq!(blocks[0]["input"]["city"], "Paris");
5179 let tool_msg = &req.messages[2];
5181 assert_eq!(tool_msg.role, "user");
5182 let result_blocks = tool_msg.content.as_array().expect("result blocks");
5183 assert_eq!(result_blocks[0]["type"], "tool_result");
5184 assert_eq!(result_blocks[0]["tool_use_id"], "call_1");
5185 }
5186
5187 #[test]
5188 fn parse_openai_request_tools_translate_to_anthropic_format() {
5189 let body = br#"{
5190 "model":"gpt-4o",
5191 "messages":[{"role":"user","content":"use a tool"}],
5192 "tools":[{"type":"function","function":{"name":"search","description":"web search","parameters":{"type":"object","properties":{"q":{"type":"string"}}}}}]
5193 }"#;
5194 let req = parse_openai_request(body, false).expect("must parse");
5195 let tools = req.tools.as_array().expect("tools array");
5196 assert_eq!(tools.len(), 1);
5197 assert_eq!(tools[0]["name"], "search");
5198 assert_eq!(tools[0]["description"], "web search");
5199 assert_eq!(tools[0]["input_schema"]["type"], "object");
5200 }
5201
5202 #[test]
5203 fn parse_openai_request_raw_carry_preserves_full_body() {
5204 let body =
5206 br#"{"model":"gpt-4o","max_tokens":100,"messages":[{"role":"user","content":"hi"}]}"#;
5207 let req = parse_openai_request(body, true).expect("must parse");
5208 assert!(req.raw.is_object(), "raw must be the full JSON object");
5209 assert_eq!(req.raw["model"], "gpt-4o");
5210 assert_eq!(req.raw["max_tokens"], 100);
5211 assert!(req.tools.is_null(), "no tools in this request");
5213 }
5214
5215 #[test]
5216 fn parse_openai_request_http_image_returns_none() {
5217 let body = br#"{"model":"gpt-4o","messages":[{"role":"user","content":[
5219 {"type":"text","text":"describe this"},
5220 {"type":"image_url","image_url":{"url":"https://example.com/cat.png"}}
5221 ]}]}"#;
5222 let result = parse_openai_request(body, false);
5223 assert!(result.is_none(), "http image URL must fail translation");
5224 }
5225
5226 #[test]
5227 fn parse_openai_request_data_url_image_translates_to_anthropic_base64() {
5228 let body = br#"{"model":"gpt-4o","messages":[{"role":"user","content":[
5229 {"type":"text","text":"describe"},
5230 {"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgo="}}
5231 ]}]}"#;
5232 let req = parse_openai_request(body, false).expect("data URL must parse");
5233 let blocks = req.messages[0].content.as_array().expect("blocks");
5234 let img = blocks
5235 .iter()
5236 .find(|b| b["type"] == "image")
5237 .expect("image block");
5238 assert_eq!(img["source"]["type"], "base64");
5239 assert_eq!(img["source"]["media_type"], "image/png");
5240 assert_eq!(img["source"]["data"], "iVBORw0KGgo=");
5241 }
5242
5243 #[test]
5246 fn openai_response_json_renders_text_response() {
5247 let resp = ModelResponse {
5248 model: "gpt-4o".to_owned(),
5249 text: "Hello!".to_owned(),
5250 in_tokens: 10,
5251 cache_write_tokens: 0,
5252 cache_read_tokens: 0,
5253 out_tokens: 5,
5254 raw: serde_json::json!({
5255 "content": [{ "type": "text", "text": "Hello!" }]
5256 }),
5257 };
5258 let json = openai_response_json(&resp);
5259 assert_eq!(json["object"], "chat.completion");
5260 assert_eq!(json["model"], "gpt-4o");
5261 assert_eq!(json["choices"][0]["message"]["role"], "assistant");
5262 assert_eq!(json["choices"][0]["message"]["content"], "Hello!");
5263 assert_eq!(json["choices"][0]["finish_reason"], "stop");
5264 assert_eq!(json["usage"]["prompt_tokens"], 10);
5265 assert_eq!(json["usage"]["completion_tokens"], 5);
5266 }
5267
5268 #[test]
5269 fn openai_response_json_renders_tool_call() {
5270 let resp = ModelResponse {
5271 model: "gpt-4o".to_owned(),
5272 text: String::new(),
5273 in_tokens: 20,
5274 cache_write_tokens: 0,
5275 cache_read_tokens: 0,
5276 out_tokens: 15,
5277 raw: serde_json::json!({
5278 "content": [{
5279 "type": "tool_use",
5280 "id": "call_abc",
5281 "name": "search",
5282 "input": {"q": "Rust async"}
5283 }]
5284 }),
5285 };
5286 let json = openai_response_json(&resp);
5287 assert_eq!(json["choices"][0]["finish_reason"], "tool_calls");
5288 let tc = &json["choices"][0]["message"]["tool_calls"][0];
5289 assert_eq!(tc["id"], "call_abc");
5290 assert_eq!(tc["type"], "function");
5291 assert_eq!(tc["function"]["name"], "search");
5292 assert_eq!(json["choices"][0]["message"]["content"], Value::Null);
5294 }
5295
5296 #[test]
5297 fn openai_sse_from_message_plain_text_has_role_then_content_then_stop() {
5298 let resp = ModelResponse {
5299 model: "gpt-4o".to_owned(),
5300 text: "Hi there!".to_owned(),
5301 in_tokens: 5,
5302 cache_write_tokens: 0,
5303 cache_read_tokens: 0,
5304 out_tokens: 3,
5305 raw: serde_json::json!({
5306 "content": [{ "type": "text", "text": "Hi there!" }]
5307 }),
5308 };
5309 let sse = openai_sse_from_message(&openai_response_json(&resp));
5310 for line in sse.lines() {
5312 assert!(
5313 line.is_empty() || line.starts_with("data: "),
5314 "bad SSE line: {line:?}"
5315 );
5316 }
5317 let frames: Vec<&str> = sse
5318 .lines()
5319 .filter_map(|l| l.strip_prefix("data: "))
5320 .collect();
5321 assert_eq!(*frames.last().unwrap(), "[DONE]");
5323 let role_frame: Value = serde_json::from_str(frames[0]).unwrap();
5325 assert_eq!(role_frame["choices"][0]["delta"]["role"], "assistant");
5326 assert!(frames.iter().any(|f| {
5328 if *f == "[DONE]" {
5329 return false;
5330 }
5331 serde_json::from_str::<Value>(f)
5332 .ok()
5333 .is_some_and(|v| v["choices"][0]["delta"]["content"] == "Hi there!")
5334 }));
5335 assert!(frames.iter().any(|f| {
5337 if *f == "[DONE]" {
5338 return false;
5339 }
5340 serde_json::from_str::<Value>(f)
5341 .ok()
5342 .is_some_and(|v| v["choices"][0]["finish_reason"] == "stop")
5343 }));
5344 }
5345
5346 #[test]
5349 fn detects_openai_tool_calls() {
5350 let with_tool_calls = Bytes::from_static(br#"{"messages":[
5351 {"role":"user","content":"hi"},
5352 {"role":"assistant","content":null,"tool_calls":[{"id":"c1","type":"function","function":{"name":"f","arguments":"{}"}}]}
5353 ]}"#);
5354 let without = Bytes::from_static(br#"{"messages":[{"role":"user","content":"hi"}]}"#);
5355 let with_tool_msg = Bytes::from_static(
5356 br#"{"messages":[
5357 {"role":"tool","tool_call_id":"c1","content":"result"}
5358 ]}"#,
5359 );
5360 assert!(openai_messages_have_tool_calls(&with_tool_calls));
5361 assert!(!openai_messages_have_tool_calls(&without));
5362 assert!(openai_messages_have_tool_calls(&with_tool_msg));
5363 }
5364
5365 #[test]
5366 fn detects_openai_http_images() {
5367 let http_img = Bytes::from_static(
5368 br#"{"messages":[{"role":"user","content":[
5369 {"type":"image_url","image_url":{"url":"https://example.com/img.png"}}
5370 ]}]}"#,
5371 );
5372 let data_img = Bytes::from_static(
5373 br#"{"messages":[{"role":"user","content":[
5374 {"type":"image_url","image_url":{"url":"data:image/png;base64,abc"}}
5375 ]}]}"#,
5376 );
5377 let no_img = Bytes::from_static(br#"{"messages":[{"role":"user","content":"hi"}]}"#);
5378 assert!(openai_has_http_images(&http_img));
5379 assert!(!openai_has_http_images(&data_img));
5380 assert!(!openai_has_http_images(&no_img));
5381 }
5382
5383 #[test]
5384 fn enforce_can_handle_openai_inbound_all_openai_ladder() {
5385 let tools_body = Bytes::from_static(br#"{"model":"gpt-4o","messages":[{"role":"assistant","content":null,"tool_calls":[{"id":"c","type":"function","function":{"name":"f","arguments":"{}"}}]}]}"#);
5387 let f = extract_openai_features(&HeaderMap::new(), &tools_body);
5388 let ladder = vec!["openai/gpt-4o-mini".to_owned(), "openai/gpt-4o".to_owned()];
5389 let providers = crate::provider::ProviderRegistry::new("http://x", "http://x");
5390 assert!(enforce_can_handle(
5391 &f,
5392 &tools_body,
5393 true,
5394 &ladder,
5395 &providers,
5396 Dialect::Openai
5397 ));
5398 }
5399
5400 #[test]
5401 fn enforce_can_handle_openai_inbound_all_anthropic_ladder_no_http_image() {
5402 let tools_body = Bytes::from_static(br#"{"model":"gpt-4o","messages":[{"role":"assistant","content":null,"tool_calls":[{"id":"c","type":"function","function":{"name":"f","arguments":"{}"}}]}]}"#);
5404 let f = extract_openai_features(&HeaderMap::new(), &tools_body);
5405 let ladder = vec!["anthropic/claude-haiku-4-5".to_owned()];
5406 let providers = crate::provider::ProviderRegistry::new("http://x", "http://x");
5407 assert!(enforce_can_handle(
5408 &f,
5409 &tools_body,
5410 true,
5411 &ladder,
5412 &providers,
5413 Dialect::Openai,
5414 ));
5415 }
5416
5417 #[test]
5418 fn enforce_can_handle_openai_inbound_http_image_falls_back() {
5419 let img_body = Bytes::from_static(
5421 br#"{"model":"gpt-4o","messages":[{"role":"user","content":[
5422 {"type":"image_url","image_url":{"url":"https://example.com/img.png"}}
5423 ]}]}"#,
5424 );
5425 let f = extract_openai_features(&HeaderMap::new(), &img_body);
5426 let ladder = vec!["anthropic/claude-haiku-4-5".to_owned()];
5427 let providers = crate::provider::ProviderRegistry::new("http://x", "http://x");
5428 assert!(!enforce_can_handle(
5429 &f,
5430 &img_body,
5431 true,
5432 &ladder,
5433 &providers,
5434 Dialect::Openai,
5435 ));
5436 }
5437
5438 fn openai_enforce_state(mock_resp: ModelResponse) -> AppState {
5442 let toml = "[[price]]\nmodel = \"mock/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"mock/m\"]\ngates = [\"non-empty\"]\n";
5443 let config = ProxyConfig::from_lookup(|k| match k {
5444 "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
5445 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
5446 _ => None,
5447 })
5448 .unwrap();
5449 let mut outs = HashMap::new();
5450 outs.insert("mock/m".to_owned(), Ok(mock_resp));
5451 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
5452 map.insert("mock".to_owned(), Arc::new(MockProvider::new("mock", outs)));
5453 let (traces, _rx) = mpsc::channel(64);
5454 std::mem::forget(_rx);
5455 let tenant_rate_limiter = build_tenant_rate_limiter(&config);
5456 AppState {
5457 config: Arc::new(config),
5458 http: reqwest::Client::new(),
5459 providers: ProviderRegistry::from_map(map),
5460 gate_health: Arc::new(GateHealthRegistry::new()),
5461 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
5462 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
5463 traces,
5464 adaptive: None,
5465 bandit: None,
5466 promoter: None,
5467 verified_cache: None,
5468 predictor: None,
5469 tenant_rate_limiter,
5470 spill: None,
5471 }
5472 }
5473
5474 #[tokio::test]
5475 async fn chat_completions_plain_text_enforce_returns_openai_shape() {
5476 let mock = model_resp("mock/m", "gated answer");
5477 let state = openai_enforce_state(mock);
5478 let body = Bytes::from_static(
5479 br#"{"model":"gpt-4o","messages":[{"role":"user","content":"hello"}]}"#,
5480 );
5481 let resp = chat_completions(
5482 State(state),
5483 Extension(TenantId("default".to_owned())),
5484 HeaderMap::new(),
5485 body,
5486 )
5487 .await;
5488 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5489 let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
5490 .await
5491 .unwrap();
5492 let json: Value = serde_json::from_slice(&bytes).expect("must be JSON");
5493 assert_eq!(json["object"], "chat.completion");
5494 assert_eq!(json["choices"][0]["message"]["role"], "assistant");
5495 assert_eq!(json["choices"][0]["message"]["content"], "gated answer");
5496 assert_eq!(json["choices"][0]["finish_reason"], "stop");
5497 assert!(
5498 json["id"]
5499 .as_str()
5500 .is_some_and(|id| id.starts_with("chatcmpl-"))
5501 );
5502 }
5503
5504 #[tokio::test]
5505 async fn chat_completions_stream_true_returns_sse_with_openai_chunks() {
5506 let mock = model_resp("mock/m", "gated answer");
5507 let state = openai_enforce_state(mock);
5508 let body = Bytes::from_static(
5509 br#"{"model":"gpt-4o","stream":true,"messages":[{"role":"user","content":"hello"}]}"#,
5510 );
5511 let resp = chat_completions(
5512 State(state),
5513 Extension(TenantId("default".to_owned())),
5514 HeaderMap::new(),
5515 body,
5516 )
5517 .await;
5518 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5519 assert!(
5520 resp.headers()
5521 .get(axum::http::header::CONTENT_TYPE)
5522 .and_then(|v| v.to_str().ok())
5523 .is_some_and(|ct| ct.starts_with("text/event-stream")),
5524 "stream:true must return SSE"
5525 );
5526 let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
5527 .await
5528 .unwrap();
5529 let text = String::from_utf8(bytes.to_vec()).unwrap();
5530 assert!(
5532 text.contains("chat.completion.chunk"),
5533 "must have OpenAI chunk frames"
5534 );
5535 assert!(text.contains("[DONE]"), "must end with [DONE]");
5536 assert!(
5537 text.contains("gated answer"),
5538 "content must be in the stream"
5539 );
5540 assert!(
5542 !text.contains("message_start"),
5543 "must not have Anthropic event types"
5544 );
5545 }
5546
5547 fn probe_state(
5551 sample_rate: f64,
5552 k: u32,
5553 outcomes: Vec<(&str, Result<ModelResponse, ProviderError>)>,
5554 ) -> (AppState, mpsc::Receiver<Trace>) {
5555 let toml = format!(
5556 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\nladder = [\"anthropic/claude-haiku-4-5\"]\ngates = [\"non-empty\"]\n\
5557 [escalation.probe]\nk = {k}\nsample_rate = {sample_rate}\n"
5558 );
5559 let config = ProxyConfig::from_lookup(|k_| match k_ {
5560 "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
5561 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
5562 _ => None,
5563 })
5564 .unwrap();
5565 let mut outs = HashMap::new();
5566 for (model, out) in outcomes {
5567 outs.insert(model.to_owned(), out);
5568 }
5569 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
5570 map.insert(
5571 "anthropic".to_owned(),
5572 Arc::new(MockProvider::new("anthropic", outs)),
5573 );
5574 let (traces, rx) = mpsc::channel(64);
5575 let state = AppState {
5576 config: Arc::new(config),
5577 http: reqwest::Client::new(),
5578 providers: ProviderRegistry::from_map(map),
5579 gate_health: Arc::new(GateHealthRegistry::new()),
5580 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
5581 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
5582 traces,
5583 adaptive: None,
5584 bandit: None,
5585 promoter: None,
5586 verified_cache: None,
5587 predictor: None,
5588 tenant_rate_limiter: None,
5589 spill: None,
5590 };
5591 (state, rx)
5592 }
5593
5594 async fn run_enforce_get_trace(state: AppState, mut rx: mpsc::Receiver<Trace>) -> Trace {
5596 let resp = messages(
5597 State(state),
5598 Extension(TenantId("default".to_owned())),
5599 HeaderMap::new(),
5600 user_body(),
5601 )
5602 .await;
5603 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5604 rx.try_recv().expect("trace must be enqueued")
5605 }
5606
5607 #[tokio::test]
5609 async fn probe_off_trace_has_no_probe_field() {
5610 let (state, rx) = enforce_state(
5611 &["anthropic/claude-haiku-4-5"],
5612 &["non-empty"],
5613 vec![(
5614 "anthropic/claude-haiku-4-5",
5615 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
5616 )],
5617 );
5618 assert!(
5619 state
5620 .config
5621 .routing
5622 .as_ref()
5623 .unwrap()
5624 .escalation
5625 .probe
5626 .is_none(),
5627 "probe must default to None"
5628 );
5629 let trace = run_enforce_get_trace(state, rx).await;
5630 assert!(
5631 trace.probe.is_none(),
5632 "probe=None config must not set trace.probe"
5633 );
5634 }
5635
5636 #[tokio::test]
5638 async fn probe_sample_rate_zero_never_fires() {
5639 let (state, rx) = probe_state(
5641 0.0,
5642 5,
5643 vec![(
5644 "anthropic/claude-haiku-4-5",
5645 Ok(model_resp("anthropic/claude-haiku-4-5", "hi")),
5646 )],
5647 );
5648 let trace = run_enforce_get_trace(state, rx).await;
5649 assert!(
5650 trace.probe.is_none(),
5651 "sample_rate=0.0 must never set trace.probe"
5652 );
5653 }
5654
5655 #[tokio::test]
5658 async fn probe_on_all_pass_sets_confident_pass() {
5659 let model = "anthropic/claude-haiku-4-5";
5661 let (state, mut rx) = probe_state(1.0, 3, vec![(model, Ok(model_resp(model, "hello")))]);
5662 let resp = messages(
5663 State(state),
5664 Extension(TenantId("default".to_owned())),
5665 HeaderMap::new(),
5666 user_body(),
5667 )
5668 .await;
5669 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5670 let json = body_json(resp).await;
5672 assert_eq!(
5673 json["content"][0]["text"], "hello",
5674 "served content unchanged"
5675 );
5676
5677 let trace = rx.try_recv().expect("trace enqueued");
5678 let sig = trace.probe.expect("probe must be set when sample_rate=1.0");
5679 assert_eq!(sig.k, 3);
5680 assert_eq!(sig.gate_pass_count, 3, "all 3 samples must pass non-empty");
5681 assert_eq!(
5682 sig.regime,
5683 firstpass_core::ProbeRegime::ConfidentPass,
5684 "all-pass → ConfidentPass"
5685 );
5686 assert!(
5687 sig.probe_cost_usd > 0.0,
5688 "k model calls must cost something"
5689 );
5690 }
5691
5692 #[tokio::test]
5695 async fn probe_on_all_fail_sets_confident_fail() {
5696 let model = "anthropic/claude-haiku-4-5";
5697 let (state, mut rx) = probe_state(1.0, 3, vec![(model, Ok(model_resp(model, "")))]);
5699 let resp = messages(
5701 State(state),
5702 Extension(TenantId("default".to_owned())),
5703 HeaderMap::new(),
5704 user_body(),
5705 )
5706 .await;
5707 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5708
5709 let trace = rx.try_recv().expect("trace enqueued");
5710 let sig = trace.probe.expect("probe must be set when sample_rate=1.0");
5711 assert_eq!(
5712 sig.gate_pass_count, 0,
5713 "empty response fails non-empty: all 0 pass"
5714 );
5715 assert_eq!(
5716 sig.regime,
5717 firstpass_core::ProbeRegime::ConfidentFail,
5718 "0 passes → ConfidentFail"
5719 );
5720 }
5721
5722 #[tokio::test]
5725 async fn probe_on_served_output_identical_to_probe_off() {
5726 let model = "anthropic/claude-haiku-4-5";
5727 let mk = |sample_rate: f64| {
5728 probe_state(
5729 sample_rate,
5730 2,
5731 vec![(model, Ok(model_resp(model, "gated answer")))],
5732 )
5733 };
5734
5735 let (state_off, mut rx_off) = mk(0.0);
5737 let resp_off = messages(
5738 State(state_off),
5739 Extension(TenantId("default".to_owned())),
5740 HeaderMap::new(),
5741 user_body(),
5742 )
5743 .await;
5744 let json_off = body_json(resp_off).await;
5745 let trace_off = rx_off.try_recv().unwrap();
5746
5747 let (state_on, mut rx_on) = mk(1.0);
5749 let resp_on = messages(
5750 State(state_on),
5751 Extension(TenantId("default".to_owned())),
5752 HeaderMap::new(),
5753 user_body(),
5754 )
5755 .await;
5756 let json_on = body_json(resp_on).await;
5757 let trace_on = rx_on.try_recv().unwrap();
5758
5759 assert_eq!(
5761 json_off["content"][0]["text"], json_on["content"][0]["text"],
5762 "served text must be identical regardless of probe"
5763 );
5764 assert!(
5766 (trace_off.final_.total_cost_usd - trace_on.final_.total_cost_usd).abs() < 1e-12,
5767 "total_cost_usd must not include probe cost: off={} on={}",
5768 trace_off.final_.total_cost_usd,
5769 trace_on.final_.total_cost_usd
5770 );
5771 assert!(trace_off.probe.is_none());
5773 assert!(trace_on.probe.is_some());
5774 assert!(
5776 trace_on.probe.as_ref().unwrap().probe_cost_usd > 0.0,
5777 "probe cost must be positive"
5778 );
5779 }
5780
5781 #[tokio::test]
5791 async fn probe_does_not_mutate_gate_health() {
5792 let model = "anthropic/claude-haiku-4-5";
5793 let (mut state, rx) = probe_state(1.0, 2, vec![(model, Ok(model_resp(model, "answer")))]);
5794
5795 let registry = GateHealthRegistry::new().with_budget("non-empty", 2, 0.4);
5798 registry.record("default", "non-empty", true);
5801 assert!(
5802 registry.enabled("default", "non-empty"),
5803 "gate must start enabled (window not full yet)"
5804 );
5805 state.gate_health = Arc::new(registry);
5806
5807 let trace = run_enforce_get_trace(state, rx).await;
5819 let sig = trace.probe.expect("probe must fire with sample_rate=1.0");
5820 assert_eq!(sig.k, 2);
5821 assert!(
5822 sig.gate_pass_count <= 2,
5823 "gate_pass_count must be in [0, k]"
5824 );
5825 }
5826
5827 fn predictor_state(
5829 enabled: bool,
5830 outcome: Result<ModelResponse, ProviderError>,
5831 ) -> (AppState, mpsc::Receiver<Trace>) {
5832 let toml = "[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/claude-haiku-4-5\"]\ngates = [\"non-empty\"]\n";
5833 let config = ProxyConfig::from_lookup(|k_| match k_ {
5834 "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
5835 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
5836 _ => None,
5837 })
5838 .unwrap();
5839 let mut outs = HashMap::new();
5840 outs.insert("anthropic/claude-haiku-4-5".to_owned(), outcome);
5841 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
5842 map.insert(
5843 "anthropic".to_owned(),
5844 Arc::new(MockProvider::new("anthropic", outs)),
5845 );
5846 let (traces, rx) = mpsc::channel(64);
5847 let predictor = enabled.then(|| {
5848 Arc::new(std::sync::Mutex::new(firstpass_core::PassPredictor::new(
5849 0.05, 1e-4,
5850 )))
5851 });
5852 let state = AppState {
5853 config: Arc::new(config),
5854 http: reqwest::Client::new(),
5855 providers: ProviderRegistry::from_map(map),
5856 gate_health: Arc::new(GateHealthRegistry::new()),
5857 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
5858 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
5859 traces,
5860 adaptive: None,
5861 bandit: None,
5862 promoter: None,
5863 verified_cache: None,
5864 predictor,
5865 tenant_rate_limiter: None,
5866 spill: None,
5867 };
5868 (state, rx)
5869 }
5870
5871 #[tokio::test]
5872 async fn predictor_off_leaves_predicted_pass_none() {
5873 let (state, rx) =
5874 predictor_state(false, Ok(model_resp("anthropic/claude-haiku-4-5", "ok")));
5875 let trace = run_enforce_get_trace(state, rx).await;
5876 assert!(
5877 trace.predicted_pass.is_none(),
5878 "predictor off => no field (byte-identical)"
5879 );
5880 let j = serde_json::to_string(&trace).unwrap();
5882 assert!(!j.contains("predicted_pass"), "None must be omitted: {j}");
5883 }
5884
5885 #[tokio::test]
5886 async fn predictor_on_records_shadow_prediction_and_serves_identically() {
5887 let (state_off, rx_off) = predictor_state(
5890 false,
5891 Ok(model_resp("anthropic/claude-haiku-4-5", "served answer")),
5892 );
5893 let off = run_enforce_get_trace(state_off, rx_off).await;
5894
5895 let (state_on, rx_on) = predictor_state(
5896 true,
5897 Ok(model_resp("anthropic/claude-haiku-4-5", "served answer")),
5898 );
5899 let on = run_enforce_get_trace(state_on, rx_on).await;
5900
5901 assert_eq!(
5902 on.final_.served_rung, off.final_.served_rung,
5903 "served rung identical"
5904 );
5905 assert_eq!(on.attempts.len(), off.attempts.len(), "same attempts");
5906 assert_eq!(
5907 on.final_.total_cost_usd, off.final_.total_cost_usd,
5908 "predictor never adds served cost"
5909 );
5910 let p = on
5911 .predicted_pass
5912 .expect("predictor on => predicted_pass recorded");
5913 assert!(p > 0.0 && p < 1.0, "shadow prediction in (0,1): {p}");
5914 }
5915
5916 fn rollout_state(
5918 percent: f64,
5919 key: &str,
5920 outcomes: Vec<(&str, Result<ModelResponse, ProviderError>)>,
5921 ) -> (AppState, mpsc::Receiver<Trace>) {
5922 let toml = format!(
5923 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
5924 ladder = [\"anthropic/claude-haiku-4-5\"]\ngates = [\"non-empty\"]\n\
5925 [route.rollout]\npercent = {percent}\nkey = \"{key}\"\n"
5926 );
5927 let config = ProxyConfig::from_lookup(|k| match k {
5928 "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
5929 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
5930 _ => None,
5931 })
5932 .unwrap();
5933 let mut outs = HashMap::new();
5934 for (model, out) in outcomes {
5935 outs.insert(model.to_owned(), out);
5936 }
5937 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
5938 map.insert(
5939 "anthropic".to_owned(),
5940 Arc::new(MockProvider::new("anthropic", outs)),
5941 );
5942 let (traces, rx) = mpsc::channel(64);
5943 let state = AppState {
5944 config: Arc::new(config),
5945 http: reqwest::Client::new(),
5946 providers: ProviderRegistry::from_map(map),
5947 gate_health: Arc::new(GateHealthRegistry::new()),
5948 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
5949 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
5950 traces,
5951 adaptive: None,
5952 bandit: None,
5953 promoter: None,
5954 verified_cache: None,
5955 predictor: None,
5956 tenant_rate_limiter: None,
5957 spill: None,
5958 };
5959 (state, rx)
5960 }
5961
5962 #[tokio::test]
5966 async fn rollout_at_zero_percent_never_enforces() {
5967 for i in 0..25 {
5968 let (state, mut rx) = rollout_state(
5969 0.0,
5970 "session",
5971 vec![(
5972 "anthropic/claude-haiku-4-5",
5973 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
5974 )],
5975 );
5976 let mut headers = HeaderMap::new();
5977 headers.insert("x-firstpass-session", format!("s{i}").parse().unwrap());
5978 let resp = messages(
5979 State(state),
5980 Extension(TenantId("default".to_owned())),
5981 headers,
5982 user_body(),
5983 )
5984 .await;
5985 let _ = resp;
5988 if let Ok(t) = rx.try_recv() {
5989 assert!(
5990 t.attempts.is_empty(),
5991 "0% rollout produced an enforced decision for session s{i}"
5992 );
5993 }
5994 }
5995 }
5996
5997 #[tokio::test]
6000 async fn rollout_at_hundred_percent_always_enforces() {
6001 for i in 0..25 {
6002 let (state, mut rx) = rollout_state(
6003 100.0,
6004 "session",
6005 vec![(
6006 "anthropic/claude-haiku-4-5",
6007 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
6008 )],
6009 );
6010 let mut headers = HeaderMap::new();
6011 headers.insert("x-firstpass-session", format!("s{i}").parse().unwrap());
6012 let resp = messages(
6013 State(state),
6014 Extension(TenantId("default".to_owned())),
6015 headers,
6016 user_body(),
6017 )
6018 .await;
6019 assert_eq!(resp.status(), axum::http::StatusCode::OK);
6020 let t = rx.try_recv().expect("trace enqueued");
6021 assert!(
6022 !t.attempts.is_empty(),
6023 "100% rollout skipped enforcement for session s{i}"
6024 );
6025 }
6026 }
6027
6028 #[tokio::test]
6036 async fn dispatch_arm_matches_the_predicted_arm_per_session() {
6037 let mut checked_enforced = 0;
6038 let mut checked_control = 0;
6039 for i in 0..24 {
6040 let session = format!("session-{i}");
6041 let (state, mut rx) = rollout_state(
6042 50.0,
6043 "session",
6044 vec![(
6045 "anthropic/claude-haiku-4-5",
6046 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
6047 )],
6048 );
6049 let expected = firstpass_core::rollout::decide(
6052 &state.config.prompt_salt,
6053 &firstpass_core::Rollout {
6054 percent: 50.0,
6055 key: firstpass_core::RolloutKey::Session,
6056 },
6057 &session,
6058 )
6059 .enforced;
6060
6061 let mut headers = HeaderMap::new();
6062 headers.insert("x-firstpass-session", session.parse().unwrap());
6063 let _ = messages(
6064 State(state),
6065 Extension(TenantId("default".to_owned())),
6066 headers,
6067 user_body(),
6068 )
6069 .await;
6070 let observed = rx
6071 .try_recv()
6072 .map(|t| !t.attempts.is_empty())
6073 .unwrap_or(false);
6074
6075 assert_eq!(
6076 observed,
6077 expected,
6078 "{session}: dispatch put it in the {} arm, bucketing predicted the {} arm",
6079 if observed { "enforced" } else { "control" },
6080 if expected { "enforced" } else { "control" }
6081 );
6082 if expected {
6083 checked_enforced += 1;
6084 } else {
6085 checked_control += 1;
6086 }
6087 }
6088 assert!(
6090 checked_enforced > 0 && checked_control > 0,
6091 "test saw only one arm ({checked_enforced} enforced, {checked_control} control)"
6092 );
6093 }
6094}