1use std::sync::Arc;
5use std::time::Instant;
6
7use axum::body::Body;
8use axum::extract::{Request, State};
9use axum::http::HeaderMap;
10use axum::middleware::Next;
11use axum::response::{IntoResponse, Response};
12use axum::routing::{get, post};
13use axum::{Extension, Json, Router};
14use bytes::Bytes;
15use firstpass_core::features::{hour_bucket, token_bucket};
16use firstpass_core::hashchain::sha256_hex;
17use firstpass_core::{
18 Attempt, DeferredVerdict, Dialect, FEATURE_VERSION, Features, FinalOutcome, GENESIS_HASH, Mode,
19 ModelRef, PolicyRef, ProbeRegime, ProbeSignal, RequestInfo, RoutingMode, Score, ServedFrom,
20 TaskKind, Trace, Verdict,
21 features::{DifficultyHint, TrajectorySignals},
22};
23use serde::Deserialize;
24use serde_json::Value;
25use std::future::Future;
26use std::time::Duration;
27use tokio::sync::mpsc::error::TrySendError;
28use uuid::Uuid;
29
30use crate::config::ProxyConfig;
31use crate::error::ProxyError;
32use crate::gate::{GateHealthRegistry, aggregate_with_policy, resolve_gates};
33use crate::provider::{Auth, ChatMessage, ModelRequest, ModelResponse, ProviderRegistry};
34use crate::router::{EnforceCtx, EngineOutcome, route_enforce};
35use crate::store;
36use crate::tenant_auth::{TenantId, auth_middleware};
37use crate::upstream::{forward_anthropic, forward_anthropic_streaming};
38use firstpass_core::Route;
39use firstpass_core::trace::ShadowSignal;
40
41#[derive(Clone)]
44pub struct AppState {
45 pub config: Arc<ProxyConfig>,
47 pub http: reqwest::Client,
49 pub providers: ProviderRegistry,
51 pub gate_health: Arc<GateHealthRegistry>,
53 pub shadow_ledger: Arc<crate::shadow::ShadowLedger>,
56 pub guardrails: Arc<crate::guard::GuardrailRegistry>,
59 pub traces: store::TraceSender,
61 pub adaptive: Option<Arc<std::sync::Mutex<firstpass_core::conformal::AdaptiveConformal>>>,
65 pub eprocess: Option<Arc<std::sync::Mutex<firstpass_core::eprocess::EProcessRiskControl>>>,
70 pub bandit: Option<Arc<std::sync::Mutex<crate::bandit::StartRungBandit>>>,
75 pub promoter: Option<Arc<crate::affinity::SessionPromoter>>,
80 pub verified_cache: Option<Arc<dyn crate::verified_cache::CacheStore>>,
84 pub predictor: Option<Arc<std::sync::Mutex<firstpass_core::PassPredictor>>>,
90 pub tenant_rate_limiter: Option<Arc<governor::DefaultKeyedRateLimiter<String>>>,
94 pub spill: Option<store::SpillHandle>,
98}
99
100#[must_use]
116fn routing_cfg_condense(state: &AppState) -> Option<&firstpass_core::config::CondenseConfig> {
117 state
118 .config
119 .routing
120 .as_ref()
121 .and_then(|r| r.escalation.condense.as_ref())
122}
123
124#[must_use]
129fn mode_now() -> jiff::Timestamp {
131 jiff::Timestamp::now()
132}
133
134fn cache_hit_trace(
142 ctx: &crate::router::EnforceCtx<'_>,
143 entry: &crate::verified_cache::Entry,
144 ts: jiff::Timestamp,
145) -> Trace {
146 Trace {
147 trace_id: uuid::Uuid::now_v7(),
148 prev_hash: firstpass_core::GENESIS_HASH.to_owned(),
149 tenant_id: ctx.tenant_id.clone(),
150 session_id: ctx.session_id.clone(),
151 ts,
152 mode: Mode::Enforce,
153 policy: PolicyRef {
154 id: format!("{}+cache", ctx.policy_id),
155 explore: false,
156 propensity: None,
157 mode_profile: None,
158 },
159 request: RequestInfo {
160 api: ctx.api.clone(),
161 prompt_hash: ctx.prompt_hash.clone(),
162 features: ctx.features.clone(),
163 },
164 attempts: Vec::new(),
165 final_: FinalOutcome {
166 served_rung: None,
167 served_from: ServedFrom::Cache,
168 total_cost_usd: 0.0,
172 gate_cost_usd: 0.0,
173 total_latency_ms: 0,
174 escalations: 0,
175 counterfactual_baseline_usd: entry.original_cost_usd,
176 savings_usd: entry.original_cost_usd,
177 cache_source: Some(entry.source),
178 },
179 deferred: Vec::new(),
180 predicted_pass: None,
181 probe: None,
182 elastic: None,
183 rollout: None,
184 shadow: None,
185 route_ix: None,
186 }
187}
188
189pub async fn build_verified_cache(
190 config: &ProxyConfig,
191) -> Result<Option<Arc<dyn crate::verified_cache::CacheStore>>, String> {
192 let Some(c) = config
193 .routing
194 .as_ref()
195 .and_then(|r| r.escalation.verified_cache.as_ref())
196 else {
197 return Ok(None);
198 };
199 if let Some(url) = c.redis_url.as_deref() {
200 #[cfg(feature = "redis-cache")]
201 {
202 let store = crate::verified_cache::RedisStore::connect(url, c.ttl_secs).await?;
203 tracing::info!("verified cache: shared via redis");
204 return Ok(Some(Arc::new(store)));
205 }
206 #[cfg(not(feature = "redis-cache"))]
207 {
208 let _ = url;
209 return Err(
210 "[escalation.verified_cache] redis_url is set, but this binary was built without \
211 the `redis-cache` feature. Rebuild with `--features redis-cache`, or remove \
212 redis_url to use the in-process cache."
213 .to_owned(),
214 );
215 }
216 }
217 Ok(Some(Arc::new(crate::verified_cache::VerifiedCache::new(
218 std::time::Duration::from_secs(c.ttl_secs),
219 c.max_entries,
220 ))))
221}
222
223pub async fn build_promoter_async(
224 config: &ProxyConfig,
225) -> Result<Option<Arc<crate::affinity::SessionPromoter>>, String> {
226 let Some(sp) = config
227 .routing
228 .as_ref()
229 .and_then(|r| r.escalation.session_promotion.as_ref())
230 else {
231 return Ok(None);
232 };
233 let promoter = crate::affinity::SessionPromoter::new(sp.clone()).map_err(|e| e.to_string())?;
234 if let Some(url) = sp.redis_url.as_deref() {
235 #[cfg(feature = "redis-cache")]
236 {
237 let ttl = sp.window_duration().map_err(|e| e.to_string())?.as_secs();
238 let store = crate::affinity::RedisPromotionStore::connect(url, ttl).await?;
239 tracing::info!("session promotion: shared via redis");
240 return Ok(Some(Arc::new(promoter.with_shared(Arc::new(store)))));
241 }
242 #[cfg(not(feature = "redis-cache"))]
243 {
244 let _ = url;
245 return Err(
246 "[escalation.session_promotion] redis_url is set, but this binary was \
247 built without the `redis-cache` feature. Rebuild with \
248 `--features redis-cache`, or remove redis_url to keep promotion \
249 in-process."
250 .to_owned(),
251 );
252 }
253 }
254 Ok(Some(Arc::new(promoter)))
255}
256
257#[must_use]
258pub fn build_tenant_rate_limiter(
259 config: &ProxyConfig,
260) -> Option<Arc<governor::DefaultKeyedRateLimiter<String>>> {
261 let per_sec = config.tenant_rate_per_sec?;
262 Some(Arc::new(governor::RateLimiter::keyed(
263 governor::Quota::per_second(per_sec),
264 )))
265}
266
267pub async fn tenant_rate_limit_middleware(
271 State(state): State<AppState>,
272 Extension(tenant): Extension<TenantId>,
273 req: Request,
274 next: Next,
275) -> Response {
276 if let Some(limiter) = &state.tenant_rate_limiter
277 && limiter.check_key(&tenant.0).is_err()
278 {
279 return ProxyError::RateLimited.into_response();
280 }
281 next.run(req).await
282}
283
284impl std::fmt::Debug for AppState {
285 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
286 f.debug_struct("AppState")
287 .field("config", &self.config)
288 .finish_non_exhaustive()
289 }
290}
291
292fn offer_trace(traces: &store::TraceSender, spill: Option<&store::SpillHandle>, trace: Trace) {
309 record_trace_metrics(&trace);
310 match traces.try_send(trace) {
311 Ok(()) => {}
312 Err(TrySendError::Full(t)) => {
313 if let Some(handle) = spill {
314 match store::append_to_spill(handle, &t) {
315 Ok(()) => {
316 metrics::counter!("firstpass_receipts_spilled_total").increment(1);
317 }
318 Err(e) => {
319 tracing::error!(%e, "durable mode: spill write failed; trace lost");
320 metrics::counter!("firstpass_traces_dropped_total").increment(1);
321 }
322 }
323 } else {
324 tracing::warn!("trace channel full; dropping trace (writer behind under load)");
325 metrics::counter!("firstpass_traces_dropped_total").increment(1);
326 }
327 }
328 Err(TrySendError::Closed(_)) => {
329 tracing::warn!("trace writer is gone; dropping trace");
330 }
331 }
332}
333
334fn record_trace_metrics(trace: &Trace) {
338 if trace.mode == Mode::Enforce {
339 metrics::histogram!("firstpass_enforce_latency_ms")
340 .record(trace.final_.total_latency_ms as f64);
341 if trace.final_.escalations > 0 {
342 metrics::counter!("firstpass_escalations_total")
343 .increment(u64::from(trace.final_.escalations));
344 }
345 }
346 let served_from = match trace.final_.served_from {
347 ServedFrom::Attempt => "attempt",
348 ServedFrom::BestAttempt => "best_attempt",
349 ServedFrom::Error => "error",
350 ServedFrom::Cache => "cache",
354 };
355 metrics::counter!("firstpass_served_total", "served_from" => served_from).increment(1);
356 if trace.final_.served_from == ServedFrom::Error {
357 let provider = trace
362 .attempts
363 .last()
364 .map_or_else(|| "unknown".to_owned(), |a| a.provider.clone());
365 metrics::counter!("firstpass_upstream_failures_total", "provider" => provider).increment(1);
366 }
367
368 for a in &trace.attempts {
374 metrics::histogram!(
375 "firstpass_attempt_latency_ms",
376 "provider" => a.provider.clone(),
377 "rung" => a.rung.to_string()
378 )
379 .record(a.latency_ms as f64);
380 metrics::counter!(
381 "firstpass_attempt_total",
382 "provider" => a.provider.clone(),
383 "rung" => a.rung.to_string()
384 )
385 .increment(1);
386 metrics::gauge!(
387 "firstpass_attempt_cost_usd_total",
388 "provider" => a.provider.clone(),
389 "rung" => a.rung.to_string()
390 )
391 .increment(a.cost_usd);
392
393 for g in &a.gates {
397 let verdict = match g.verdict {
398 Verdict::Pass => "pass",
399 Verdict::Fail => "fail",
400 Verdict::Abstain => "abstain",
401 };
402 metrics::counter!(
403 "firstpass_gate_verdict_total",
404 "gate_id" => g.gate_id.clone(),
405 "verdict" => verdict
406 )
407 .increment(1);
408 metrics::histogram!("firstpass_gate_latency_ms", "gate_id" => g.gate_id.clone())
409 .record(g.ms as f64);
410 metrics::gauge!("firstpass_gate_cost_by_id_usd_total", "gate_id" => g.gate_id.clone())
411 .increment(g.cost_usd);
412 }
413 }
414 metrics::gauge!("firstpass_cost_usd_total").increment(trace.final_.total_cost_usd);
418 metrics::gauge!("firstpass_gate_cost_usd_total").increment(trace.final_.gate_cost_usd);
419 metrics::gauge!("firstpass_baseline_usd_total")
420 .increment(trace.final_.counterfactual_baseline_usd);
421 metrics::gauge!("firstpass_savings_usd_total").increment(trace.final_.savings_usd);
422 if let Some(rung) = trace.final_.served_rung {
424 let model = trace
425 .attempts
426 .iter()
427 .find(|a| a.rung == rung)
428 .map(|a| a.model.clone())
429 .unwrap_or_else(|| "unknown".to_owned());
430 metrics::counter!(
431 "firstpass_served_rung_total",
432 "rung" => rung.to_string(),
433 "model" => model
434 )
435 .increment(1);
436 }
437}
438
439const MAX_BODY_BYTES: usize = 16 * 1024 * 1024;
443
444pub(crate) fn u01(seed: u128) -> f64 {
455 let lo = splitmix64_finalise(seed as u64);
456 let hi = splitmix64_finalise((seed >> 64) as u64);
457 ((lo ^ hi) >> 11) as f64 * (1.0_f64 / (1u64 << 53) as f64)
459}
460
461fn splitmix64_finalise(mut z: u64) -> u64 {
462 z = z.wrapping_add(0x9E37_79B9_7F4A_7C15);
463 z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
464 z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
465 z ^ (z >> 31)
466}
467
468#[must_use]
475pub(crate) fn epsilon_propensity(chosen: u32, greedy: u32, epsilon: f64, k: usize) -> f64 {
476 let greedy_term = if chosen == greedy { 1.0 - epsilon } else { 0.0 };
477 greedy_term + epsilon / k as f64
478}
479
480pub fn app(state: AppState) -> Result<Router, ProxyError> {
487 crate::metrics::install()?;
488 let max_concurrency = state.config.max_concurrency;
489
490 let business = Router::new()
499 .route("/v1/messages", post(messages))
500 .route("/v1/chat/completions", post(chat_completions))
501 .route("/v1/responses", post(responses))
502 .route("/v1/feedback", post(feedback))
503 .route("/v1/capabilities", get(capabilities))
504 .route("/v1/models", get(models))
505 .route("/v1/receipts", get(receipts))
508 .layer(axum::middleware::from_fn_with_state(
509 state.clone(),
510 tenant_rate_limit_middleware,
511 ))
512 .layer(axum::middleware::from_fn_with_state(
513 state.clone(),
514 auth_middleware,
515 ));
516
517 Ok(Router::new()
518 .merge(business)
519 .route("/healthz", get(healthz))
520 .route("/metrics", get(crate::metrics::handler))
521 .route("/panel", get(panel))
524 .layer(axum::extract::DefaultBodyLimit::max(MAX_BODY_BYTES))
526 .layer(tower::limit::GlobalConcurrencyLimitLayer::new(
529 max_concurrency,
530 ))
531 .with_state(state))
532}
533
534async fn healthz() -> impl IntoResponse {
536 Json(serde_json::json!({
542 "status": "ok",
543 "service": "firstpass",
544 "version": env!("CARGO_PKG_VERSION"),
545 }))
546}
547
548const PANEL_HTML: &str = include_str!("panel.html");
552
553async fn receipts(
559 State(state): State<AppState>,
560 Extension(TenantId(tenant)): Extension<TenantId>,
561 axum::extract::Query(q): axum::extract::Query<ReceiptsQuery>,
562) -> Response {
563 let limit = q.limit.unwrap_or(20).clamp(1, 200);
565 let db = state.config.db_path.clone();
566 let traces =
567 match tokio::task::spawn_blocking(move || crate::store::load_tenant_traces(&db, &tenant))
568 .await
569 {
570 Ok(Ok(t)) => t,
571 _ => Vec::new(),
574 };
575 let items: Vec<serde_json::Value> = traces
576 .iter()
577 .rev()
578 .take(limit)
579 .map(|t| {
580 let f = &t.final_;
581 serde_json::json!({
582 "trace_id": t.trace_id,
583 "served_rung": f.served_rung,
584 "served_model": f.served_rung
589 .and_then(|r| t.attempts.iter().find(|a| a.rung == r))
590 .map(|a| a.model.clone()),
591 "attempts": t.attempts.iter().map(|a| serde_json::json!({
592 "rung": a.rung,
593 "model": a.model,
594 "verdict": a.verdict,
595 "cost_usd": a.cost_usd,
596 "latency_ms": a.latency_ms,
597 })).collect::<Vec<_>>(),
598 "total_cost_usd": f.total_cost_usd,
599 "baseline_usd": f.counterfactual_baseline_usd,
600 "savings_usd": f.savings_usd,
601 })
602 })
603 .collect();
604 axum::Json(serde_json::json!({ "receipts": items })).into_response()
605}
606
607#[derive(Debug, serde::Deserialize)]
609struct ReceiptsQuery {
610 limit: Option<usize>,
612}
613
614async fn panel() -> impl IntoResponse {
620 (
621 [(axum::http::header::CONTENT_TYPE, "text/html; charset=utf-8")],
622 PANEL_HTML,
623 )
624}
625
626async fn capabilities(State(state): State<AppState>) -> impl IntoResponse {
627 let (ladder, gates) = state
630 .config
631 .routing
632 .as_ref()
633 .and_then(|c| c.routes.iter().find(|r| r.mode == Mode::Enforce))
634 .map(|r| (r.ladder.clone(), r.gates.clone()))
635 .unwrap_or_default();
636 let routing_modes: Vec<serde_json::Value> = RoutingMode::ALL
637 .iter()
638 .map(|m| {
639 let p = m.preset();
640 serde_json::json!({
641 "name": m.as_str(),
642 "description": p.description,
643 "tradeoff": p.tradeoff,
644 })
645 })
646 .collect();
647 Json(serde_json::json!({
648 "service": "firstpass",
649 "version": env!("CARGO_PKG_VERSION"),
650 "feature_version": FEATURE_VERSION,
651 "modes": ["observe", "enforce"],
652 "routing_modes": routing_modes,
653 "wire_apis": ["anthropic.messages", "openai.chat_completions", "openai.responses"],
654 "ladder": ladder,
655 "gates": gates,
656 "feedback_api": "POST /v1/feedback",
657 "offboarding": "unset ANTHROPIC_BASE_URL (or OPENAI_BASE_URL for OpenAI clients)",
658 }))
659}
660
661async fn models(State(state): State<AppState>) -> impl IntoResponse {
673 Json(model_list(state.config.routing.as_ref()))
674}
675
676fn model_list(routing: Option<&firstpass_core::config::Config>) -> serde_json::Value {
679 let prices = routing.map(|c| c.price_table()).unwrap_or_default();
680 let mut seen = std::collections::HashSet::new();
683 let data: Vec<serde_json::Value> = routing
684 .map(|c| c.routes.as_slice())
685 .unwrap_or_default()
686 .iter()
687 .flat_map(|r| r.ladder.iter())
688 .filter(|id| seen.insert(id.as_str()))
689 .map(|id| {
690 let price = prices.get(id);
691 serde_json::json!({
692 "id": id,
693 "object": "model",
694 "created": 0,
697 "owned_by": id.split_once('/').map_or("firstpass", |(p, _)| p),
698 "firstpass": {
699 "input_per_mtok": price.map(|p| p.input_per_mtok),
700 "output_per_mtok": price.map(|p| p.output_per_mtok),
701 },
702 })
703 })
704 .collect();
705 serde_json::json!({ "object": "list", "data": data })
706}
707
708#[derive(Debug, Deserialize)]
710struct FeedbackRequest {
711 trace_id: String,
713 gate_id: String,
715 verdict: String,
717 #[serde(default)]
719 score: Option<f64>,
720 reporter: String,
722}
723
724async fn feedback(
729 State(state): State<AppState>,
730 Extension(TenantId(tenant)): Extension<TenantId>,
731 body: Bytes,
732) -> Response {
733 let req: FeedbackRequest = match serde_json::from_slice(&body) {
734 Ok(r) => r,
735 Err(e) => {
736 return ProxyError::BadRequest(format!("invalid feedback body: {e}")).into_response();
737 }
738 };
739 let verdict = match req.verdict.as_str() {
740 "pass" => Verdict::Pass,
741 "fail" => Verdict::Fail,
742 "abstain" => Verdict::Abstain,
743 other => {
744 return ProxyError::BadRequest(format!("unknown verdict {other:?}")).into_response();
745 }
746 };
747 let score = match req.score {
748 Some(s) => match Score::new(s) {
749 Ok(sc) => Some(sc),
750 Err(_) => {
751 return ProxyError::BadRequest(format!("score {s} out of range [0,1]"))
752 .into_response();
753 }
754 },
755 None => None,
756 };
757
758 let db = state.config.db_path.clone();
759
760 let (db_check, tenant_check, tid_check) = (db.clone(), tenant.clone(), req.trace_id.clone());
765 match tokio::task::spawn_blocking(move || {
766 store::trace_exists(&db_check, &tenant_check, &tid_check)
767 })
768 .await
769 {
770 Ok(Ok(true)) => {}
771 Ok(Ok(false)) => {
772 return ProxyError::NotFound(format!("unknown trace_id {:?}", req.trace_id))
773 .into_response();
774 }
775 Ok(Err(e)) => {
776 tracing::error!(%e, "feedback: trace_exists check failed");
777 return ProxyError::Internal(e.to_string()).into_response();
778 }
779 Err(e) => {
780 tracing::error!(%e, "feedback: trace_exists task panicked");
781 return ProxyError::Internal(e.to_string()).into_response();
782 }
783 }
784
785 if verdict == Verdict::Fail
790 && let Some(cache) = state.verified_cache.as_ref()
791 {
792 let reported = req.trace_id.parse::<uuid::Uuid>().ok();
797 let origin = {
798 let (d, t, i) = (db.clone(), tenant.clone(), req.trace_id.clone());
799 tokio::task::spawn_blocking(move || store::trace_cache_source(&d, &t, &i))
800 .await
801 .ok()
802 .and_then(Result::ok)
803 .flatten()
804 .flatten()
805 };
806 let mut dropped = 0usize;
807 for id in reported.into_iter().chain(origin) {
808 dropped += cache.retract(id).await;
809 }
810 if dropped > 0 {
811 tracing::info!(
812 trace = %req.trace_id,
813 ?origin,
814 dropped,
815 "verified cache: retracted a disproven answer"
816 );
817 metrics::counter!("firstpass_cache_retractions_total").increment(dropped as u64);
818 }
819 }
820
821 let feedback_signal = match verdict {
823 Verdict::Pass => Some(true),
824 Verdict::Fail => Some(false),
825 Verdict::Abstain => None,
826 };
827 let dv = DeferredVerdict {
828 gate_id: req.gate_id,
829 verdict,
830 score,
831 reported_at: jiff::Timestamp::now(),
832 reporter: req.reporter,
833 };
834 let trace_id = req.trace_id.clone();
835 match tokio::task::spawn_blocking(move || store::append_deferred(&db, &req.trace_id, &dv)).await
836 {
837 Ok(Ok(())) => {
838 if let (Some(a), Some(correct)) = (state.adaptive.as_ref(), feedback_signal)
840 && let Ok(mut g) = a.lock()
841 {
842 g.observe_served(correct);
843 metrics::gauge!("firstpass_serve_threshold").set(g.threshold());
844 metrics::gauge!("firstpass_realized_served_failure")
845 .set(g.realized_served_failure());
846 }
847
848 if let (Some(e), Some(correct)) = (state.eprocess.as_ref(), feedback_signal) {
871 let db_s = state.config.db_path.clone();
872 let (tenant_s, tid_s) = (tenant.clone(), trace_id.clone());
873 let served = tokio::task::spawn_blocking(move || {
874 store::load_trace_view(&db_s, &tenant_s, &tid_s)
875 .ok()
876 .flatten()
877 .and_then(|t| {
878 let rung = t.final_.served_rung?;
879 let a = t.attempts.iter().find(|a| a.rung == rung)?;
880 Some(crate::calibrate::gate_score(&a.gates, a.verdict))
881 })
882 })
883 .await
884 .ok()
885 .flatten();
886 if let (Some(sc), Ok(mut g)) = (served, e.lock()) {
889 g.observe_served(sc, correct);
890 metrics::counter!("firstpass_eprocess_rounds_total").increment(1);
891 }
892 }
893
894 let attributed_route = tokio::task::spawn_blocking({
899 let (db, ten, tid) = (
900 state.config.db_path.clone(),
901 tenant.clone(),
902 trace_id.clone(),
903 );
904 move || store::trace_route(&db, &ten, &tid)
905 })
906 .await
907 .ok()
908 .and_then(Result::ok)
909 .flatten()
910 .flatten()
911 .unwrap_or(0) as usize;
912
913 if let (Some(cfg), Some(correct)) = (
918 state.config.routing.as_ref().and_then(|r| r.guardrail),
919 feedback_signal,
920 ) {
921 let cooldown = state
922 .config
923 .routing
924 .as_ref()
925 .map_or(3_600, |r| r.guardrail_cooldown_secs);
926 let reaction = state.guardrails.record(
927 &tenant,
928 attributed_route,
929 &cfg,
930 correct,
931 jiff::Timestamp::now().as_second(),
932 cooldown,
933 );
934 match &reaction {
935 crate::guard::Reaction::Demoted(v) => {
936 metrics::counter!("firstpass_guardrail_demotions_total").increment(1);
937 tracing::error!(
938 tenant = %tenant,
939 n = v.n,
940 rate = v.rate,
941 bound = v.bound,
942 alpha = cfg.alpha,
943 "GUARDRAIL: served-failure bound exceeded target — route demoted to observe; traffic now serves as it would without Firstpass"
944 );
945 }
946 crate::guard::Reaction::Alarmed(v) => {
947 metrics::counter!("firstpass_guardrail_breaches_total").increment(1);
948 tracing::error!(
949 tenant = %tenant,
950 n = v.n,
951 rate = v.rate,
952 bound = v.bound,
953 alpha = cfg.alpha,
954 "GUARDRAIL: served-failure bound exceeded target (alarm only — routing unchanged)"
955 );
956 }
957 crate::guard::Reaction::None => {}
958 }
959 }
960 (
961 axum::http::StatusCode::ACCEPTED,
962 Json(serde_json::json!({ "status": "recorded", "trace_id": trace_id })),
963 )
964 .into_response()
965 }
966 Ok(Err(e)) => {
967 tracing::error!(%e, "feedback: append_deferred failed");
968 ProxyError::Internal(e.to_string()).into_response()
969 }
970 Err(e) => {
971 tracing::error!(%e, "feedback: append_deferred task panicked");
972 ProxyError::Internal(e.to_string()).into_response()
973 }
974 }
975}
976
977const SESSION_HEADER: &str = "x-firstpass-session";
980
981const AGENT_HEADER: &str = "x-firstpass-agent";
983const SUBAGENT_HEADER: &str = "x-firstpass-subagent";
985#[expect(
991 clippy::too_many_arguments,
992 reason = "mirrors the enforce context it evaluates; a wrapper struct would only move the list"
993)]
994fn spawn_shadow(
995 state: &AppState,
996 route: &firstpass_core::Route,
997 route_ix: usize,
998 body: Bytes,
999 auth: Auth,
1000 features: Features,
1001 tenant: String,
1002 session_id: String,
1003 api: &str,
1004) {
1005 let Some(shadow) = route.shadow else {
1006 return;
1007 };
1008 if !shadow.sampled(&state.config.prompt_salt, &session_id) {
1011 return;
1012 }
1013 let (state, route, api) = (state.clone(), route.clone(), api.to_owned());
1014 tokio::spawn(async move {
1015 let signal = evaluate_shadow(
1016 &state, &route, shadow, route_ix, &body, auth, &features, tenant, session_id, &api, 0.0,
1017 )
1018 .await;
1019 tracing::debug!(
1021 would_pass = signal.would_pass,
1022 projected_usd = signal.projected_cost_usd,
1023 skipped = ?signal.skipped,
1024 "shadow evaluation complete"
1025 );
1026 });
1027}
1028
1029#[expect(
1036 clippy::too_many_arguments,
1037 reason = "mirrors the enforce context; grouping them into a struct would only move the list"
1038)]
1039async fn evaluate_shadow(
1040 state: &AppState,
1041 route: &firstpass_core::Route,
1042 shadow: firstpass_core::rollout::Shadow,
1043 route_ix: usize,
1044 body: &Bytes,
1045 auth: Auth,
1046 features: &Features,
1047 tenant: String,
1048 session_id: String,
1049 api: &str,
1050 actual_cost_usd: f64,
1051) -> ShadowSignal {
1052 let skipped = |why: &str| ShadowSignal {
1053 would_serve_rung: None,
1054 would_pass: false,
1055 projected_cost_usd: 0.0,
1056 actual_cost_usd,
1057 skipped: Some(why.to_owned()),
1058 };
1059
1060 let now = jiff::Timestamp::now();
1061 if !state
1062 .shadow_ledger
1063 .may_spend(&tenant, route_ix, shadow.max_usd_per_day, now)
1064 {
1065 return skipped("budget_exhausted");
1068 }
1069
1070 let Some(base_request) = parse_model_request(body) else {
1071 return skipped("unparseable_request");
1072 };
1073 let gate_defs = state
1074 .config
1075 .routing
1076 .as_ref()
1077 .map_or(&[][..], |cfg| &cfg.gate_defs);
1078 let gates = resolve_gates(
1079 &route.gates,
1080 gate_defs,
1081 &state.providers,
1082 &auth,
1083 &state.config.prices,
1084 );
1085 let (budget, max_rungs) = state
1086 .config
1087 .routing
1088 .as_ref()
1089 .map_or((None, u32::MAX), |cfg| {
1090 (
1091 cfg.budget.per_request_usd,
1092 cfg.escalation.max_rungs_per_request,
1093 )
1094 });
1095
1096 let mut base_request = base_request;
1097 base_request.cache_prefix = state
1101 .config
1102 .routing
1103 .as_ref()
1104 .is_some_and(|r| r.escalation.prompt_cache);
1105 let ctx = EnforceCtx {
1106 condense: routing_cfg_condense(state),
1107 ladder: &route.ladder,
1108 gates: &gates,
1109 health: &state.gate_health,
1110 base_request: &base_request,
1111 providers: &state.providers,
1112 auth: &auth,
1113 prices: &state.config.prices,
1114 budget_per_request_usd: budget,
1115 max_rungs,
1116 speculation: 0,
1119 serve_threshold: None,
1120 elastic: None,
1121 features: features.clone(),
1122 start_rung: 0,
1123 tenant_id: tenant.clone(),
1124 session_id,
1125 prompt_hash: prompt_hash(&state.config.prompt_salt, body),
1126 api: api.to_owned(),
1127 policy_id: "shadow".to_owned(),
1128 };
1129
1130 let (outcome, trace) = route_enforce(ctx).await;
1131 let spent = trace.final_.total_cost_usd;
1132 state.shadow_ledger.debit(&tenant, route_ix, spent, now);
1133
1134 let would_pass = matches!(outcome, EngineOutcome::Served(_));
1137 let would_serve_rung = if would_pass {
1138 trace.final_.served_rung
1139 } else {
1140 None
1141 };
1142 ShadowSignal {
1143 would_serve_rung,
1144 would_pass,
1145 projected_cost_usd: spent,
1146 actual_cost_usd,
1147 skipped: None,
1148 }
1149}
1150
1151const MODE_PROFILE_HEADER: &str = "x-firstpass-mode";
1154
1155fn resolve_mode(headers: &HeaderMap, route: &Route, config: &ProxyConfig) -> RoutingMode {
1164 if let Some(val) = header_str(headers, MODE_PROFILE_HEADER) {
1166 match val.trim().to_ascii_lowercase().as_str() {
1167 "observe" => return RoutingMode::Observe,
1168 "cost" => return RoutingMode::Cost,
1169 "balanced" => return RoutingMode::Balanced,
1170 "quality" => return RoutingMode::Quality,
1171 "latency" => return RoutingMode::Latency,
1172 "max" => return RoutingMode::Max,
1173 other => {
1174 tracing::warn!(
1175 value = other,
1176 "unknown x-firstpass-mode value; ignoring \
1177 (valid: observe|cost|balanced|quality|latency|max)"
1178 );
1179 }
1180 }
1181 }
1182 if let Some(m) = route.routing_mode {
1184 return m;
1185 }
1186 config.default_routing_mode
1188}
1189
1190async fn messages(
1195 State(state): State<AppState>,
1196 Extension(TenantId(tenant)): Extension<TenantId>,
1197 headers: HeaderMap,
1198 body: Bytes,
1199) -> Response {
1200 let session_header = header_str(&headers, SESSION_HEADER);
1201
1202 if let Some(routing) = state.config.routing.as_ref() {
1205 let features = extract_features(&headers, &body);
1206 if let Some(route) = routing
1207 .route_for(&features)
1208 .filter(|r| r.mode == Mode::Enforce && !r.ladder.is_empty())
1209 {
1210 let route_ix = routing
1213 .routes
1214 .iter()
1215 .position(|r| std::ptr::eq(r, route))
1216 .unwrap_or(0);
1217 let route = route.clone();
1220 let routing_mode = resolve_mode(&headers, &route, &state.config);
1222 if routing_mode == RoutingMode::Observe {
1224 return observe_passthrough(state, headers, body, session_header, tenant).await;
1225 }
1226 if state
1230 .guardrails
1231 .is_demoted(&tenant, route_ix, jiff::Timestamp::now().as_second())
1232 {
1233 return observe_passthrough(state, headers, body, session_header, tenant).await;
1234 }
1235 if let Some(rollout) = route.rollout.as_ref() {
1240 let key_value = match rollout.key {
1241 firstpass_core::RolloutKey::Session => session_header
1242 .clone()
1243 .unwrap_or_else(|| firstpass_core::rollout::request_identity(&body)),
1244 firstpass_core::RolloutKey::Request => {
1250 firstpass_core::rollout::request_identity(&body)
1251 }
1252 firstpass_core::RolloutKey::Tenant => tenant.to_string(),
1253 };
1254 let decision =
1255 firstpass_core::rollout::decide(&state.config.prompt_salt, rollout, &key_value);
1256 if !decision.enforced {
1257 let shadow_ctx = route.shadow.map(|_| {
1259 (
1260 body.clone(),
1261 Auth::from_headers(&headers),
1262 features.clone(),
1263 tenant.clone(),
1264 session_header
1265 .clone()
1266 .unwrap_or_else(|| Uuid::now_v7().to_string()),
1267 )
1268 });
1269 let resp =
1270 observe_passthrough(state.clone(), headers, body, session_header, tenant)
1271 .await;
1272 if let Some((b, auth, feats, ten, sess)) = shadow_ctx {
1275 spawn_shadow(
1276 &state,
1277 &route,
1278 route_ix,
1279 b,
1280 auth,
1281 feats,
1282 ten,
1283 sess,
1284 "anthropic",
1285 );
1286 }
1287 return resp;
1288 }
1289 }
1290 if enforce_can_handle(
1291 &features,
1292 &body,
1293 routing.escalation.enforce_structured,
1294 &route.ladder,
1295 &state.providers,
1296 Dialect::Anthropic,
1297 ) {
1298 return handle_enforce(
1299 &state,
1300 &headers,
1301 &body,
1302 features,
1303 &route,
1304 route_ix,
1305 session_header,
1306 tenant,
1307 routing_mode,
1308 )
1309 .await;
1310 }
1311 tracing::info!(
1315 "enforce route matched but structured request can't be routed faithfully (flag/ladder); serving via observe passthrough"
1316 );
1317 }
1318 }
1319 observe_passthrough(state, headers, body, session_header, tenant).await
1320}
1321
1322fn enforce_can_handle(
1336 features: &Features,
1337 body: &[u8],
1338 enforce_structured: bool,
1339 ladder: &[String],
1340 providers: &crate::provider::ProviderRegistry,
1341 inbound: Dialect,
1342) -> bool {
1343 let structured = features.tool_count > 0
1344 || features.has_images
1345 || match inbound {
1346 Dialect::Anthropic => messages_have_tool_blocks(body),
1347 Dialect::Openai => openai_messages_have_tool_calls(body),
1348 Dialect::Gemini => false,
1349 };
1350 if !structured {
1351 return true;
1352 }
1353 if !enforce_structured {
1354 return false;
1355 }
1356 let all_verbatim = ladder.iter().all(|rung| {
1358 let provider_id = rung.split('/').next().unwrap_or_default();
1359 providers
1360 .get(provider_id)
1361 .is_some_and(|p| p.carries_structured_verbatim(inbound))
1362 });
1363 if all_verbatim {
1364 return true;
1365 }
1366 if inbound == Dialect::Openai && !openai_has_http_images(body) {
1370 let all_anthropic = ladder.iter().all(|rung| {
1371 let pid = rung.split('/').next().unwrap_or_default();
1372 providers
1373 .get(pid)
1374 .is_some_and(|p| p.carries_structured_verbatim(Dialect::Anthropic))
1375 });
1376 if all_anthropic {
1377 return true;
1378 }
1379 }
1380 false
1381}
1382
1383fn messages_have_tool_blocks(body: &[u8]) -> bool {
1386 serde_json::from_slice::<Value>(body)
1387 .ok()
1388 .and_then(|json| {
1389 json.get("messages")
1390 .and_then(Value::as_array)
1391 .map(|messages| messages.iter().any(message_has_tool_block))
1392 })
1393 .unwrap_or(false)
1394}
1395
1396fn message_has_tool_block(message: &Value) -> bool {
1398 message
1399 .get("content")
1400 .and_then(Value::as_array)
1401 .is_some_and(|blocks| {
1402 blocks.iter().any(|block| {
1403 matches!(
1404 block.get("type").and_then(Value::as_str),
1405 Some("tool_use" | "tool_result")
1406 )
1407 })
1408 })
1409}
1410
1411fn is_stream_request(body: &[u8]) -> bool {
1413 serde_json::from_slice::<Value>(body)
1414 .ok()
1415 .and_then(|json| json.get("stream").and_then(Value::as_bool))
1416 .unwrap_or(false)
1417}
1418
1419fn header_str(headers: &HeaderMap, name: &str) -> Option<String> {
1421 headers
1422 .get(name)
1423 .and_then(|v| v.to_str().ok())
1424 .map(str::to_owned)
1425}
1426
1427fn trajectory_signals(body: &[u8]) -> TrajectorySignals {
1441 let Ok(json) = serde_json::from_slice::<Value>(body) else {
1442 return TrajectorySignals::default();
1443 };
1444 let Some(messages) = json.get("messages").and_then(Value::as_array) else {
1445 return TrajectorySignals::default();
1446 };
1447
1448 const WINDOW: usize = 12;
1453 let recent = &messages[messages.len().saturating_sub(WINDOW)..];
1454
1455 let assistant_turns = u32::try_from(
1464 messages
1465 .iter()
1466 .filter(|m| m.get("role").and_then(Value::as_str) == Some("assistant"))
1467 .count(),
1468 )
1469 .unwrap_or(u32::MAX);
1470 let mut s = TrajectorySignals {
1471 assistant_turns,
1472 ..Default::default()
1473 };
1474 let mut seen: Vec<u64> = Vec::new();
1478
1479 for m in recent {
1480 match m.get("role").and_then(Value::as_str) {
1481 Some("assistant") => {}
1483 Some("tool") => {
1486 s.tool_results += 1;
1487 if openai_tool_content_looks_like_error(m) {
1488 s.tool_errors += 1;
1489 }
1490 }
1491 _ => {}
1492 }
1493
1494 if let Some(blocks) = m.get("content").and_then(Value::as_array) {
1497 for b in blocks {
1498 match b.get("type").and_then(Value::as_str) {
1499 Some("tool_result") => {
1500 s.tool_results += 1;
1501 if b.get("is_error").and_then(Value::as_bool).unwrap_or(false) {
1502 s.tool_errors += 1;
1503 }
1504 }
1505 Some("tool_use") => {
1506 let fp = tool_call_fingerprint(
1507 b.get("name").and_then(Value::as_str).unwrap_or_default(),
1508 b.get("input"),
1509 );
1510 if seen.contains(&fp) {
1511 s.repeated_tool_calls += 1;
1512 } else {
1513 seen.push(fp);
1514 }
1515 }
1516 _ => {}
1517 }
1518 }
1519 }
1520
1521 if let Some(calls) = m.get("tool_calls").and_then(Value::as_array) {
1523 for c in calls {
1524 let f = c.get("function");
1525 let fp = tool_call_fingerprint(
1526 f.and_then(|f| f.get("name"))
1527 .and_then(Value::as_str)
1528 .unwrap_or_default(),
1529 f.and_then(|f| f.get("arguments")),
1530 );
1531 if seen.contains(&fp) {
1532 s.repeated_tool_calls += 1;
1533 } else {
1534 seen.push(fp);
1535 }
1536 }
1537 }
1538 }
1539 s
1540}
1541
1542fn tool_call_fingerprint(name: &str, args: Option<&Value>) -> u64 {
1547 use std::hash::{Hash, Hasher};
1548 let mut h = std::collections::hash_map::DefaultHasher::new();
1549 name.hash(&mut h);
1550 if let Some(a) = args {
1555 a.to_string().hash(&mut h);
1556 }
1557 h.finish()
1558}
1559
1560fn openai_tool_content_looks_like_error(message: &Value) -> bool {
1569 let content = message.get("content");
1573 let text: &str = match content.and_then(Value::as_str) {
1578 Some(t) => t,
1579 None => {
1580 let Some(t) = content.and_then(Value::as_array).and_then(|parts| {
1581 parts
1582 .iter()
1583 .find_map(|p| p.get("text").and_then(Value::as_str))
1584 }) else {
1585 return false;
1586 };
1587 t
1588 }
1589 };
1590 let lowered: String = text
1606 .chars()
1607 .take(128)
1608 .collect::<String>()
1609 .trim_start()
1610 .chars()
1611 .take(64)
1612 .collect::<String>()
1613 .to_ascii_lowercase();
1614 lowered.starts_with("error")
1615 || lowered.starts_with("exception")
1616 || lowered.starts_with("traceback")
1617 || lowered.starts_with("failed")
1618 || lowered.starts_with("fatal")
1619}
1620
1621fn extract_features(headers: &HeaderMap, body: &[u8]) -> Features {
1622 let (_model, tool_count, has_images) = request_features(body);
1623 let mut f = Features::new(TaskKind::Other);
1624 f.agent = header_str(headers, AGENT_HEADER);
1625 f.subagent = header_str(headers, SUBAGENT_HEADER);
1626 f.tool_count = tool_count;
1627 f.has_images = has_images;
1628 f.prompt_token_bucket = token_bucket(body.len() as u64);
1631 f.hour_bucket = hour_bucket(jiff::Timestamp::now());
1632 f.difficulty_hint = DifficultyHint::score(trajectory_signals(body)).as_u8();
1636 f
1637}
1638
1639#[allow(clippy::too_many_arguments)]
1642async fn handle_enforce(
1643 state: &AppState,
1644 headers: &HeaderMap,
1645 body: &Bytes,
1646 features: Features,
1647 route: &Route,
1648 route_ix: usize,
1651 session_header: Option<String>,
1652 tenant: String,
1653 routing_mode: RoutingMode,
1654) -> Response {
1655 if is_stream_request(body) {
1663 let (tx, rx) = tokio::sync::oneshot::channel::<Result<Value, ProxyError>>();
1664 let (state_c, headers_c, body_c, route_c) =
1665 (state.clone(), headers.clone(), body.clone(), route.clone());
1666 tokio::spawn(async move {
1667 let out = enforce_pipeline(
1668 &state_c,
1669 &headers_c,
1670 &body_c,
1671 features,
1672 &route_c,
1673 route_ix,
1674 session_header,
1675 tenant,
1676 routing_mode,
1677 )
1678 .await;
1679 let _ = tx.send(out);
1680 });
1681 return sse_keepalive_response(rx, anthropic_sse_from_message);
1682 }
1683 match enforce_pipeline(
1684 state,
1685 headers,
1686 body,
1687 features,
1688 route,
1689 route_ix,
1690 session_header,
1691 tenant,
1692 routing_mode,
1693 )
1694 .await
1695 {
1696 Ok(message) => (axum::http::StatusCode::OK, Json(message)).into_response(),
1697 Err(e) => e.into_response(),
1698 }
1699}
1700
1701#[allow(clippy::too_many_arguments)] async fn enforce_pipeline_inner(
1706 state: &AppState,
1707 body: &Bytes,
1708 base_request: ModelRequest,
1709 auth: Auth,
1710 features: Features,
1711 route: &Route,
1712 session_header: Option<String>,
1713 tenant: String,
1714 api: &str,
1715 routing_mode: RoutingMode,
1716 route_ix: usize,
1717) -> Result<ModelResponse, ProxyError> {
1718 let gate_defs = state
1719 .config
1720 .routing
1721 .as_ref()
1722 .map_or(&[][..], |cfg| &cfg.gate_defs);
1723 let gates = resolve_gates(
1724 &route.gates,
1725 gate_defs,
1726 &state.providers,
1727 &auth,
1728 &state.config.prices,
1729 );
1730 let session_id = session_header.unwrap_or_else(|| Uuid::now_v7().to_string());
1731 let (budget, max_rungs, speculation, serve_threshold, elastic) =
1732 match state.config.routing.as_ref() {
1733 Some(cfg) => (
1734 cfg.budget.per_request_usd,
1735 cfg.escalation.max_rungs_per_request,
1736 cfg.escalation.speculation,
1737 cfg.escalation.serve_threshold,
1738 cfg.escalation.elastic.as_ref(),
1739 ),
1740 None => (None, 3, 0, None, None),
1741 };
1742 let certified = state
1757 .eprocess
1758 .as_ref()
1759 .and_then(|e| e.lock().ok().and_then(|g| g.certified_threshold()));
1760 if let Some(c) = &certified {
1761 metrics::gauge!("firstpass_eprocess_certified_threshold").set(c.threshold);
1762 metrics::gauge!("firstpass_eprocess_e_value").set(c.e_value);
1763 }
1764 let serve_threshold = certified
1765 .map(|c| c.threshold)
1766 .or_else(|| {
1767 state
1768 .adaptive
1769 .as_ref()
1770 .and_then(|a| a.lock().ok().map(|g| g.threshold()))
1771 })
1772 .or(serve_threshold);
1773
1774 let preset = routing_mode.preset();
1777 let max_rungs = if let Some(delta) = preset.max_rungs_delta {
1778 (max_rungs as i32 + delta).max(1) as u32
1779 } else {
1780 max_rungs
1781 };
1782 let speculation = preset.speculation.unwrap_or(speculation);
1783
1784 let bandit_ctx = crate::bandit::ContextBucket::from_features(&features);
1788
1789 let (greedy_rung, base_policy_id, ts_propensity) = {
1793 let (chosen, ts_p) = state
1794 .bandit
1795 .as_ref()
1796 .and_then(|b| b.lock().ok())
1797 .map(|mut b| {
1798 b.choose_start_with_propensity(&bandit_ctx, &route.ladder, &state.config.prices)
1799 })
1800 .unwrap_or((0, None));
1801 let policy = if ts_p.is_some() {
1802 "bandit@v2-ts".to_owned()
1803 } else if chosen > 0 {
1804 "bandit@v1".to_owned()
1805 } else {
1806 "static-ladder@v0".to_owned()
1807 };
1808 (chosen, policy, ts_p)
1809 };
1810
1811 let exploration_epsilon = state
1816 .config
1817 .routing
1818 .as_ref()
1819 .and_then(|cfg| cfg.escalation.exploration.as_ref())
1820 .map(|e| e.epsilon);
1821
1822 let (start_rung, policy_id, explore_flag, propensity) = if ts_propensity.is_some() {
1823 if exploration_epsilon.is_some() {
1826 tracing::warn!(
1827 "bandit.algorithm = thompson already logs propensities; \
1828 [escalation.exploration] epsilon is ignored"
1829 );
1830 }
1831 (greedy_rung, base_policy_id, false, ts_propensity)
1832 } else if let Some(epsilon) = exploration_epsilon {
1833 let k = route.ladder.len().max(1);
1834 let u = u01(Uuid::now_v7().as_u128());
1836 let (chosen, eps_branch) = if u < epsilon {
1837 let idx = ((u / epsilon) * k as f64) as u32;
1839 (idx.min(k as u32 - 1), true)
1840 } else {
1841 (greedy_rung, false)
1842 };
1843 let p = epsilon_propensity(chosen, greedy_rung, epsilon, k);
1844 (chosen, format!("{base_policy_id}+eps"), eps_branch, Some(p))
1845 } else {
1846 (greedy_rung, base_policy_id, false, None)
1847 };
1848
1849 let promotion = state.promoter.as_ref().cloned();
1854 let promotion = match promotion {
1857 Some(p) => {
1858 p.decide_async(&tenant, &session_id, std::time::Instant::now())
1859 .await
1860 }
1861 None => crate::affinity::Decision::Cold,
1862 };
1863 let start_rung = start_rung.max(promotion.start_rung());
1864 let promo_key = state
1867 .promoter
1868 .as_ref()
1869 .map(|_| (tenant.clone(), session_id.clone()));
1870
1871 let start_rung = if preset.start_at_top {
1874 route.ladder.len().saturating_sub(1) as u32
1875 } else {
1876 start_rung
1877 };
1878
1879 let speculation = match state
1885 .config
1886 .routing
1887 .as_ref()
1888 .and_then(|cfg| cfg.escalation.speculation_band)
1889 {
1890 Some([lo, hi]) if speculation > 0 => {
1891 let estimate = state
1892 .bandit
1893 .as_ref()
1894 .and_then(|b| b.lock().ok())
1895 .and_then(|b| b.pass_estimate(&bandit_ctx, start_rung));
1896 match estimate {
1897 Some(p) if p < lo || p > hi => {
1898 metrics::counter!("firstpass_speculation_skipped_total").increment(1);
1899 0
1900 }
1901 _ => speculation,
1902 }
1903 }
1904 _ => speculation,
1905 };
1906
1907 if state.bandit.is_some() {
1909 metrics::counter!(
1910 "firstpass_bandit_start_rung",
1911 "rung" => start_rung.to_string()
1912 )
1913 .increment(1);
1914 }
1915
1916 let mut base_request = base_request;
1917 base_request.cache_prefix = state
1921 .config
1922 .routing
1923 .as_ref()
1924 .is_some_and(|r| r.escalation.prompt_cache);
1925 let ctx = EnforceCtx {
1926 condense: routing_cfg_condense(state),
1927 ladder: &route.ladder,
1928 gates: &gates,
1929 health: &state.gate_health,
1930 base_request: &base_request,
1931 providers: &state.providers,
1932 auth: &auth,
1933 prices: &state.config.prices,
1934 budget_per_request_usd: budget,
1935 max_rungs,
1936 speculation,
1937 serve_threshold,
1938 elastic,
1939 features,
1940 start_rung,
1941 tenant_id: tenant,
1944 session_id,
1945 prompt_hash: prompt_hash(&state.config.prompt_salt, body),
1946 api: api.to_owned(),
1947 policy_id,
1948 };
1949
1950 if let Some(cache) = state.verified_cache.as_ref()
1955 && let Some(entry) = cache
1956 .get(
1957 &crate::verified_cache::VerifiedCache::compose_key(
1958 &ctx.tenant_id,
1959 &ctx.prompt_hash,
1960 &route.ladder,
1961 ),
1962 std::time::Instant::now(),
1963 )
1964 .await
1965 {
1966 metrics::counter!("firstpass_cache_hits_total").increment(1);
1967 let trace = cache_hit_trace(&ctx, &entry, mode_now());
1968 offer_trace(&state.traces, state.spill.as_ref(), trace);
1969 return Ok(entry.response);
1970 }
1971
1972 let (outcome, mut trace) = route_enforce(ctx).await;
1973
1974 trace.route_ix = Some(u32::try_from(route_ix).unwrap_or(0));
1978 trace.policy.explore = explore_flag;
1981 trace.policy.propensity = propensity;
1982 if routing_mode != RoutingMode::Balanced {
1985 trace.policy.mode_profile = Some(routing_mode.as_str().to_owned());
1986 }
1987
1988 if let Some(bandit) = state.bandit.as_ref()
1992 && let Ok(mut b) = bandit.lock()
1993 {
1994 for attempt in &trace.attempts {
1995 b.observe(&bandit_ctx, attempt.rung, attempt.verdict);
1996 }
1997 }
1998
1999 if let Some(promoter) = state.promoter.as_ref()
2004 && let Some((tenant_key, session_key)) = promo_key.as_ref()
2005 && let Some(served) = trace.final_.served_rung
2006 {
2007 promoter
2008 .record_async(
2009 tenant_key,
2010 session_key,
2011 served,
2012 served > start_rung,
2013 std::time::Instant::now(),
2014 )
2015 .await;
2016 }
2017
2018 if let Some(predictor) = state.predictor.as_ref()
2024 && let Ok(mut p) = predictor.lock()
2025 {
2026 let predicted = p.predict(&trace.request.features, start_rung);
2028 for attempt in &trace.attempts {
2029 match attempt.verdict {
2030 Verdict::Pass => p.update(&trace.request.features, attempt.rung, true),
2031 Verdict::Fail => p.update(&trace.request.features, attempt.rung, false),
2032 Verdict::Abstain => {} }
2034 }
2035 trace.predicted_pass = Some(predicted);
2036 metrics::histogram!("firstpass_predictor_pass_prob").record(predicted);
2037 }
2038
2039 if let Some(probe_cfg) = state
2046 .config
2047 .routing
2048 .as_ref()
2049 .and_then(|c| c.escalation.probe)
2050 && u01(Uuid::now_v7().as_u128()) < probe_cfg.sample_rate
2051 {
2052 let probe_rung = (start_rung as usize).min(route.ladder.len().saturating_sub(1));
2054 if let Some(probe_model_str) = route.ladder.get(probe_rung).cloned() {
2055 let probe_provider = ModelRef::parse(&probe_model_str)
2056 .ok()
2057 .and_then(|m| state.providers.get(&m.provider));
2058
2059 if let Some(probe_provider) = probe_provider {
2060 let mut join_set = tokio::task::JoinSet::new();
2063 for _ in 0..probe_cfg.k {
2064 let mut probe_req = base_request.clone();
2065 probe_req.model = probe_model_str.clone();
2066 let probe_auth = auth.clone();
2067 let prov = probe_provider.clone();
2068 join_set.spawn(async move { prov.complete(&probe_req, &probe_auth).await });
2069 }
2070
2071 let fail_closed_owned: std::collections::HashSet<String> = gates
2074 .iter()
2075 .filter(|g| g.abstain_fails_closed())
2076 .map(|g| g.id().to_owned())
2077 .collect();
2078
2079 let mut gate_pass_count = 0u32;
2080 let mut probe_cost_usd = 0.0f64;
2081
2082 while let Some(task_result) = join_set.join_next().await {
2083 let Ok(Ok(probe_resp)) = task_result else {
2084 continue; };
2086 probe_cost_usd += state
2089 .config
2090 .prices
2091 .cost_usd_with_cache(
2092 &probe_model_str,
2093 probe_resp.in_tokens,
2094 probe_resp.cache_write_tokens,
2095 probe_resp.cache_read_tokens,
2096 probe_resp.out_tokens,
2097 )
2098 .unwrap_or(0.0);
2099
2100 let mut probe_gate_req = base_request.clone();
2101 probe_gate_req.model = probe_model_str.clone();
2102
2103 let mut probe_gate_results = Vec::with_capacity(gates.len());
2106 for g in &gates {
2107 if !state.gate_health.enabled(&trace.tenant_id, g.id()) {
2109 continue;
2110 }
2111 let r = g.evaluate(&probe_gate_req, &probe_resp).await;
2112 probe_gate_results.push(r);
2114 }
2115
2116 let fail_closed_refs: std::collections::HashSet<&str> =
2117 fail_closed_owned.iter().map(|s| s.as_str()).collect();
2118 let verdict = aggregate_with_policy(&probe_gate_results, &fail_closed_refs);
2119 let passes = match serve_threshold {
2122 None => verdict == Verdict::Pass,
2123 Some(t) => crate::calibrate::gate_score(&probe_gate_results, verdict) >= t,
2124 };
2125 if passes {
2126 gate_pass_count += 1;
2127 }
2128 }
2129
2130 let regime = ProbeRegime::classify(gate_pass_count, probe_cfg.k);
2131 let regime_label = match regime {
2132 ProbeRegime::ConfidentPass => "confident_pass",
2133 ProbeRegime::ConfidentFail => "confident_fail",
2134 ProbeRegime::Ambiguous => "ambiguous",
2135 };
2136 metrics::counter!(
2137 "firstpass_probe_regime_total",
2138 "regime" => regime_label
2139 )
2140 .increment(1);
2141 metrics::gauge!("firstpass_probe_cost_usd_total").increment(probe_cost_usd);
2142 trace.probe = Some(ProbeSignal {
2143 k: probe_cfg.k,
2144 gate_pass_count,
2145 regime,
2146 probe_cost_usd,
2147 });
2148 }
2149 }
2150 }
2151 if let Some(cache) = state.verified_cache.as_ref()
2164 && let EngineOutcome::Served(resp) = &outcome
2165 && crate::verified_cache::is_cacheable(&trace)
2166 {
2167 let now = std::time::Instant::now();
2168 cache
2169 .put(
2170 &crate::verified_cache::VerifiedCache::compose_key(
2171 &trace.tenant_id,
2172 &trace.request.prompt_hash,
2173 &route.ladder,
2174 ),
2175 trace.trace_id,
2176 crate::verified_cache::Entry::new(resp.clone(), &trace, now),
2177 now,
2178 )
2179 .await;
2180 metrics::counter!("firstpass_cache_stores_total").increment(1);
2181 }
2182
2183 offer_trace(&state.traces, state.spill.as_ref(), trace);
2185
2186 match outcome {
2187 EngineOutcome::Served(resp) => Ok(resp),
2188 EngineOutcome::Failed(msg) => Err(ProxyError::Engine(msg)),
2189 }
2190}
2191
2192#[allow(clippy::too_many_arguments)]
2195async fn enforce_pipeline(
2196 state: &AppState,
2197 headers: &HeaderMap,
2198 body: &Bytes,
2199 features: Features,
2200 route: &Route,
2201 route_ix: usize,
2203 session_header: Option<String>,
2204 tenant: String,
2205 routing_mode: RoutingMode,
2206) -> Result<Value, ProxyError> {
2207 let Some(base_request) = parse_model_request(body) else {
2208 return Err(ProxyError::BadRequest(
2209 "request body is not a valid Anthropic Messages request".to_owned(),
2210 ));
2211 };
2212 let auth = Auth::from_headers(headers);
2213 let resp = enforce_pipeline_inner(
2214 state,
2215 body,
2216 base_request,
2217 auth,
2218 features,
2219 route,
2220 session_header,
2221 tenant,
2222 "anthropic.messages",
2223 routing_mode,
2224 route_ix,
2225 )
2226 .await?;
2227 Ok(anthropic_response_json(&resp))
2228}
2229
2230#[allow(clippy::too_many_arguments)]
2233async fn enforce_pipeline_openai(
2234 state: &AppState,
2235 headers: &HeaderMap,
2236 body: &Bytes,
2237 features: Features,
2238 route: &Route,
2239 route_ix: usize,
2241 session_header: Option<String>,
2242 tenant: String,
2243 routing_mode: RoutingMode,
2244) -> Result<Value, ProxyError> {
2245 enforce_pipeline_openai_as(
2246 state,
2247 headers,
2248 body,
2249 body,
2250 "openai.chat_completions",
2251 features,
2252 route,
2253 route_ix,
2254 session_header,
2255 tenant,
2256 routing_mode,
2257 )
2258 .await
2259}
2260
2261#[allow(clippy::too_many_arguments)]
2270async fn enforce_pipeline_openai_as(
2271 state: &AppState,
2272 headers: &HeaderMap,
2273 body: &Bytes,
2274 receipt_body: &Bytes,
2275 api: &str,
2276 features: Features,
2277 route: &Route,
2278 route_ix: usize,
2279 session_header: Option<String>,
2280 tenant: String,
2281 routing_mode: RoutingMode,
2282) -> Result<Value, ProxyError> {
2283 let providers = &state.providers;
2285 let all_openai = route.ladder.iter().all(|rung| {
2286 let pid = rung.split('/').next().unwrap_or_default();
2287 providers
2288 .get(pid)
2289 .is_some_and(|p| p.carries_structured_verbatim(Dialect::Openai))
2290 });
2291 let Some(base_request) = parse_openai_request(body, all_openai) else {
2292 return Err(ProxyError::BadRequest(
2293 "request body is not a valid OpenAI Chat Completions request".to_owned(),
2294 ));
2295 };
2296 let auth = Auth::from_headers(headers);
2297 let resp = enforce_pipeline_inner(
2298 state,
2299 receipt_body,
2300 base_request,
2301 auth,
2302 features,
2303 route,
2304 session_header,
2305 tenant,
2306 api,
2307 routing_mode,
2308 route_ix,
2309 )
2310 .await?;
2311 Ok(openai_response_json(&resp))
2312}
2313
2314const SSE_KEEPALIVE_EVERY: Duration = Duration::from_secs(5);
2316
2317fn sse_keepalive_response(
2326 rx: tokio::sync::oneshot::Receiver<Result<Value, ProxyError>>,
2327 format_message: fn(&Value) -> String,
2328) -> Response {
2329 let mut ticks = tokio::time::interval(SSE_KEEPALIVE_EVERY);
2330 ticks.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
2331 ticks.reset(); let stream = KeepaliveStream {
2333 rx: Some(rx),
2334 ticks,
2335 format_message,
2336 };
2337 (
2338 axum::http::StatusCode::OK,
2339 [(
2340 axum::http::header::CONTENT_TYPE,
2341 "text/event-stream; charset=utf-8",
2342 )],
2343 axum::body::Body::from_stream(stream),
2344 )
2345 .into_response()
2346}
2347
2348struct KeepaliveStream {
2351 rx: Option<tokio::sync::oneshot::Receiver<Result<Value, ProxyError>>>,
2353 ticks: tokio::time::Interval,
2354 format_message: fn(&Value) -> String,
2356}
2357
2358impl futures_core::Stream for KeepaliveStream {
2359 type Item = Result<Bytes, std::convert::Infallible>;
2360
2361 fn poll_next(
2362 mut self: std::pin::Pin<&mut Self>,
2363 cx: &mut std::task::Context<'_>,
2364 ) -> std::task::Poll<Option<Self::Item>> {
2365 use std::task::Poll;
2366 let Some(rx) = self.rx.as_mut() else {
2367 return Poll::Ready(None); };
2369 if let Poll::Ready(out) = std::pin::Pin::new(rx).poll(cx) {
2370 let fmt = self.format_message;
2371 let frame = match out {
2372 Ok(Ok(message)) => fmt(&message),
2373 Ok(Err(e)) => sse_error_event(&e),
2374 Err(_) => sse_error_event(&ProxyError::Internal(
2375 "enforce pipeline task dropped".to_owned(),
2376 )),
2377 };
2378 self.rx = None;
2379 return Poll::Ready(Some(Ok(Bytes::from(frame))));
2380 }
2381 if self.ticks.poll_tick(cx).is_ready() {
2382 return Poll::Ready(Some(Ok(Bytes::from_static(b": firstpass routing\n\n"))));
2383 }
2384 Poll::Pending
2385 }
2386}
2387
2388fn sse_error_event(e: &ProxyError) -> String {
2391 let mut out = String::new();
2392 sse_event(
2393 &mut out,
2394 "error",
2395 &serde_json::json!({
2396 "type": "error",
2397 "error": { "type": "api_error", "message": e.client_message() }
2398 }),
2399 );
2400 out
2401}
2402
2403fn parse_model_request(body: &[u8]) -> Option<ModelRequest> {
2412 let json: Value = serde_json::from_slice(body).ok()?;
2413 let raw = json.clone();
2414 let messages_json = json.get("messages")?.as_array()?;
2415 let messages = messages_json
2416 .iter()
2417 .map(|m| ChatMessage {
2418 role: m
2419 .get("role")
2420 .and_then(Value::as_str)
2421 .unwrap_or("user")
2422 .to_owned(),
2423 content: m
2424 .get("content")
2425 .cloned()
2426 .unwrap_or_else(|| Value::String(String::new())),
2427 })
2428 .collect();
2429 let system = json
2430 .get("system")
2431 .and_then(Value::as_str)
2432 .map(str::to_owned);
2433 let max_tokens = json
2434 .get("max_tokens")
2435 .and_then(Value::as_u64)
2436 .and_then(|n| u32::try_from(n).ok())
2437 .unwrap_or(1024);
2438 let tools = json.get("tools").cloned().unwrap_or(Value::Null);
2439 Some(ModelRequest {
2440 model: json
2441 .get("model")
2442 .and_then(Value::as_str)
2443 .unwrap_or_default()
2444 .to_owned(),
2445 system,
2446 messages,
2447 max_tokens,
2448 tools,
2449 raw,
2450 cache_prefix: false,
2451 })
2452}
2453
2454fn anthropic_response_json(resp: &ModelResponse) -> Value {
2464 let content = resp
2465 .raw
2466 .get("content")
2467 .filter(|c| c.is_array())
2468 .cloned()
2469 .unwrap_or_else(|| serde_json::json!([{ "type": "text", "text": resp.text }]));
2470 serde_json::json!({
2471 "id": format!("msg_{}", Uuid::now_v7()),
2472 "type": "message",
2473 "role": "assistant",
2474 "model": resp.model,
2475 "content": content,
2476 "usage": { "input_tokens": resp.in_tokens, "output_tokens": resp.out_tokens },
2477 })
2478}
2479
2480fn sse_event(out: &mut String, event: &str, data: &Value) {
2482 out.push_str("event: ");
2483 out.push_str(event);
2484 out.push_str("\ndata: ");
2485 out.push_str(&data.to_string());
2486 out.push_str("\n\n");
2487}
2488
2489fn anthropic_sse_from_message(message: &Value) -> String {
2496 let mut out = String::new();
2497
2498 let mut start_msg = message.clone();
2500 start_msg["content"] = Value::Array(Vec::new());
2501 sse_event(
2502 &mut out,
2503 "message_start",
2504 &serde_json::json!({ "type": "message_start", "message": start_msg }),
2505 );
2506
2507 let empty = Vec::new();
2508 let blocks = message
2509 .get("content")
2510 .and_then(Value::as_array)
2511 .unwrap_or(&empty);
2512 for (i, block) in blocks.iter().enumerate() {
2513 match block.get("type").and_then(Value::as_str) {
2514 Some("tool_use") => {
2515 let mut shell = block.clone();
2517 shell["input"] = serde_json::json!({});
2518 sse_event(
2519 &mut out,
2520 "content_block_start",
2521 &serde_json::json!({ "type": "content_block_start", "index": i, "content_block": shell }),
2522 );
2523 let input_json = block
2524 .get("input")
2525 .map_or_else(|| "{}".to_owned(), std::string::ToString::to_string);
2526 sse_event(
2527 &mut out,
2528 "content_block_delta",
2529 &serde_json::json!({ "type": "content_block_delta", "index": i,
2530 "delta": { "type": "input_json_delta", "partial_json": input_json } }),
2531 );
2532 }
2533 _ => {
2534 let text = block.get("text").and_then(Value::as_str).unwrap_or("");
2536 sse_event(
2537 &mut out,
2538 "content_block_start",
2539 &serde_json::json!({ "type": "content_block_start", "index": i,
2540 "content_block": { "type": "text", "text": "" } }),
2541 );
2542 sse_event(
2543 &mut out,
2544 "content_block_delta",
2545 &serde_json::json!({ "type": "content_block_delta", "index": i,
2546 "delta": { "type": "text_delta", "text": text } }),
2547 );
2548 }
2549 }
2550 sse_event(
2551 &mut out,
2552 "content_block_stop",
2553 &serde_json::json!({ "type": "content_block_stop", "index": i }),
2554 );
2555 }
2556
2557 let out_tokens = message
2558 .pointer("/usage/output_tokens")
2559 .cloned()
2560 .unwrap_or_else(|| Value::from(0));
2561 sse_event(
2562 &mut out,
2563 "message_delta",
2564 &serde_json::json!({ "type": "message_delta", "delta": { "stop_reason": "end_turn" },
2565 "usage": { "output_tokens": out_tokens } }),
2566 );
2567 sse_event(
2568 &mut out,
2569 "message_stop",
2570 &serde_json::json!({ "type": "message_stop" }),
2571 );
2572 out
2573}
2574
2575async fn observe_passthrough(
2577 state: AppState,
2578 headers: HeaderMap,
2579 body: Bytes,
2580 session_header: Option<String>,
2581 tenant: String,
2582) -> Response {
2583 if is_stream_request(&body) {
2585 return observe_stream(state, headers, body, session_header, tenant).await;
2586 }
2587 let start = Instant::now();
2588 let result = forward_anthropic(
2589 &state.http,
2590 &state.config.upstream_anthropic,
2591 &headers,
2592 body.clone(),
2593 )
2594 .await;
2595 let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
2596
2597 match result {
2598 Ok((status, resp_headers, resp_body)) => {
2599 spawn_trace(
2603 &state,
2604 body,
2605 Some(resp_body.clone()),
2606 latency_ms,
2607 session_header,
2608 tenant,
2609 );
2610 (status, resp_headers, resp_body).into_response()
2611 }
2612 Err(err) => {
2613 spawn_trace(&state, body, None, latency_ms, session_header, tenant);
2614 err.into_response()
2615 }
2616 }
2617}
2618
2619async fn observe_stream(
2628 state: AppState,
2629 headers: HeaderMap,
2630 body: Bytes,
2631 session_header: Option<String>,
2632 tenant: String,
2633) -> Response {
2634 let start = Instant::now();
2635 let result = forward_anthropic_streaming(
2636 &state.http,
2637 &state.config.upstream_anthropic,
2638 &headers,
2639 body.clone(),
2640 )
2641 .await;
2642 let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
2643
2644 match result {
2645 Ok((status, resp_headers, response)) => {
2646 spawn_stream_trace(&state, body, latency_ms, session_header, tenant);
2647 let stream_body = Body::from_stream(response.bytes_stream());
2648 (status, resp_headers, stream_body).into_response()
2649 }
2650 Err(err) => {
2651 spawn_trace(&state, body, None, latency_ms, session_header, tenant);
2652 err.into_response()
2653 }
2654 }
2655}
2656
2657fn spawn_stream_trace(
2659 state: &AppState,
2660 req_body: Bytes,
2661 latency_ms: u64,
2662 session_header: Option<String>,
2663 tenant: String,
2664) {
2665 let config = state.config.clone();
2666 let traces = state.traces.clone();
2667 let spill = state.spill.clone();
2668 tokio::spawn(async move {
2669 let mut trace =
2670 build_stream_trace(&config, &req_body, latency_ms, session_header.as_deref());
2671 trace.tenant_id = tenant;
2673 offer_trace(&traces, spill.as_ref(), trace);
2674 });
2675}
2676
2677fn spawn_trace(
2682 state: &AppState,
2683 req_body: Bytes,
2684 resp_body: Option<Bytes>,
2685 latency_ms: u64,
2686 session_header: Option<String>,
2687 tenant: String,
2688) {
2689 let config = state.config.clone();
2690 let traces = state.traces.clone();
2691 let spill = state.spill.clone();
2692 tokio::spawn(async move {
2693 let mut trace = match resp_body {
2694 Some(resp) => build_trace(
2695 &config,
2696 &req_body,
2697 &resp,
2698 latency_ms,
2699 session_header.as_deref(),
2700 ),
2701 None => build_error_trace(&config, &req_body, latency_ms, session_header.as_deref()),
2702 };
2703 trace.tenant_id = tenant;
2705 offer_trace(&traces, spill.as_ref(), trace);
2706 });
2707}
2708
2709fn session_id(session_header: Option<&str>, trace_id: Uuid) -> String {
2711 session_header
2712 .map(str::to_owned)
2713 .unwrap_or_else(|| trace_id.to_string())
2714}
2715
2716fn prompt_hash(salt: &str, body: &[u8]) -> String {
2719 let mut salted = Vec::with_capacity(salt.len() + body.len());
2720 salted.extend_from_slice(salt.as_bytes());
2721 salted.extend_from_slice(body);
2722 sha256_hex(&salted)
2723}
2724
2725fn request_features(body: &[u8]) -> (Option<String>, u32, bool) {
2729 let Ok(json) = serde_json::from_slice::<Value>(body) else {
2730 return (None, 0, false);
2731 };
2732 let model = json.get("model").and_then(Value::as_str).map(str::to_owned);
2733 let tool_count = json
2734 .get("tools")
2735 .and_then(Value::as_array)
2736 .map_or(0, |tools| u32::try_from(tools.len()).unwrap_or(u32::MAX));
2737 let has_images = json
2738 .get("messages")
2739 .and_then(Value::as_array)
2740 .is_some_and(|messages| messages.iter().any(message_has_image));
2741 (model, tool_count, has_images)
2742}
2743
2744fn message_has_image(message: &Value) -> bool {
2746 message
2747 .get("content")
2748 .and_then(Value::as_array)
2749 .is_some_and(|blocks| {
2750 blocks
2751 .iter()
2752 .any(|block| block.get("type").and_then(Value::as_str) == Some("image"))
2753 })
2754}
2755
2756#[derive(Debug, Default, Clone, PartialEq, Eq)]
2758struct ResponseUsage {
2759 model: Option<String>,
2760 in_tokens: u64,
2761 cache_write_tokens: u64,
2762 cache_read_tokens: u64,
2763 out_tokens: u64,
2764}
2765
2766fn response_usage(body: &[u8]) -> ResponseUsage {
2774 let Ok(json) = serde_json::from_slice::<Value>(body) else {
2775 return ResponseUsage::default();
2776 };
2777 let u = |ptr: &str| json.pointer(ptr).and_then(Value::as_u64).unwrap_or(0);
2778 ResponseUsage {
2779 model: json.get("model").and_then(Value::as_str).map(str::to_owned),
2780 in_tokens: u("/usage/input_tokens"),
2781 cache_write_tokens: u("/usage/cache_creation_input_tokens"),
2782 cache_read_tokens: u("/usage/cache_read_input_tokens"),
2783 out_tokens: u("/usage/output_tokens"),
2784 }
2785}
2786
2787fn build_trace(
2789 config: &ProxyConfig,
2790 req_body: &Bytes,
2791 resp_body: &Bytes,
2792 latency_ms: u64,
2793 session_header: Option<&str>,
2794) -> Trace {
2795 let (req_model, tool_count, has_images) = request_features(req_body);
2796 let usage = response_usage(resp_body);
2797 let (resp_model, in_tokens, out_tokens) = (usage.model, usage.in_tokens, usage.out_tokens);
2798 let (cache_write_tokens, cache_read_tokens) =
2799 (usage.cache_write_tokens, usage.cache_read_tokens);
2800 let model = resp_model
2801 .or(req_model)
2802 .unwrap_or_else(|| "unknown".to_owned());
2803
2804 let cost_usd = config
2805 .prices
2806 .cost_usd_with_cache(
2807 &format!("anthropic/{model}"),
2808 in_tokens,
2809 cache_write_tokens,
2810 cache_read_tokens,
2811 out_tokens,
2812 )
2813 .unwrap_or(0.0);
2814
2815 let attempt = Attempt {
2816 rung: 0,
2817 model,
2818 provider: "anthropic".to_owned(),
2819 in_tokens,
2820 cache_write_tokens,
2821 cache_read_tokens,
2822 out_tokens,
2823 cost_usd,
2824 latency_ms,
2825 gates: Vec::new(),
2826 verdict: Verdict::Pass,
2827 };
2828
2829 let mut trace = base_trace(config, req_body, latency_ms, session_header);
2830 trace.request.features.prompt_token_bucket = token_bucket(in_tokens);
2831 trace.request.features.tool_count = tool_count;
2832 trace.request.features.has_images = has_images;
2833 trace.attempts.push(attempt);
2834 trace.final_ = FinalOutcome {
2835 served_rung: Some(0),
2836 served_from: ServedFrom::Attempt,
2837 total_cost_usd: cost_usd,
2838 gate_cost_usd: 0.0,
2839 total_latency_ms: latency_ms,
2840 escalations: 0,
2841 counterfactual_baseline_usd: cost_usd,
2842 savings_usd: 0.0,
2843 cache_source: None,
2844 };
2845 trace.recompute_savings();
2846 trace
2847}
2848
2849fn build_stream_trace(
2853 config: &ProxyConfig,
2854 req_body: &Bytes,
2855 latency_ms: u64,
2856 session_header: Option<&str>,
2857) -> Trace {
2858 let (req_model, tool_count, has_images) = request_features(req_body);
2859 let model = req_model.unwrap_or_else(|| "unknown".to_owned());
2860
2861 let attempt = Attempt {
2862 rung: 0,
2863 model,
2864 provider: "anthropic".to_owned(),
2865 in_tokens: 0,
2866 cache_write_tokens: 0,
2868 cache_read_tokens: 0,
2869 out_tokens: 0,
2870 cost_usd: 0.0,
2871 latency_ms,
2872 gates: Vec::new(),
2873 verdict: Verdict::Pass,
2874 };
2875
2876 let mut trace = base_trace(config, req_body, latency_ms, session_header);
2877 trace.request.features.tool_count = tool_count;
2878 trace.request.features.has_images = has_images;
2879 trace.attempts.push(attempt);
2880 trace.final_ = FinalOutcome {
2881 served_rung: Some(0),
2882 served_from: ServedFrom::Attempt,
2883 total_cost_usd: 0.0,
2884 gate_cost_usd: 0.0,
2885 total_latency_ms: latency_ms,
2886 escalations: 0,
2887 counterfactual_baseline_usd: 0.0,
2888 savings_usd: 0.0,
2889 cache_source: None,
2890 };
2891 trace.recompute_savings();
2892 trace
2893}
2894
2895fn build_error_trace(
2899 config: &ProxyConfig,
2900 req_body: &Bytes,
2901 latency_ms: u64,
2902 session_header: Option<&str>,
2903) -> Trace {
2904 let (_, tool_count, has_images) = request_features(req_body);
2905 let mut trace = base_trace(config, req_body, latency_ms, session_header);
2906 trace.request.features.tool_count = tool_count;
2907 trace.request.features.has_images = has_images;
2908 trace.final_ = FinalOutcome {
2909 served_rung: None,
2910 served_from: ServedFrom::Error,
2911 total_cost_usd: 0.0,
2912 gate_cost_usd: 0.0,
2913 total_latency_ms: latency_ms,
2914 escalations: 0,
2915 counterfactual_baseline_usd: 0.0,
2916 savings_usd: 0.0,
2917 cache_source: None,
2918 };
2919 trace.recompute_savings();
2920 trace
2921}
2922
2923fn base_trace(
2926 config: &ProxyConfig,
2927 req_body: &Bytes,
2928 latency_ms: u64,
2929 session_header: Option<&str>,
2930) -> Trace {
2931 let trace_id = Uuid::now_v7();
2932 let mut features = Features::new(TaskKind::Other);
2933 features.hour_bucket = hour_bucket(jiff::Timestamp::now());
2934
2935 Trace {
2936 trace_id,
2937 prev_hash: GENESIS_HASH.to_owned(),
2938 tenant_id: config.tenant_id.clone(),
2939 session_id: session_id(session_header, trace_id),
2940 ts: jiff::Timestamp::now(),
2941 mode: Mode::Observe,
2942 policy: PolicyRef {
2943 id: "observe-passthrough@v0".to_owned(),
2944 explore: false,
2945 propensity: None,
2946 mode_profile: None,
2947 },
2948 request: RequestInfo {
2949 api: "anthropic.messages".to_owned(),
2950 prompt_hash: prompt_hash(&config.prompt_salt, req_body),
2951 features,
2952 },
2953 attempts: Vec::new(),
2954 deferred: Vec::new(),
2955 final_: FinalOutcome {
2956 served_rung: None,
2957 served_from: ServedFrom::Error,
2958 total_cost_usd: 0.0,
2959 gate_cost_usd: 0.0,
2960 total_latency_ms: latency_ms,
2961 escalations: 0,
2962 counterfactual_baseline_usd: 0.0,
2963 savings_usd: 0.0,
2964 cache_source: None,
2965 },
2966 probe: None,
2967 rollout: None,
2968 shadow: None,
2969 route_ix: None,
2970 predicted_pass: None,
2971 elastic: None,
2972 }
2973}
2974
2975fn openai_messages_have_tool_calls(body: &[u8]) -> bool {
2980 serde_json::from_slice::<Value>(body)
2981 .ok()
2982 .and_then(|json| {
2983 json.get("messages").and_then(Value::as_array).map(|msgs| {
2984 msgs.iter().any(|m| {
2985 m.get("tool_calls").is_some()
2986 || m.get("role").and_then(Value::as_str) == Some("tool")
2987 })
2988 })
2989 })
2990 .unwrap_or(false)
2991}
2992
2993fn openai_has_http_images(body: &[u8]) -> bool {
2997 serde_json::from_slice::<Value>(body)
2998 .ok()
2999 .and_then(|json| {
3000 json.get("messages").and_then(Value::as_array).map(|msgs| {
3001 msgs.iter().any(|m| {
3002 m.get("content")
3003 .and_then(Value::as_array)
3004 .is_some_and(|parts| {
3005 parts.iter().any(|p| {
3006 p.get("type").and_then(Value::as_str) == Some("image_url")
3007 && p.pointer("/image_url/url")
3008 .and_then(Value::as_str)
3009 .is_some_and(|u| {
3010 u.starts_with("http://") || u.starts_with("https://")
3011 })
3012 })
3013 })
3014 })
3015 })
3016 })
3017 .unwrap_or(false)
3018}
3019
3020fn openai_messages_have_images(body: &[u8]) -> bool {
3023 serde_json::from_slice::<Value>(body)
3024 .ok()
3025 .and_then(|json| {
3026 json.get("messages").and_then(Value::as_array).map(|msgs| {
3027 msgs.iter().any(|m| {
3028 m.get("content")
3029 .and_then(Value::as_array)
3030 .is_some_and(|parts| {
3031 parts
3032 .iter()
3033 .any(|p| p.get("type").and_then(Value::as_str) == Some("image_url"))
3034 })
3035 })
3036 })
3037 })
3038 .unwrap_or(false)
3039}
3040
3041fn extract_openai_features(headers: &HeaderMap, body: &[u8]) -> Features {
3044 let Ok(json) = serde_json::from_slice::<Value>(body) else {
3045 let mut f = Features::new(TaskKind::Other);
3046 f.hour_bucket = hour_bucket(jiff::Timestamp::now());
3047 return f;
3048 };
3049 let tool_count = json
3050 .get("tools")
3051 .and_then(Value::as_array)
3052 .map_or(0, |tools| u32::try_from(tools.len()).unwrap_or(u32::MAX));
3053 let has_images = openai_messages_have_images(body);
3054 let mut f = Features::new(TaskKind::Other);
3055 f.agent = header_str(headers, AGENT_HEADER);
3056 f.subagent = header_str(headers, SUBAGENT_HEADER);
3057 f.tool_count = tool_count;
3058 f.has_images = has_images;
3059 f.prompt_token_bucket = token_bucket(body.len() as u64);
3060 f.hour_bucket = hour_bucket(jiff::Timestamp::now());
3061 f.difficulty_hint = DifficultyHint::score(trajectory_signals(body)).as_u8();
3065 f
3066}
3067
3068fn parse_data_url(url: &str) -> Option<(&str, &str)> {
3072 let rest = url.strip_prefix("data:")?;
3073 let (meta, data) = rest.split_once(',')?;
3074 let media_type = meta.strip_suffix(";base64")?;
3075 Some((media_type, data))
3076}
3077
3078fn translate_openai_user_content(content: &Value) -> Option<Value> {
3081 match content {
3082 Value::String(_) => Some(content.clone()),
3084 Value::Array(parts) => {
3085 let mut blocks: Vec<Value> = Vec::with_capacity(parts.len());
3086 for part in parts {
3087 match part.get("type").and_then(Value::as_str) {
3088 Some("text") => {
3089 let text = part.get("text").and_then(Value::as_str).unwrap_or("");
3090 blocks.push(serde_json::json!({ "type": "text", "text": text }));
3091 }
3092 Some("image_url") => {
3093 let url = part.pointer("/image_url/url").and_then(Value::as_str)?;
3094 if url.starts_with("http://") || url.starts_with("https://") {
3095 return None; }
3097 let (media_type, data) = parse_data_url(url)?;
3099 blocks.push(serde_json::json!({
3100 "type": "image",
3101 "source": { "type": "base64", "media_type": media_type, "data": data }
3102 }));
3103 }
3104 _ => {} }
3106 }
3107 Some(Value::Array(blocks))
3108 }
3109 _ => Some(Value::String(String::new())),
3110 }
3111}
3112
3113fn translate_openai_tools(tools: &Value) -> Value {
3117 let Some(arr) = tools.as_array() else {
3118 return Value::Null;
3119 };
3120 let converted: Vec<Value> = arr
3121 .iter()
3122 .map(|tool| {
3123 let func = tool.get("function").unwrap_or(&Value::Null);
3124 let mut out = serde_json::json!({
3125 "name": func.get("name").cloned().unwrap_or(Value::String(String::new())),
3126 "input_schema": func.get("parameters").cloned()
3127 .unwrap_or_else(|| serde_json::json!({ "type": "object" })),
3128 });
3129 if let Some(desc) = func.get("description") {
3130 out["description"] = desc.clone();
3131 }
3132 out
3133 })
3134 .collect();
3135 Value::Array(converted)
3136}
3137
3138fn translate_openai_tool_choice(tc: &Value) -> Value {
3140 match tc {
3141 Value::String(s) => match s.as_str() {
3142 "auto" => serde_json::json!({ "type": "auto" }),
3143 "required" => serde_json::json!({ "type": "any" }),
3144 _ => serde_json::json!({ "type": "auto" }),
3146 },
3147 Value::Object(_) => {
3148 if tc.get("type").and_then(Value::as_str) == Some("function") {
3150 let name = tc.pointer("/function/name").cloned().unwrap_or(Value::Null);
3151 serde_json::json!({ "type": "tool", "name": name })
3152 } else {
3153 serde_json::json!({ "type": "auto" })
3154 }
3155 }
3156 _ => serde_json::json!({ "type": "auto" }),
3157 }
3158}
3159
3160pub fn parse_openai_request(body: &[u8], carry_raw: bool) -> Option<ModelRequest> {
3173 let json: Value = serde_json::from_slice(body).ok()?;
3174 let raw = if carry_raw { json.clone() } else { Value::Null };
3175
3176 let messages_json = json.get("messages")?.as_array()?;
3177
3178 let mut system: Option<String> = None;
3179 let mut messages: Vec<ChatMessage> = Vec::with_capacity(messages_json.len());
3180 let mut tools = Value::Null;
3181 let mut tool_choice_override: Option<Value> = None;
3182
3183 for msg in messages_json {
3184 let role = msg.get("role").and_then(Value::as_str).unwrap_or("user");
3185 match role {
3186 "system" => {
3187 if let Some(s) = msg.get("content").and_then(Value::as_str) {
3192 system = Some(s.to_owned());
3193 }
3194 }
3195 "user" => {
3196 let content_val = msg.get("content").unwrap_or(&Value::Null);
3197 let translated = translate_openai_user_content(content_val)?;
3198 messages.push(ChatMessage {
3199 role: "user".to_owned(),
3200 content: translated,
3201 });
3202 }
3203 "assistant" => {
3204 if let Some(tc_arr) = msg.get("tool_calls").and_then(Value::as_array) {
3205 let mut blocks: Vec<Value> = Vec::new();
3207 if let Some(text) = msg.get("content").and_then(Value::as_str)
3209 && !text.is_empty()
3210 {
3211 blocks.push(serde_json::json!({ "type": "text", "text": text }));
3212 }
3213 for tc in tc_arr {
3214 let id = tc.get("id").and_then(Value::as_str).unwrap_or("");
3215 let func = tc.get("function").unwrap_or(&Value::Null);
3216 let name = func.get("name").and_then(Value::as_str).unwrap_or("");
3217 let args_str = func
3218 .get("arguments")
3219 .and_then(Value::as_str)
3220 .unwrap_or("{}");
3221 let input: Value = serde_json::from_str(args_str)
3222 .unwrap_or_else(|_| serde_json::json!({}));
3223 blocks.push(serde_json::json!({
3224 "type": "tool_use",
3225 "id": id,
3226 "name": name,
3227 "input": input,
3228 }));
3229 }
3230 messages.push(ChatMessage {
3231 role: "assistant".to_owned(),
3232 content: Value::Array(blocks),
3233 });
3234 } else {
3235 let content = msg
3237 .get("content")
3238 .cloned()
3239 .unwrap_or_else(|| Value::String(String::new()));
3240 messages.push(ChatMessage {
3241 role: "assistant".to_owned(),
3242 content,
3243 });
3244 }
3245 }
3246 "tool" => {
3247 let tool_call_id = msg
3249 .get("tool_call_id")
3250 .and_then(Value::as_str)
3251 .unwrap_or("");
3252 let content = msg
3253 .get("content")
3254 .cloned()
3255 .unwrap_or_else(|| Value::String(String::new()));
3256 let result_block = serde_json::json!({
3257 "type": "tool_result",
3258 "tool_use_id": tool_call_id,
3259 "content": content,
3260 });
3261 messages.push(ChatMessage {
3262 role: "user".to_owned(),
3263 content: Value::Array(vec![result_block]),
3264 });
3265 }
3266 _ => {} }
3268 }
3269
3270 if !carry_raw {
3272 if let Some(t) = json.get("tools") {
3273 tools = translate_openai_tools(t);
3274 }
3275 if let Some(tc) = json.get("tool_choice") {
3276 tool_choice_override = Some(translate_openai_tool_choice(tc));
3277 }
3278 } else {
3279 tools = json.get("tools").cloned().unwrap_or(Value::Null);
3280 }
3281
3282 let max_tokens = json
3283 .get("max_tokens")
3284 .and_then(Value::as_u64)
3285 .and_then(|n| u32::try_from(n).ok())
3286 .unwrap_or(1024);
3287
3288 let model = json
3289 .get("model")
3290 .and_then(Value::as_str)
3291 .unwrap_or_default()
3292 .to_owned();
3293
3294 let _ = tool_choice_override; Some(ModelRequest {
3305 model,
3306 system,
3307 messages,
3308 max_tokens,
3309 tools,
3310 raw,
3311 cache_prefix: false,
3312 })
3313}
3314
3315fn extract_openai_content_and_tools(raw: &Value, text: &str) -> (Value, Option<Value>) {
3322 if let Some(blocks) = raw.get("content").and_then(Value::as_array) {
3324 let mut text_parts: Vec<&str> = Vec::new();
3325 let mut tool_calls: Vec<Value> = Vec::new();
3326 for block in blocks {
3327 match block.get("type").and_then(Value::as_str) {
3328 Some("text") => {
3329 if let Some(t) = block.get("text").and_then(Value::as_str) {
3330 text_parts.push(t);
3331 }
3332 }
3333 Some("tool_use") => {
3334 let id = block.get("id").and_then(Value::as_str).unwrap_or("");
3335 let name = block.get("name").and_then(Value::as_str).unwrap_or("");
3336 let input_str = block
3337 .get("input")
3338 .map_or_else(|| "{}".to_owned(), std::string::ToString::to_string);
3339 tool_calls.push(serde_json::json!({
3340 "id": id,
3341 "type": "function",
3342 "function": { "name": name, "arguments": input_str },
3343 }));
3344 }
3345 _ => {}
3346 }
3347 }
3348 let content_text = if tool_calls.is_empty() || !text_parts.is_empty() {
3349 Value::String(text_parts.join(""))
3350 } else {
3351 Value::Null };
3353 let tc = if tool_calls.is_empty() {
3354 None
3355 } else {
3356 Some(Value::Array(tool_calls))
3357 };
3358 return (content_text, tc);
3359 }
3360
3361 if let Some(msg) = raw.pointer("/choices/0/message") {
3363 let content = msg
3364 .get("content")
3365 .cloned()
3366 .unwrap_or(Value::String(text.to_owned()));
3367 let tc = msg.get("tool_calls").cloned();
3368 return (content, tc);
3369 }
3370
3371 (Value::String(text.to_owned()), None)
3373}
3374
3375fn openai_response_json(resp: &ModelResponse) -> Value {
3378 let (content_text, tool_calls) = extract_openai_content_and_tools(&resp.raw, &resp.text);
3379 let finish_reason = if tool_calls.is_some() {
3380 "tool_calls"
3381 } else {
3382 "stop"
3383 };
3384 let mut message = serde_json::json!({
3385 "role": "assistant",
3386 "content": content_text,
3387 });
3388 if let Some(tc) = tool_calls {
3389 message["tool_calls"] = tc;
3390 }
3391 serde_json::json!({
3392 "id": format!("chatcmpl-{}", Uuid::now_v7()),
3393 "object": "chat.completion",
3394 "created": jiff::Timestamp::now().as_second(),
3395 "model": resp.model,
3396 "choices": [{
3397 "index": 0,
3398 "message": message,
3399 "finish_reason": finish_reason,
3400 }],
3401 "usage": {
3402 "prompt_tokens": resp.in_tokens,
3403 "completion_tokens": resp.out_tokens,
3404 "total_tokens": resp.in_tokens + resp.out_tokens,
3405 }
3406 })
3407}
3408
3409fn openai_sse_from_message(message: &Value) -> String {
3413 let id = message
3414 .get("id")
3415 .and_then(Value::as_str)
3416 .unwrap_or("chatcmpl-unknown")
3417 .to_owned();
3418 let created = message.get("created").cloned().unwrap_or(Value::from(0));
3419 let model = message
3420 .get("model")
3421 .and_then(Value::as_str)
3422 .unwrap_or("unknown");
3423
3424 let choices = message.get("choices").and_then(Value::as_array);
3425 let msg = choices
3426 .and_then(|c| c.first())
3427 .and_then(|c| c.get("message"));
3428 let content = msg
3429 .and_then(|m| m.get("content"))
3430 .cloned()
3431 .unwrap_or(Value::Null);
3432 let tool_calls = msg.and_then(|m| m.get("tool_calls")).cloned();
3433 let finish_reason = choices
3434 .and_then(|c| c.first())
3435 .and_then(|c| c.get("finish_reason"))
3436 .cloned()
3437 .unwrap_or_else(|| Value::String("stop".to_owned()));
3438
3439 let mut out = String::new();
3440 let chunk = |delta: Value| {
3441 serde_json::json!({
3442 "id": id,
3443 "object": "chat.completion.chunk",
3444 "created": created,
3445 "model": model,
3446 "choices": [{ "index": 0, "delta": delta, "finish_reason": Value::Null }]
3447 })
3448 };
3449
3450 let role_chunk = chunk(serde_json::json!({ "role": "assistant", "content": "" }));
3452 out.push_str("data: ");
3453 out.push_str(&role_chunk.to_string());
3454 out.push_str("\n\n");
3455
3456 if let Value::String(text) = &content
3458 && !text.is_empty()
3459 {
3460 let content_chunk = chunk(serde_json::json!({ "content": text }));
3461 out.push_str("data: ");
3462 out.push_str(&content_chunk.to_string());
3463 out.push_str("\n\n");
3464 }
3465
3466 if let Some(tc) = tool_calls {
3468 let tc_chunk = chunk(serde_json::json!({ "tool_calls": tc }));
3469 out.push_str("data: ");
3470 out.push_str(&tc_chunk.to_string());
3471 out.push_str("\n\n");
3472 }
3473
3474 let finish_chunk = serde_json::json!({
3476 "id": id,
3477 "object": "chat.completion.chunk",
3478 "created": created,
3479 "model": model,
3480 "choices": [{ "index": 0, "delta": {}, "finish_reason": finish_reason }]
3481 });
3482 out.push_str("data: ");
3483 out.push_str(&finish_chunk.to_string());
3484 out.push_str("\n\n");
3485
3486 out.push_str("data: [DONE]\n\n");
3487 out
3488}
3489
3490#[allow(clippy::too_many_arguments)]
3495async fn handle_enforce_openai(
3496 state: &AppState,
3497 headers: &HeaderMap,
3498 body: &Bytes,
3499 features: Features,
3500 route: &Route,
3501 route_ix: usize,
3504 session_header: Option<String>,
3505 tenant: String,
3506 routing_mode: RoutingMode,
3507) -> Response {
3508 if is_stream_request(body) {
3509 let (tx, rx) = tokio::sync::oneshot::channel::<Result<Value, ProxyError>>();
3510 let (state_c, headers_c, body_c, route_c) =
3511 (state.clone(), headers.clone(), body.clone(), route.clone());
3512 tokio::spawn(async move {
3513 let out = enforce_pipeline_openai(
3514 &state_c,
3515 &headers_c,
3516 &body_c,
3517 features,
3518 &route_c,
3519 route_ix,
3520 session_header,
3521 tenant,
3522 routing_mode,
3523 )
3524 .await;
3525 let _ = tx.send(out);
3526 });
3527 return sse_keepalive_response(rx, openai_sse_from_message);
3528 }
3529 match enforce_pipeline_openai(
3530 state,
3531 headers,
3532 body,
3533 features,
3534 route,
3535 route_ix,
3536 session_header,
3537 tenant,
3538 routing_mode,
3539 )
3540 .await
3541 {
3542 Ok(message) => (axum::http::StatusCode::OK, Json(message)).into_response(),
3543 Err(e) => e.into_response(),
3544 }
3545}
3546
3547async fn observe_passthrough_openai(
3553 state: AppState,
3554 headers: HeaderMap,
3555 body: Bytes,
3556 session_header: Option<String>,
3557 tenant: String,
3558) -> Response {
3559 observe_passthrough_openai_path(
3560 state,
3561 headers,
3562 body,
3563 session_header,
3564 tenant,
3565 crate::upstream::OPENAI_CHAT_PATH,
3566 )
3567 .await
3568}
3569
3570async fn observe_passthrough_openai_path(
3576 state: AppState,
3577 headers: HeaderMap,
3578 body: Bytes,
3579 session_header: Option<String>,
3580 tenant: String,
3581 path: &'static str,
3582) -> Response {
3583 if is_stream_request(&body) {
3584 return observe_stream_openai_path(state, headers, body, session_header, tenant, path)
3585 .await;
3586 }
3587 let start = Instant::now();
3588 let result = crate::upstream::forward_openai_path(
3589 &state.http,
3590 &state.config.upstream_openai,
3591 path,
3592 &headers,
3593 body.clone(),
3594 )
3595 .await;
3596 let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
3597 match result {
3598 Ok((status, resp_headers, resp_body)) => {
3599 spawn_trace(
3600 &state,
3601 body,
3602 Some(resp_body.clone()),
3603 latency_ms,
3604 session_header,
3605 tenant,
3606 );
3607 (status, resp_headers, resp_body).into_response()
3608 }
3609 Err(err) => {
3610 spawn_trace(&state, body, None, latency_ms, session_header, tenant);
3611 err.into_response()
3612 }
3613 }
3614}
3615
3616async fn observe_stream_openai_path(
3620 state: AppState,
3621 headers: HeaderMap,
3622 body: Bytes,
3623 session_header: Option<String>,
3624 tenant: String,
3625 path: &'static str,
3626) -> Response {
3627 let start = Instant::now();
3628 let result = crate::upstream::forward_openai_streaming_path(
3629 &state.http,
3630 &state.config.upstream_openai,
3631 path,
3632 &headers,
3633 body.clone(),
3634 )
3635 .await;
3636 let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
3637 match result {
3638 Ok((status, resp_headers, response)) => {
3639 spawn_stream_trace(&state, body, latency_ms, session_header, tenant);
3640 let stream_body = Body::from_stream(response.bytes_stream());
3641 (status, resp_headers, stream_body).into_response()
3642 }
3643 Err(err) => {
3644 spawn_trace(&state, body, None, latency_ms, session_header, tenant);
3645 err.into_response()
3646 }
3647 }
3648}
3649
3650async fn chat_completions(
3655 State(state): State<AppState>,
3656 Extension(TenantId(tenant)): Extension<TenantId>,
3657 headers: HeaderMap,
3658 body: Bytes,
3659) -> Response {
3660 let session_header = header_str(&headers, SESSION_HEADER);
3661
3662 if let Some(routing) = state.config.routing.as_ref() {
3663 let features = extract_openai_features(&headers, &body);
3664 if let Some(route) = routing
3665 .route_for(&features)
3666 .filter(|r| r.mode == Mode::Enforce && !r.ladder.is_empty())
3667 {
3668 let route_ix = routing
3670 .routes
3671 .iter()
3672 .position(|r| std::ptr::eq(r, route))
3673 .unwrap_or(0);
3674 let route = route.clone();
3675 let routing_mode = resolve_mode(&headers, &route, &state.config);
3677 if routing_mode == RoutingMode::Observe {
3679 return observe_passthrough_openai(state, headers, body, session_header, tenant)
3680 .await;
3681 }
3682 if enforce_can_handle(
3683 &features,
3684 &body,
3685 routing.escalation.enforce_structured,
3686 &route.ladder,
3687 &state.providers,
3688 Dialect::Openai,
3689 ) {
3690 return handle_enforce_openai(
3691 &state,
3692 &headers,
3693 &body,
3694 features,
3695 &route,
3696 route_ix,
3697 session_header,
3698 tenant,
3699 routing_mode,
3700 )
3701 .await;
3702 }
3703 tracing::info!(
3704 "enforce route matched but OpenAI structured request can't be routed faithfully (flag/ladder); serving via observe passthrough"
3705 );
3706 }
3707 }
3708 observe_passthrough_openai(state, headers, body, session_header, tenant).await
3709}
3710
3711async fn responses(
3720 State(state): State<AppState>,
3721 Extension(TenantId(tenant)): Extension<TenantId>,
3722 headers: HeaderMap,
3723 body: Bytes,
3724) -> Response {
3725 let session_header = header_str(&headers, SESSION_HEADER);
3726
3727 if is_stream_request(&body) {
3731 return observe_passthrough_openai_path(
3732 state,
3733 headers,
3734 body,
3735 session_header,
3736 tenant,
3737 crate::upstream::OPENAI_RESPONSES_PATH,
3738 )
3739 .await;
3740 }
3741
3742 let Ok(parsed) = serde_json::from_slice::<Value>(&body) else {
3743 return observe_passthrough_openai_path(
3744 state,
3745 headers,
3746 body,
3747 session_header,
3748 tenant,
3749 crate::upstream::OPENAI_RESPONSES_PATH,
3750 )
3751 .await;
3752 };
3753 if crate::responses::has_untranslatable_content(&parsed) {
3757 return observe_passthrough_openai_path(
3758 state,
3759 headers,
3760 body,
3761 session_header,
3762 tenant,
3763 crate::upstream::OPENAI_RESPONSES_PATH,
3764 )
3765 .await;
3766 }
3767 let chat_body = Bytes::from(crate::responses::request_to_chat(&parsed).to_string());
3768
3769 if let Some(routing) = state.config.routing.as_ref() {
3770 let features = extract_openai_features(&headers, &chat_body);
3771 if let Some(route) = routing
3772 .route_for(&features)
3773 .filter(|r| r.mode == Mode::Enforce && !r.ladder.is_empty())
3774 {
3775 let route_ix = routing
3776 .routes
3777 .iter()
3778 .position(|r| std::ptr::eq(r, route))
3779 .unwrap_or(0);
3780 let route = route.clone();
3781 let routing_mode = resolve_mode(&headers, &route, &state.config);
3782 if routing_mode != RoutingMode::Observe
3783 && enforce_can_handle(
3784 &features,
3785 &chat_body,
3786 routing.escalation.enforce_structured,
3787 &route.ladder,
3788 &state.providers,
3789 Dialect::Openai,
3790 )
3791 {
3792 return match enforce_pipeline_openai_as(
3793 &state,
3794 &headers,
3795 &chat_body,
3796 &body,
3797 "openai.responses",
3798 features,
3799 &route,
3800 route_ix,
3801 session_header,
3802 tenant,
3803 routing_mode,
3804 )
3805 .await
3806 {
3807 Ok(chat) => Json(crate::responses::response_from_chat(&chat)).into_response(),
3808 Err(e) => e.into_response(),
3809 };
3810 }
3811 }
3812 }
3813 observe_passthrough_openai_path(
3814 state,
3815 headers,
3816 body,
3817 session_header,
3818 tenant,
3819 crate::upstream::OPENAI_RESPONSES_PATH,
3820 )
3821 .await
3822}
3823
3824#[cfg(test)]
3825mod tests {
3826 use bytes::Bytes;
3827
3828 use super::*;
3829
3830 #[test]
3834 fn anthropic_tool_errors_are_extracted() {
3835 let body = br#"{"messages":[
3836 {"role":"assistant","content":[{"type":"tool_use","name":"bash","input":{"cmd":"a"}}]},
3837 {"role":"user","content":[{"type":"tool_result","is_error":true,"content":"boom"}]},
3838 {"role":"assistant","content":[{"type":"tool_use","name":"bash","input":{"cmd":"b"}}]},
3839 {"role":"user","content":[{"type":"tool_result","is_error":true,"content":"boom"}]}
3840 ]}"#;
3841 let s = trajectory_signals(body);
3842 assert_eq!(
3843 s.tool_errors, 2,
3844 "both failing tool_results must be counted"
3845 );
3846 assert_eq!(s.tool_results, 2);
3847 assert_eq!(s.assistant_turns, 2);
3848 assert!(
3849 DifficultyHint::score(s) >= DifficultyHint::Medium,
3850 "an all-failing window must not read as an easy session"
3851 );
3852 }
3853
3854 #[test]
3859 fn openai_tool_errors_are_extracted_from_a_different_shape() {
3860 let body = br#"{"messages":[
3861 {"role":"assistant","tool_calls":[{"function":{"name":"run","arguments":"{\"x\":1}"}}]},
3862 {"role":"tool","content":"Error: command not found"},
3863 {"role":"assistant","tool_calls":[{"function":{"name":"run","arguments":"{\"x\":2}"}}]},
3864 {"role":"tool","content":"Traceback (most recent call last): ..."}
3865 ]}"#;
3866 let s = trajectory_signals(body);
3867 assert_eq!(s.tool_results, 2, "role:tool messages are tool results");
3868 assert_eq!(
3869 s.tool_errors, 2,
3870 "both error-shaped tool outputs must count"
3871 );
3872 assert_eq!(s.assistant_turns, 2);
3873 }
3874
3875 #[test]
3882 fn a_flood_of_leading_whitespace_does_not_defeat_the_bound() {
3883 let flood = format!("{}Error: boom", " ".repeat(4_000_000));
3884 let small = format!("{}Error: boom", " ".repeat(400_000));
3885 let m_big = serde_json::json!({ "role": "tool", "content": flood });
3886 let m_small = serde_json::json!({ "role": "tool", "content": small });
3887
3888 let t = |m: &serde_json::Value| {
3889 let start = std::time::Instant::now();
3890 for _ in 0..20 {
3891 std::hint::black_box(openai_tool_content_looks_like_error(m));
3892 }
3893 start.elapsed()
3894 };
3895 let _warm = t(&m_small);
3896 let small_t = t(&m_small);
3897 let big_t = t(&m_big);
3898 assert!(
3899 big_t < small_t * 5 + std::time::Duration::from_millis(5),
3900 "10x the leading whitespace took {big_t:?} vs {small_t:?} — trimming is scanning the \
3901 whole payload before the bound applies"
3902 );
3903
3904 let indented = serde_json::json!({ "role": "tool", "content": " Error: boom" });
3906 assert!(
3907 openai_tool_content_looks_like_error(&indented),
3908 "ordinary leading whitespace must still be trimmed"
3909 );
3910 }
3911
3912 #[test]
3921 fn a_long_real_conversation_can_actually_reach_the_deep_signal() {
3922 let mut msgs = Vec::new();
3924 for i in 0..40 {
3925 msgs.push(format!(
3926 r#"{{"role":"assistant","content":[{{"type":"tool_use","name":"b","input":{{"i":{i}}}}}]}}"#
3927 ));
3928 msgs.push(
3929 r#"{"role":"user","content":[{"type":"tool_result","is_error":true}]}"#.to_owned(),
3930 );
3931 }
3932 let body = format!(r#"{{"messages":[{}]}}"#, msgs.join(","));
3933 let s = trajectory_signals(body.as_bytes());
3934 assert!(
3935 s.assistant_turns >= 8,
3936 "a 40-turn failing conversation must register as deep, got {} assistant turns — the \
3937 window is counting turns it cannot see",
3938 s.assistant_turns
3939 );
3940 assert_eq!(
3941 DifficultyHint::score(s),
3942 DifficultyHint::High,
3943 "the hardest possible conversation must reach the top bucket, or High is dead code"
3944 );
3945 }
3946
3947 #[test]
3954 fn openai_array_form_tool_content_is_read() {
3955 let arr = serde_json::json!({
3956 "role": "tool",
3957 "content": [{"type": "text", "text": "Error: command not found"}]
3958 });
3959 assert!(
3960 openai_tool_content_looks_like_error(&arr),
3961 "an error in array-form content must be detected"
3962 );
3963
3964 let clean = serde_json::json!({
3965 "role": "tool",
3966 "content": [{"type": "text", "text": "All 42 tests passed"}]
3967 });
3968 assert!(
3969 !openai_tool_content_looks_like_error(&clean),
3970 "clean array-form content must not be a failure"
3971 );
3972
3973 for weird in [
3975 serde_json::json!({"role": "tool", "content": []}),
3976 serde_json::json!({"role": "tool", "content": [{"type": "image"}]}),
3977 serde_json::json!({"role": "tool", "content": [null, 42]}),
3978 ] {
3979 assert!(!openai_tool_content_looks_like_error(&weird));
3980 }
3981
3982 let body = br#"{"messages":[
3984 {"role":"tool","content":[{"type":"text","text":"Error: boom"}]},
3985 {"role":"tool","content":[{"type":"text","text":"Error: boom again"}]},
3986 {"role":"tool","content":[{"type":"text","text":"Error: still broken"}]}
3987 ]}"#;
3988 let s = trajectory_signals(body);
3989 assert_eq!(s.tool_errors, 3, "array-form errors must reach the signals");
3990 assert!(DifficultyHint::score(s) >= DifficultyHint::Medium);
3991 }
3992
3993 #[test]
3999 fn tool_output_merely_mentioning_errors_is_not_counted_as_failure() {
4000 let body = br#"{"messages":[
4001 {"role":"tool","content":"0 errors, 0 warnings"},
4002 {"role":"tool","content":"All 42 tests passed (no failures)"},
4003 {"role":"tool","content":"lint: checked 12 files for errors"}
4004 ]}"#;
4005 let s = trajectory_signals(body);
4006 assert_eq!(s.tool_results, 3);
4007 assert_eq!(
4008 s.tool_errors, 0,
4009 "clean output that mentions the word 'error' must not be scored as a failure"
4010 );
4011 }
4012
4013 #[test]
4023 fn a_multibyte_character_does_not_defeat_the_prefix_bound() {
4024 let padded = format!("{}é{}", "x".repeat(63), "y".repeat(200_000));
4026 let msg = serde_json::json!({ "role": "tool", "content": padded });
4027 assert!(
4028 !openai_tool_content_looks_like_error(&msg),
4029 "a huge non-error payload must not be classified as a failure"
4030 );
4031
4032 let big = format!("{}é{}", "x".repeat(63), "y".repeat(2_000_000));
4040 let small = format!("{}é{}", "x".repeat(63), "y".repeat(200_000));
4041 let m_big = serde_json::json!({ "role": "tool", "content": big });
4042 let m_small = serde_json::json!({ "role": "tool", "content": small });
4043
4044 let t = |m: &serde_json::Value| {
4045 let start = std::time::Instant::now();
4046 for _ in 0..20 {
4047 std::hint::black_box(openai_tool_content_looks_like_error(m));
4048 }
4049 start.elapsed()
4050 };
4051 let _warm = t(&m_small);
4052 let small_t = t(&m_small);
4053 let big_t = t(&m_big);
4054 assert!(
4055 big_t < small_t * 5 + std::time::Duration::from_millis(5),
4056 "scanning a 10x larger payload took {big_t:?} vs {small_t:?} — the prefix bound is not \
4057 holding, so work scales with attacker-controlled input size"
4058 );
4059
4060 let err = format!("Error: boom 💥{}", "z".repeat(200_000));
4062 let msg2 = serde_json::json!({ "role": "tool", "content": err });
4063 assert!(
4064 openai_tool_content_looks_like_error(&msg2),
4065 "bounding the scan must not cost the detection it exists to perform"
4066 );
4067
4068 let late = format!("{}Error: too late", "w".repeat(200));
4071 let msg3 = serde_json::json!({ "role": "tool", "content": late });
4072 assert!(
4073 !openai_tool_content_looks_like_error(&msg3),
4074 "the scan must stay bounded to the head of the message"
4075 );
4076 }
4077
4078 #[test]
4081 fn identical_repeated_tool_calls_are_detected() {
4082 let body = br#"{"messages":[
4083 {"role":"assistant","content":[{"type":"tool_use","name":"ls","input":{"p":"/x"}}]},
4084 {"role":"assistant","content":[{"type":"tool_use","name":"ls","input":{"p":"/x"}}]},
4085 {"role":"assistant","content":[{"type":"tool_use","name":"ls","input":{"p":"/x"}}]},
4086 {"role":"user","content":[{"type":"tool_result","content":"ok"}]}
4087 ]}"#;
4088 let s = trajectory_signals(body);
4089 assert_eq!(
4090 s.repeated_tool_calls, 2,
4091 "three identical calls are two repeats"
4092 );
4093 let distinct = br#"{"messages":[
4095 {"role":"assistant","content":[{"type":"tool_use","name":"ls","input":{"p":"/x"}}]},
4096 {"role":"assistant","content":[{"type":"tool_use","name":"ls","input":{"p":"/y"}}]}
4097 ]}"#;
4098 assert_eq!(trajectory_signals(distinct).repeated_tool_calls, 0);
4099 }
4100
4101 #[test]
4104 fn malformed_bodies_yield_no_signal_and_never_panic() {
4105 for body in [
4106 &b""[..],
4107 b"not json at all",
4108 b"{}",
4109 br#"{"messages":"not-an-array"}"#,
4110 br#"{"messages":[{"role":"assistant","content":"plain text"}]}"#,
4111 br#"{"messages":[null,42,"x"]}"#,
4112 ] {
4113 let s = trajectory_signals(body);
4114 assert_eq!(
4115 DifficultyHint::score(s),
4116 DifficultyHint::None,
4117 "a body with no usable trajectory must score None, not a difficulty level"
4118 );
4119 }
4120
4121 let quiet_success = br#"{"messages":[{"content":[{"type":"tool_result"}]}]}"#;
4127 let s = trajectory_signals(quiet_success);
4128 assert_eq!(s.tool_results, 1);
4129 assert_eq!(s.tool_errors, 0);
4130 assert_eq!(
4131 DifficultyHint::score(s),
4132 DifficultyHint::Low,
4133 "a tool result that reported no error is evidence of health, not absence of evidence"
4134 );
4135 }
4136
4137 #[test]
4141 fn only_the_recent_window_counts_so_old_failures_do_not_ratchet() {
4142 let mut msgs = String::from("{\"messages\":[");
4143 for _ in 0..30 {
4145 msgs.push_str(r#"{"role":"user","content":[{"type":"tool_result","is_error":true}]},"#);
4146 }
4147 for i in 0..12 {
4149 msgs.push_str(&format!(
4150 r#"{{"role":"user","content":[{{"type":"tool_result","is_error":false,"content":"ok{i}"}}]}}{}"#,
4151 if i == 11 { "" } else { "," }
4152 ));
4153 }
4154 msgs.push_str("]}");
4155 let s = trajectory_signals(msgs.as_bytes());
4156 assert_eq!(
4157 s.tool_errors, 0,
4158 "failures outside the window must not count; a recovered session is not a hard one"
4159 );
4160 assert!(s.tool_results > 0, "the recent clean results must be seen");
4161 }
4162
4163 #[test]
4166 fn the_hint_reaches_the_feature_vector_for_both_dialects() {
4167 let hard = br#"{"messages":[
4168 {"role":"assistant","content":[{"type":"tool_use","name":"b","input":{"c":1}}]},
4169 {"role":"user","content":[{"type":"tool_result","is_error":true}]},
4170 {"role":"assistant","content":[{"type":"tool_use","name":"b","input":{"c":1}}]},
4171 {"role":"user","content":[{"type":"tool_result","is_error":true}]},
4172 {"role":"assistant","content":[{"type":"tool_use","name":"b","input":{"c":1}}]},
4173 {"role":"user","content":[{"type":"tool_result","is_error":true}]}
4174 ]}"#;
4175 let h = HeaderMap::new();
4176 assert!(
4177 extract_features(&h, hard).difficulty_hint > 0,
4178 "a visibly struggling Anthropic conversation must carry a non-zero hint"
4179 );
4180 assert!(
4181 extract_openai_features(&h, hard).difficulty_hint > 0,
4182 "the OpenAI extractor must populate the hint too, not silently leave it zero"
4183 );
4184 let plain = br#"{"messages":[{"role":"user","content":"hello"}]}"#;
4185 assert_eq!(
4186 extract_features(&h, plain).difficulty_hint,
4187 0,
4188 "a single-shot request must be unchanged by this feature"
4189 );
4190 }
4191
4192 fn test_config() -> ProxyConfig {
4193 ProxyConfig::from_lookup(|_| None).unwrap()
4194 }
4195
4196 #[test]
4197 fn build_trace_maps_request_and_response_fields() {
4198 let config = test_config();
4199 let req = Bytes::from_static(
4200 br#"{"model":"claude-haiku-4-5","tools":[{"name":"a"}],"messages":[]}"#,
4201 );
4202 let resp = Bytes::from_static(
4203 br#"{"model":"claude-haiku-4-5","usage":{"input_tokens":1200,"output_tokens":300}}"#,
4204 );
4205
4206 let trace = build_trace(&config, &req, &resp, 42, Some("sess-1"));
4207
4208 assert_eq!(trace.request.api, "anthropic.messages");
4209 assert_eq!(trace.session_id, "sess-1");
4210 assert_eq!(trace.attempts.len(), 1);
4211 let attempt = &trace.attempts[0];
4212 assert_eq!(attempt.model, "claude-haiku-4-5");
4213 assert_eq!(attempt.provider, "anthropic");
4214 assert_eq!(attempt.in_tokens, 1200);
4215 assert_eq!(attempt.out_tokens, 300);
4216 assert!(attempt.cost_usd > 0.0);
4217 assert_eq!(trace.request.features.tool_count, 1);
4218 assert!(!trace.request.features.has_images);
4219 assert_eq!(trace.final_.served_rung, Some(0));
4220 }
4221
4222 #[test]
4223 fn build_trace_falls_back_to_trace_id_session_when_header_absent() {
4224 let config = test_config();
4225 let req = Bytes::from_static(b"{}");
4226 let resp = Bytes::from_static(b"{}");
4227
4228 let trace = build_trace(&config, &req, &resp, 1, None);
4229
4230 assert_eq!(trace.session_id, trace.trace_id.to_string());
4231 }
4232
4233 #[test]
4234 fn build_error_trace_has_no_attempts_and_served_from_error() {
4235 let config = test_config();
4236 let req = Bytes::from_static(br#"{"model":"claude-haiku-4-5"}"#);
4237
4238 let trace = build_error_trace(&config, &req, 7, None);
4239
4240 assert!(trace.attempts.is_empty());
4241 assert_eq!(trace.final_.served_from, ServedFrom::Error);
4242 assert_eq!(trace.final_.served_rung, None);
4243 }
4244
4245 #[test]
4246 fn message_with_image_block_sets_has_images() {
4247 let req = br#"{"messages":[{"role":"user","content":[{"type":"image"}]}]}"#;
4248 let (_, _, has_images) = request_features(req);
4249 assert!(has_images);
4250 }
4251
4252 #[test]
4253 fn prompt_hash_never_contains_raw_prompt_text() {
4254 let hash = prompt_hash("salt", b"super secret prompt");
4255 assert!(!hash.contains("secret"));
4256 assert_eq!(hash.len(), 64);
4257 }
4258
4259 #[test]
4260 fn parse_model_request_preserves_content_verbatim_and_projects_text() {
4261 let body = br#"{"model":"m","system":"sys","max_tokens":50,
4262 "messages":[{"role":"user","content":[{"type":"text","text":"a"},{"type":"text","text":"b"}]},
4263 {"role":"assistant","content":"c"}]}"#;
4264 let req = parse_model_request(body).unwrap();
4265 assert_eq!(req.system.as_deref(), Some("sys"));
4266 assert_eq!(req.max_tokens, 50);
4267 assert_eq!(req.messages.len(), 2);
4268 assert_eq!(
4270 req.messages[0].content,
4271 serde_json::json!([{"type":"text","text":"a"},{"type":"text","text":"b"}])
4272 );
4273 assert_eq!(req.messages[1].content, Value::String("c".to_owned()));
4275 assert_eq!(req.messages[0].text_view(), "a\nb");
4277 assert_eq!(req.messages[1].text_view(), "c");
4278 }
4279
4280 #[test]
4281 fn tool_and_image_blocks_survive_the_request_round_trip() {
4282 let body = br#"{"model":"m","max_tokens":50,"messages":[
4284 {"role":"assistant","content":[{"type":"tool_use","id":"t1","name":"calc","input":{"x":1}}]},
4285 {"role":"user","content":[
4286 {"type":"tool_result","tool_use_id":"t1","content":"2"},
4287 {"type":"image","source":{"type":"base64","media_type":"image/png","data":"AA=="}}
4288 ]}]}"#;
4289 let req = parse_model_request(body).unwrap();
4290 let round_tripped = serde_json::to_value(&req.messages).unwrap();
4291 assert_eq!(
4292 round_tripped,
4293 serde_json::json!([
4294 {"role":"assistant","content":[{"type":"tool_use","id":"t1","name":"calc","input":{"x":1}}]},
4295 {"role":"user","content":[
4296 {"type":"tool_result","tool_use_id":"t1","content":"2"},
4297 {"type":"image","source":{"type":"base64","media_type":"image/png","data":"AA=="}}
4298 ]}
4299 ])
4300 );
4301 }
4302
4303 #[test]
4304 fn text_message_serializes_byte_identical_to_a_plain_string() {
4305 let m = ChatMessage::text("user", "hello");
4307 assert_eq!(
4308 serde_json::to_string(&m).unwrap(),
4309 r#"{"role":"user","content":"hello"}"#
4310 );
4311 }
4312
4313 #[test]
4314 fn parse_model_request_rejects_non_message_bodies() {
4315 assert!(parse_model_request(b"not json").is_none());
4316 assert!(parse_model_request(br#"{"no":"messages"}"#).is_none());
4317 }
4318
4319 use crate::provider::{MockProvider, ModelResponse, Provider, ProviderError, ProviderRegistry};
4322 use axum::extract::State;
4323 use std::collections::HashMap;
4324 use std::sync::Arc;
4325 use tokio::sync::mpsc;
4326
4327 fn model_resp(model: &str, text: &str) -> ModelResponse {
4328 ModelResponse {
4329 model: model.to_owned(),
4330 text: text.to_owned(),
4331 in_tokens: 1000,
4332 cache_write_tokens: 0,
4333 cache_read_tokens: 0,
4334 out_tokens: 400,
4335 raw: serde_json::Value::Null,
4336 }
4337 }
4338
4339 fn enforce_state(
4342 ladder: &[&str],
4343 gates: &[&str],
4344 outcomes: Vec<(&str, Result<ModelResponse, ProviderError>)>,
4345 ) -> (AppState, mpsc::Receiver<Trace>) {
4346 let toml = format!(
4347 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\nladder = [{}]\ngates = [{}]\n",
4348 ladder
4349 .iter()
4350 .map(|m| format!("\"{m}\""))
4351 .collect::<Vec<_>>()
4352 .join(", "),
4353 gates
4354 .iter()
4355 .map(|g| format!("\"{g}\""))
4356 .collect::<Vec<_>>()
4357 .join(", "),
4358 );
4359 let config = ProxyConfig::from_lookup(|k| match k {
4360 "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
4361 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
4362 _ => None,
4363 })
4364 .unwrap();
4365
4366 let mut outs = HashMap::new();
4367 for (model, out) in outcomes {
4368 outs.insert(model.to_owned(), out);
4369 }
4370 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
4371 map.insert(
4372 "anthropic".to_owned(),
4373 Arc::new(MockProvider::new("anthropic", outs)),
4374 );
4375 let providers = ProviderRegistry::from_map(map);
4376
4377 let (traces, rx) = mpsc::channel(64);
4378 let state = AppState {
4379 config: Arc::new(config),
4380 http: reqwest::Client::new(),
4381 providers,
4382 gate_health: Arc::new(GateHealthRegistry::new()),
4383 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
4384 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
4385 traces,
4386 adaptive: None,
4387 eprocess: None,
4388 bandit: None,
4389 promoter: None,
4390 verified_cache: None,
4391 predictor: None,
4392 tenant_rate_limiter: None,
4393 spill: None,
4394 };
4395 (state, rx)
4396 }
4397
4398 fn user_body() -> Bytes {
4399 Bytes::from_static(
4400 br#"{"model":"ignored","max_tokens":64,"messages":[{"role":"user","content":"hi"}]}"#,
4401 )
4402 }
4403
4404 async fn body_json(resp: Response) -> Value {
4405 let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
4406 .await
4407 .unwrap();
4408 serde_json::from_slice(&bytes).unwrap()
4409 }
4410
4411 #[tokio::test]
4412 async fn mode_profile_stamped_on_trace_when_non_balanced() {
4413 let (state, mut rx) = enforce_state(
4414 &["anthropic/claude-haiku-4-5"],
4415 &["non-empty"],
4416 vec![(
4417 "anthropic/claude-haiku-4-5",
4418 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
4419 )],
4420 );
4421 let mut headers = HeaderMap::new();
4422 headers.insert("x-firstpass-mode", "quality".parse().unwrap());
4423 let resp = messages(
4424 State(state),
4425 Extension(TenantId("default".to_owned())),
4426 headers,
4427 user_body(),
4428 )
4429 .await;
4430 assert_eq!(resp.status(), axum::http::StatusCode::OK);
4431 let trace = rx.try_recv().expect("trace enqueued");
4432 assert_eq!(
4433 trace.policy.mode_profile.as_deref(),
4434 Some("quality"),
4435 "mode_profile must be stamped when quality mode is active"
4436 );
4437 }
4438
4439 #[tokio::test]
4440 async fn mode_profile_absent_from_trace_when_balanced() {
4441 let (state, mut rx) = enforce_state(
4442 &["anthropic/claude-haiku-4-5"],
4443 &["non-empty"],
4444 vec![(
4445 "anthropic/claude-haiku-4-5",
4446 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
4447 )],
4448 );
4449 let resp = messages(
4451 State(state),
4452 Extension(TenantId("default".to_owned())),
4453 HeaderMap::new(),
4454 user_body(),
4455 )
4456 .await;
4457 assert_eq!(resp.status(), axum::http::StatusCode::OK);
4458 let trace = rx.try_recv().expect("trace enqueued");
4459 assert!(
4460 trace.policy.mode_profile.is_none(),
4461 "mode_profile must be absent when Balanced (byte-identical invariant)"
4462 );
4463 }
4464
4465 #[tokio::test]
4466 async fn enforce_serves_first_pass_and_returns_anthropic_shape() {
4467 let (state, mut rx) = enforce_state(
4468 &["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"],
4469 &["non-empty"],
4470 vec![(
4471 "anthropic/claude-haiku-4-5",
4472 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
4473 )],
4474 );
4475 let resp = messages(
4476 State(state),
4477 Extension(TenantId("default".to_owned())),
4478 HeaderMap::new(),
4479 user_body(),
4480 )
4481 .await;
4482 assert_eq!(resp.status(), axum::http::StatusCode::OK);
4483 let json = body_json(resp).await;
4484 assert_eq!(json["type"], "message");
4485 assert_eq!(json["content"][0]["text"], "hello");
4486 assert_eq!(json["model"], "anthropic/claude-haiku-4-5");
4487
4488 let trace = rx.try_recv().expect("a trace was enqueued");
4489 assert_eq!(trace.mode, Mode::Enforce);
4490 assert_eq!(trace.final_.served_rung, Some(0));
4491 assert_eq!(trace.attempts.len(), 1);
4492 }
4493
4494 #[tokio::test]
4495 async fn enforce_escalates_then_serves_and_traces_two_attempts() {
4496 let (state, mut rx) = enforce_state(
4497 &["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"],
4498 &["non-empty"],
4499 vec![
4500 (
4501 "anthropic/claude-haiku-4-5",
4502 Ok(model_resp("anthropic/claude-haiku-4-5", " ")),
4503 ), (
4505 "anthropic/claude-sonnet-5",
4506 Ok(model_resp("anthropic/claude-sonnet-5", "answer")),
4507 ),
4508 ],
4509 );
4510 let resp = messages(
4511 State(state),
4512 Extension(TenantId("default".to_owned())),
4513 HeaderMap::new(),
4514 user_body(),
4515 )
4516 .await;
4517 let json = body_json(resp).await;
4518 assert_eq!(json["content"][0]["text"], "answer");
4519
4520 let trace = rx.try_recv().expect("trace enqueued");
4521 assert_eq!(trace.attempts.len(), 2);
4522 assert_eq!(trace.final_.escalations, 1);
4523 assert_eq!(trace.final_.served_rung, Some(1));
4524 }
4525
4526 #[tokio::test]
4527 async fn enforce_all_rungs_error_returns_502() {
4528 let (state, mut rx) = enforce_state(
4529 &["anthropic/claude-haiku-4-5"],
4530 &["non-empty"],
4531 vec![(
4532 "anthropic/claude-haiku-4-5",
4533 Err(ProviderError::Transport("down".into())),
4534 )],
4535 );
4536 let resp = messages(
4537 State(state),
4538 Extension(TenantId("default".to_owned())),
4539 HeaderMap::new(),
4540 user_body(),
4541 )
4542 .await;
4543 assert_eq!(resp.status(), axum::http::StatusCode::BAD_GATEWAY);
4544 assert!(rx.try_recv().is_ok());
4546 }
4547
4548 #[tokio::test]
4549 async fn no_routing_config_falls_through_to_observe_not_enforce() {
4550 let config = ProxyConfig::from_lookup(|k| match k {
4554 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
4555 _ => None,
4556 })
4557 .unwrap();
4558 let (traces, _rx) = mpsc::channel(64);
4559 let state = AppState {
4560 config: Arc::new(config),
4561 http: reqwest::Client::new(),
4562 providers: ProviderRegistry::new("http://127.0.0.1:1", "http://127.0.0.1:1"),
4563 gate_health: Arc::new(GateHealthRegistry::new()),
4564 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
4565 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
4566 traces,
4567 adaptive: None,
4568 eprocess: None,
4569 bandit: None,
4570 promoter: None,
4571 verified_cache: None,
4572 predictor: None,
4573 tenant_rate_limiter: None,
4574 spill: None,
4575 };
4576 let resp = messages(
4577 State(state),
4578 Extension(TenantId("default".to_owned())),
4579 HeaderMap::new(),
4580 user_body(),
4581 )
4582 .await;
4583 assert_ne!(resp.status(), axum::http::StatusCode::OK);
4585 }
4586
4587 fn bare_enforce_route() -> Route {
4590 use firstpass_core::config::{Match, Mode};
4591 Route {
4592 match_: Match::default(),
4593 mode: Mode::Enforce,
4594 ladder: vec!["anthropic/claude-haiku-4-5".to_owned()],
4595 gates: vec![],
4596 deferred_gates: vec![],
4597 routing_mode: None,
4598 rollout: None,
4599 shadow: None,
4600 }
4601 }
4602
4603 #[test]
4604 fn balanced_preset_application_is_noop() {
4605 let base_max_rungs = 3u32;
4607 let base_speculation = 2u32;
4608 let preset = RoutingMode::Balanced.preset();
4609 let max_rungs = if let Some(d) = preset.max_rungs_delta {
4610 (base_max_rungs as i32 + d).max(1) as u32
4611 } else {
4612 base_max_rungs
4613 };
4614 let speculation = preset.speculation.unwrap_or(base_speculation);
4615 let start_at_top = preset.start_at_top;
4616 assert_eq!(max_rungs, 3, "Balanced must not change max_rungs");
4617 assert_eq!(speculation, 2, "Balanced must not change speculation");
4618 assert!(!start_at_top, "Balanced must not set start_at_top");
4619 }
4620
4621 #[test]
4622 fn resolve_mode_header_wins_over_route_and_global() {
4623 let mut headers = HeaderMap::new();
4624 headers.insert("x-firstpass-mode", "cost".parse().unwrap());
4625 let mut route = bare_enforce_route();
4626 route.routing_mode = Some(RoutingMode::Quality); let mut config = test_config();
4628 config.default_routing_mode = RoutingMode::Max; assert_eq!(
4630 resolve_mode(&headers, &route, &config),
4631 RoutingMode::Cost,
4632 "header must win"
4633 );
4634 }
4635
4636 #[test]
4637 fn resolve_mode_route_wins_over_global_when_no_header() {
4638 let mut route = bare_enforce_route();
4639 route.routing_mode = Some(RoutingMode::Latency);
4640 let mut config = test_config();
4641 config.default_routing_mode = RoutingMode::Max;
4642 assert_eq!(
4643 resolve_mode(&HeaderMap::new(), &route, &config),
4644 RoutingMode::Latency,
4645 "route must beat global default"
4646 );
4647 }
4648
4649 #[test]
4650 fn resolve_mode_global_when_header_and_route_absent() {
4651 let mut config = test_config();
4652 config.default_routing_mode = RoutingMode::Quality;
4653 assert_eq!(
4654 resolve_mode(&HeaderMap::new(), &bare_enforce_route(), &config),
4655 RoutingMode::Quality
4656 );
4657 }
4658
4659 #[test]
4660 fn resolve_mode_unknown_header_falls_through_to_route() {
4661 let mut headers = HeaderMap::new();
4662 headers.insert("x-firstpass-mode", "turbo-mode".parse().unwrap());
4664 let mut route = bare_enforce_route();
4665 route.routing_mode = Some(RoutingMode::Cost);
4666 let config = test_config();
4667 assert_eq!(
4668 resolve_mode(&headers, &route, &config),
4669 RoutingMode::Cost,
4670 "unknown header value must fall through to route"
4671 );
4672 }
4673
4674 #[test]
4675 fn resolve_mode_header_case_insensitive() {
4676 let mut headers = HeaderMap::new();
4677 headers.insert("x-firstpass-mode", "QUALITY".parse().unwrap());
4678 assert_eq!(
4679 resolve_mode(&headers, &bare_enforce_route(), &test_config()),
4680 RoutingMode::Quality
4681 );
4682 }
4683
4684 #[test]
4685 fn resolve_mode_no_mode_set_returns_balanced() {
4686 assert_eq!(
4688 resolve_mode(&HeaderMap::new(), &bare_enforce_route(), &test_config()),
4689 RoutingMode::Balanced
4690 );
4691 }
4692
4693 #[test]
4694 fn capabilities_json_includes_routing_modes() {
4695 let modes: Vec<&'static str> = RoutingMode::ALL.iter().map(|m| m.as_str()).collect();
4696 assert!(modes.contains(&"balanced"));
4697 assert!(modes.contains(&"cost"));
4698 assert!(modes.contains(&"quality"));
4699 assert!(modes.contains(&"latency"));
4700 assert!(modes.contains(&"max"));
4701 assert!(modes.contains(&"observe"));
4702 }
4703
4704 #[test]
4705 fn model_list_dedups_across_ladders_keeping_cheapest_first_order() {
4706 let config = firstpass_core::config::Config::parse(
4709 r#"
4710 [[route]]
4711 mode = "enforce"
4712 ladder = ["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"]
4713 [[route]]
4714 mode = "enforce"
4715 ladder = ["anthropic/claude-sonnet-5", "anthropic/claude-opus-4-8"]
4716 "#,
4717 )
4718 .expect("test config parses");
4719
4720 let out = model_list(Some(&config));
4721
4722 let ids: Vec<&str> = out["data"]
4723 .as_array()
4724 .expect("data is an array")
4725 .iter()
4726 .map(|m| m["id"].as_str().expect("id is a string"))
4727 .collect();
4728 assert_eq!(
4729 ids,
4730 [
4731 "anthropic/claude-haiku-4-5",
4732 "anthropic/claude-sonnet-5",
4733 "anthropic/claude-opus-4-8",
4734 ],
4735 "sonnet appears once, and ladder order is preserved"
4736 );
4737 assert_eq!(out["object"], "list");
4738 assert_eq!(out["data"][0]["owned_by"], "anthropic");
4739 let haiku_in = out["data"][0]["firstpass"]["input_per_mtok"]
4742 .as_f64()
4743 .expect("a first-party rung has a price");
4744 assert!(haiku_in > 0.0, "expected a real price, got {haiku_in}");
4745 let opus_in = out["data"][2]["firstpass"]["input_per_mtok"]
4747 .as_f64()
4748 .expect("a first-party rung has a price");
4749 assert!(haiku_in < opus_in, "{haiku_in} should undercut {opus_in}");
4750 }
4751
4752 #[tokio::test]
4753 async fn healthz_names_the_service_so_a_client_can_tell_us_from_a_stranger() {
4754 let body = healthz().await.into_response();
4758 let bytes = axum::body::to_bytes(body.into_body(), 64 * 1024)
4759 .await
4760 .expect("healthz body");
4761 let json: serde_json::Value = serde_json::from_slice(&bytes).expect("healthz is json");
4762 assert_eq!(json["status"], "ok");
4763 assert_eq!(json["service"], "firstpass");
4764 }
4765
4766 #[test]
4767 fn model_list_is_empty_without_routing_config() {
4768 let out = model_list(None);
4769 assert_eq!(out["object"], "list");
4770 assert_eq!(
4771 out["data"].as_array().expect("data is an array").len(),
4772 0,
4773 "an observe-only deployment advertises no models"
4774 );
4775 }
4776
4777 #[test]
4778 fn detects_stream_requests() {
4779 assert!(is_stream_request(br#"{"stream": true}"#));
4780 assert!(!is_stream_request(br#"{"stream": false}"#));
4781 assert!(!is_stream_request(br#"{"model":"m"}"#));
4782 assert!(!is_stream_request(b"not json"));
4783 }
4784
4785 #[test]
4786 fn detects_tool_blocks_in_messages() {
4787 let with =
4788 br#"{"messages":[{"role":"user","content":[{"type":"tool_result","content":"42"}]}]}"#;
4789 let without = br#"{"messages":[{"role":"user","content":"hi"}]}"#;
4790 assert!(messages_have_tool_blocks(with));
4791 assert!(!messages_have_tool_blocks(without));
4792 }
4793
4794 #[test]
4795 fn enforce_only_handles_plain_text() {
4796 let plain =
4797 Bytes::from_static(br#"{"model":"m","messages":[{"role":"user","content":"hi"}]}"#);
4798 let tools = Bytes::from_static(
4799 br#"{"model":"m","tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
4800 );
4801 let f_plain = extract_features(&HeaderMap::new(), &plain);
4802 let f_tools = extract_features(&HeaderMap::new(), &tools);
4803 let anthropic_ladder = vec!["anthropic/claude-haiku-4-5".to_owned()];
4804 let providers = test_registry();
4805 assert!(enforce_can_handle(
4807 &f_plain,
4808 &plain,
4809 false,
4810 &anthropic_ladder,
4811 &providers,
4812 Dialect::Anthropic,
4813 ));
4814 assert!(!enforce_can_handle(
4815 &f_tools,
4816 &tools,
4817 false,
4818 &anthropic_ladder,
4819 &providers,
4820 Dialect::Anthropic,
4821 ));
4822 }
4823
4824 #[test]
4825 fn structured_enforce_routes_tools_and_streaming() {
4826 let tools = Bytes::from_static(
4829 br#"{"model":"m","tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
4830 );
4831 let streaming_tools = Bytes::from_static(
4832 br#"{"model":"m","stream":true,"tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
4833 );
4834 let f = extract_features(&HeaderMap::new(), &tools);
4835 let anthropic_ladder = vec![
4836 "anthropic/claude-haiku-4-5".to_owned(),
4837 "anthropic/claude-sonnet-5".to_owned(),
4838 ];
4839 let providers = test_registry();
4840 assert!(enforce_can_handle(
4841 &f,
4842 &tools,
4843 true,
4844 &anthropic_ladder,
4845 &providers,
4846 Dialect::Anthropic,
4847 ));
4848 assert!(enforce_can_handle(
4849 &f,
4850 &streaming_tools,
4851 true,
4852 &anthropic_ladder,
4853 &providers,
4854 Dialect::Anthropic,
4855 ));
4856 }
4857
4858 fn test_registry() -> crate::provider::ProviderRegistry {
4860 crate::provider::ProviderRegistry::new("http://localhost", "http://localhost")
4861 }
4862
4863 #[test]
4864 fn fidelity_guard_blocks_structured_on_non_verbatim_ladder() {
4865 let tools = Bytes::from_static(
4869 br#"{"model":"m","tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
4870 );
4871 let plain =
4872 Bytes::from_static(br#"{"model":"m","messages":[{"role":"user","content":"hi"}]}"#);
4873 let f_tools = extract_features(&HeaderMap::new(), &tools);
4874 let f_plain = extract_features(&HeaderMap::new(), &plain);
4875 let providers = test_registry();
4876 let mixed_ladder = vec![
4877 "openai/gpt-4.1-mini".to_owned(),
4878 "anthropic/claude-sonnet-5".to_owned(),
4879 ];
4880 assert!(!enforce_can_handle(
4881 &f_tools,
4882 &tools,
4883 true,
4884 &mixed_ladder,
4885 &providers,
4886 Dialect::Anthropic,
4887 ));
4888 assert!(enforce_can_handle(
4889 &f_plain,
4890 &plain,
4891 true,
4892 &mixed_ladder,
4893 &providers,
4894 Dialect::Anthropic,
4895 ));
4896 }
4897
4898 #[test]
4899 fn enforce_sse_reemission_preserves_text_and_tool_use() {
4900 let resp = ModelResponse {
4903 model: "anthropic/claude-haiku-4-5".to_owned(),
4904 text: "let me check".to_owned(),
4905 in_tokens: 5,
4906 cache_write_tokens: 0,
4907 cache_read_tokens: 0,
4908 out_tokens: 7,
4909 raw: serde_json::json!({
4910 "content": [
4911 { "type": "text", "text": "let me check" },
4912 { "type": "tool_use", "id": "tu_1", "name": "get_weather", "input": { "city": "Paris" } }
4913 ]
4914 }),
4915 };
4916 let sse = anthropic_sse_from_message(&anthropic_response_json(&resp));
4917
4918 let frames: Vec<Value> = sse
4920 .lines()
4921 .filter_map(|l| l.strip_prefix("data: "))
4922 .map(|d| serde_json::from_str::<Value>(d).expect("each SSE data frame is valid JSON"))
4923 .collect();
4924
4925 assert_eq!(frames.first().unwrap()["type"], "message_start");
4927 assert_eq!(frames.last().unwrap()["type"], "message_stop");
4928 assert!(frames.iter().any(|f| f["delta"]["type"] == "text_delta"
4930 && f["delta"]["text"] == "let me check"));
4931 assert!(
4934 frames
4935 .iter()
4936 .any(|f| f["content_block"]["type"] == "tool_use"
4937 && f["content_block"]["name"] == "get_weather"
4938 && f["content_block"]["id"] == "tu_1")
4939 );
4940 assert!(
4941 frames
4942 .iter()
4943 .any(|f| f["delta"]["type"] == "input_json_delta"
4944 && f["delta"]["partial_json"] == r#"{"city":"Paris"}"#)
4945 );
4946 }
4947
4948 #[tokio::test]
4953 async fn enforce_falls_back_to_observe_for_tool_requests() {
4954 let toml = "[[price]]\nmodel = \"anthropic/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/m\"]\ngates = [\"non-empty\"]\n";
4955 let config = ProxyConfig::from_lookup(|k| match k {
4956 "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
4957 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
4958 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
4959 _ => None,
4960 })
4961 .unwrap();
4962 let mut outs = HashMap::new();
4963 outs.insert(
4964 "anthropic/m".to_owned(),
4965 Ok(model_resp("anthropic/m", "hello")),
4966 );
4967 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
4968 map.insert(
4969 "anthropic".to_owned(),
4970 Arc::new(MockProvider::new("anthropic", outs)),
4971 );
4972 let (traces, _rx) = mpsc::channel(64);
4973 let state = AppState {
4974 config: Arc::new(config),
4975 http: reqwest::Client::new(),
4976 providers: ProviderRegistry::from_map(map),
4977 gate_health: Arc::new(GateHealthRegistry::new()),
4978 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
4979 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
4980 traces,
4981 adaptive: None,
4982 eprocess: None,
4983 bandit: None,
4984 promoter: None,
4985 verified_cache: None,
4986 predictor: None,
4987 tenant_rate_limiter: None,
4988 spill: None,
4989 };
4990
4991 let plain =
4993 Bytes::from_static(br#"{"model":"m","messages":[{"role":"user","content":"hi"}]}"#);
4994 let resp = messages(
4995 State(state.clone()),
4996 Extension(TenantId("default".to_owned())),
4997 HeaderMap::new(),
4998 plain,
4999 )
5000 .await;
5001 assert_eq!(
5002 resp.status(),
5003 axum::http::StatusCode::OK,
5004 "plain text should enforce"
5005 );
5006
5007 let tools = Bytes::from_static(
5010 br#"{"model":"m","tools":[{"name":"get_weather"}],"messages":[{"role":"user","content":"hi"}]}"#,
5011 );
5012 let resp = messages(
5013 State(state.clone()),
5014 Extension(TenantId("default".to_owned())),
5015 HeaderMap::new(),
5016 tools.clone(),
5017 )
5018 .await;
5019 assert_eq!(
5020 resp.status(),
5021 axum::http::StatusCode::OK,
5022 "tool request must route through enforce by default (ADR 0005 default-on)"
5023 );
5024
5025 let toml_off = "[[price]]\nmodel = \"anthropic/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/m\"]\ngates = [\"non-empty\"]\n[escalation]\nenforce_structured = false\n";
5028 let config_off = ProxyConfig::from_lookup(|k| match k {
5029 "FIRSTPASS_CONFIG_TOML" => Some(toml_off.to_owned()),
5030 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
5031 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
5032 _ => None,
5033 })
5034 .unwrap();
5035 let state_off = AppState {
5036 config: Arc::new(config_off),
5037 ..state.clone()
5038 };
5039 let resp = messages(
5040 State(state_off),
5041 Extension(TenantId("default".to_owned())),
5042 HeaderMap::new(),
5043 tools,
5044 )
5045 .await;
5046 assert_ne!(
5047 resp.status(),
5048 axum::http::StatusCode::OK,
5049 "with enforce_structured = false a tool request must fall back to observe"
5050 );
5051
5052 let toolres = Bytes::from_static(
5054 br#"{"model":"m","messages":[{"role":"user","content":[{"type":"tool_result","tool_use_id":"x","content":"42"}]}]}"#,
5055 );
5056 let resp = messages(
5057 State(state),
5058 Extension(TenantId("default".to_owned())),
5059 HeaderMap::new(),
5060 toolres,
5061 )
5062 .await;
5063 assert_eq!(
5064 resp.status(),
5065 axum::http::StatusCode::OK,
5066 "tool_result blocks route through enforce by default too (verbatim carry)"
5067 );
5068 }
5069
5070 #[tokio::test]
5074 async fn panel_is_public_markup_but_receipts_stay_behind_auth() {
5075 use tower::ServiceExt;
5076 let (state, _rx) = enforce_state(
5077 &["anthropic/claude-haiku-4-5"],
5078 &["non-empty"],
5079 vec![(
5080 "anthropic/claude-haiku-4-5",
5081 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
5082 )],
5083 );
5084 let router = app(state).expect("prometheus recorder installs");
5085
5086 let page = router
5087 .clone()
5088 .oneshot(
5089 axum::http::Request::builder()
5090 .uri("/panel")
5091 .body(Body::empty())
5092 .unwrap(),
5093 )
5094 .await
5095 .unwrap();
5096 assert_eq!(page.status(), axum::http::StatusCode::OK);
5097 let html = axum::body::to_bytes(page.into_body(), 1 << 20)
5098 .await
5099 .unwrap();
5100 let html = String::from_utf8_lossy(&html);
5101 assert!(
5102 html.contains("firstpass") && html.contains("/v1/receipts"),
5103 "the page must fetch its data rather than embed it"
5104 );
5105 assert!(
5108 !html.contains("http://") && !html.contains("https://"),
5109 "the panel must not reference any external origin"
5110 );
5111 assert!(html.contains("const esc ="), "the escaper must exist");
5116 for raw in [
5117 "${a.model}",
5118 "${a.rung}",
5119 "${a.verdict}",
5120 "${r.trace_id}",
5121 "${r.served_rung}",
5122 ] {
5123 assert!(
5124 !html.contains(raw),
5125 "receipt field interpolated unescaped into innerHTML: {raw}"
5126 );
5127 }
5128
5129 let data = router
5130 .oneshot(
5131 axum::http::Request::builder()
5132 .uri("/v1/receipts?limit=5")
5133 .body(Body::empty())
5134 .unwrap(),
5135 )
5136 .await
5137 .unwrap();
5138 assert_eq!(data.status(), axum::http::StatusCode::OK);
5142 let body = axum::body::to_bytes(data.into_body(), 1 << 20)
5143 .await
5144 .unwrap();
5145 let v: Value = serde_json::from_slice(&body).unwrap();
5146 assert!(v.get("receipts").and_then(Value::as_array).is_some());
5147 }
5148
5149 async fn feedback_state() -> (AppState, std::path::PathBuf, String) {
5153 feedback_state_with(false).await
5154 }
5155
5156 async fn feedback_state_with(served: bool) -> (AppState, std::path::PathBuf, String) {
5162 let db = std::env::temp_dir().join(format!("firstpass-feedback-{}.db", Uuid::now_v7()));
5163 let (tx, handle) = crate::store::open(&db).unwrap();
5164
5165 let mut trace = build_error_trace(
5166 &ProxyConfig::from_lookup(|_| None).unwrap(),
5167 &Bytes::from_static(b"{}"),
5168 5,
5169 Some("sess-fb"),
5170 );
5171 trace.attempts.push(Attempt {
5172 rung: 0,
5173 model: "anthropic/claude-haiku-4-5".into(),
5174 provider: "anthropic".into(),
5175 in_tokens: 10,
5176 cache_write_tokens: 0,
5177 cache_read_tokens: 0,
5178 out_tokens: 5,
5179 cost_usd: 0.001,
5180 latency_ms: 5,
5181 gates: vec![],
5182 verdict: Verdict::Pass,
5183 });
5184 let trace_id = trace.trace_id.to_string();
5185 if served {
5186 trace.final_.served_rung = Some(0);
5187 }
5188 tx.try_send(trace).unwrap();
5189 drop(tx);
5190 handle.await.unwrap();
5191
5192 let db_str = db.to_string_lossy().into_owned();
5193 let config = ProxyConfig::from_lookup(move |k| match k {
5194 "FIRSTPASS_DB" => Some(db_str.clone()),
5195 _ => None,
5196 })
5197 .unwrap();
5198 let (traces, _rx) = mpsc::channel(64);
5199 let state = AppState {
5200 config: Arc::new(config),
5201 http: reqwest::Client::new(),
5202 providers: ProviderRegistry::new("http://127.0.0.1:1", "http://127.0.0.1:1"),
5203 gate_health: Arc::new(GateHealthRegistry::new()),
5204 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
5205 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
5206 traces,
5207 adaptive: None,
5208 eprocess: None,
5209 bandit: None,
5210 promoter: None,
5211 verified_cache: None,
5212 predictor: None,
5213 tenant_rate_limiter: None,
5214 spill: None,
5215 };
5216 (state, db, trace_id)
5217 }
5218
5219 #[tokio::test]
5233 async fn feedback_without_a_score_still_advances_the_eprocess() {
5234 use firstpass_core::eprocess::{DEFAULT_BET, EProcessRiskControl};
5235 let (mut state, _db, trace_id) = feedback_state_with(true).await;
5236 let grid: Vec<f64> = (0..=20).map(|i| f64::from(i) / 20.0).collect();
5237 let ep = Arc::new(std::sync::Mutex::new(EProcessRiskControl::new(
5238 0.2,
5239 0.05,
5240 DEFAULT_BET,
5241 &grid,
5242 )));
5243 state.eprocess = Some(ep.clone());
5244 assert_eq!(ep.lock().unwrap().rounds(), 0);
5245
5246 let body = Bytes::from(
5248 serde_json::json!({
5249 "trace_id": trace_id, "gate_id": "tests", "verdict": "pass", "reporter": "ci"
5250 })
5251 .to_string(),
5252 );
5253 let resp = feedback(
5254 State(state.clone()),
5255 Extension(TenantId("default".to_owned())),
5256 body,
5257 )
5258 .await
5259 .into_response();
5260 assert_eq!(resp.status(), axum::http::StatusCode::ACCEPTED);
5262
5263 assert_eq!(
5264 ep.lock().unwrap().rounds(),
5265 1,
5266 "a scoreless feedback call must still advance the controller — keying on the optional \
5267 payload score left it permanently inert"
5268 );
5269 }
5270
5271 #[tokio::test]
5272 async fn feedback_nudges_the_adaptive_threshold() {
5273 use firstpass_core::conformal::AdaptiveConformal;
5274 let (mut state, _db, trace_id) = feedback_state().await;
5275 let aci = Arc::new(std::sync::Mutex::new(AdaptiveConformal::new(0.1, 0.2, 0.5)));
5276 state.adaptive = Some(aci.clone());
5277 let before = aci.lock().unwrap().threshold();
5278
5279 let fail = Bytes::from(
5281 serde_json::json!({ "trace_id": trace_id, "gate_id": "tests", "verdict": "fail", "reporter": "ci" })
5282 .to_string(),
5283 );
5284 assert_eq!(
5285 feedback(
5286 State(state.clone()),
5287 Extension(TenantId("default".to_owned())),
5288 fail
5289 )
5290 .await
5291 .status(),
5292 axum::http::StatusCode::ACCEPTED
5293 );
5294 let after_fail = aci.lock().unwrap().threshold();
5295 assert!(
5296 after_fail > before,
5297 "served fail should raise the live threshold: {before} -> {after_fail}"
5298 );
5299
5300 let pass = Bytes::from(
5302 serde_json::json!({ "trace_id": trace_id, "gate_id": "tests", "verdict": "pass", "reporter": "ci" })
5303 .to_string(),
5304 );
5305 let _ = feedback(
5306 State(state),
5307 Extension(TenantId("default".to_owned())),
5308 pass,
5309 )
5310 .await;
5311 assert!(aci.lock().unwrap().threshold() < after_fail);
5312 }
5313
5314 #[tokio::test]
5315 async fn feedback_records_a_deferred_verdict_without_breaking_the_chain() {
5316 let (state, db, trace_id) = feedback_state().await;
5317 let body = Bytes::from(
5318 serde_json::json!({
5319 "trace_id": trace_id,
5320 "gate_id": "tests",
5321 "verdict": "pass",
5322 "score": 1.0,
5323 "reporter": "ci",
5324 })
5325 .to_string(),
5326 );
5327 let resp = feedback(
5328 State(state),
5329 Extension(TenantId("default".to_owned())),
5330 body,
5331 )
5332 .await;
5333 assert_eq!(resp.status(), axum::http::StatusCode::ACCEPTED);
5334
5335 let view = crate::store::load_trace_view(&db, "default", &trace_id)
5337 .unwrap()
5338 .unwrap();
5339 assert_eq!(view.deferred.len(), 1);
5340 assert_eq!(view.deferred[0].gate_id, "tests");
5341 let traces = crate::store::load_all_traces(&db).unwrap();
5343 firstpass_core::verify_chain(&traces, GENESIS_HASH).unwrap();
5344
5345 let _ = std::fs::remove_file(&db);
5346 }
5347
5348 #[tokio::test]
5349 async fn feedback_for_unknown_trace_is_404() {
5350 let (state, db, _trace_id) = feedback_state().await;
5351 let body = Bytes::from(
5352 serde_json::json!({
5353 "trace_id": "does-not-exist",
5354 "gate_id": "tests",
5355 "verdict": "pass",
5356 "reporter": "ci",
5357 })
5358 .to_string(),
5359 );
5360 let resp = feedback(
5361 State(state),
5362 Extension(TenantId("default".to_owned())),
5363 body,
5364 )
5365 .await;
5366 assert_eq!(resp.status(), axum::http::StatusCode::NOT_FOUND);
5367 let _ = std::fs::remove_file(&db);
5368 }
5369
5370 #[tokio::test]
5373 async fn feedback_across_tenants_is_404_not_403() {
5374 let (state, db, trace_id) = feedback_state().await;
5375 let body = Bytes::from(
5376 serde_json::json!({
5377 "trace_id": trace_id,
5378 "gate_id": "tests",
5379 "verdict": "pass",
5380 "score": 1.0,
5381 "reporter": "attacker",
5382 })
5383 .to_string(),
5384 );
5385 let resp = feedback(
5387 State(state),
5388 Extension(TenantId("tenant-b".to_owned())),
5389 body,
5390 )
5391 .await;
5392 assert_eq!(
5393 resp.status(),
5394 axum::http::StatusCode::NOT_FOUND,
5395 "cross-tenant feedback must look exactly like a missing trace"
5396 );
5397 let _ = std::fs::remove_file(&db);
5398 }
5399
5400 #[tokio::test]
5401 async fn feedback_rejects_bad_verdict_and_score() {
5402 let (state, db, trace_id) = feedback_state().await;
5403 let bad_verdict = Bytes::from(
5404 serde_json::json!({ "trace_id": trace_id, "gate_id": "g", "verdict": "maybe", "reporter": "x" })
5405 .to_string(),
5406 );
5407 assert_eq!(
5408 feedback(
5409 State(state.clone()),
5410 Extension(TenantId("default".to_owned())),
5411 bad_verdict
5412 )
5413 .await
5414 .status(),
5415 axum::http::StatusCode::BAD_REQUEST
5416 );
5417 let bad_score = Bytes::from(
5418 serde_json::json!({ "trace_id": trace_id, "gate_id": "g", "verdict": "pass", "score": 9.0, "reporter": "x" })
5419 .to_string(),
5420 );
5421 assert_eq!(
5422 feedback(
5423 State(state),
5424 Extension(TenantId("default".to_owned())),
5425 bad_score
5426 )
5427 .await
5428 .status(),
5429 axum::http::StatusCode::BAD_REQUEST
5430 );
5431 let _ = std::fs::remove_file(&db);
5432 }
5433
5434 #[tokio::test]
5435 async fn metrics_endpoint_renders_after_a_real_request() {
5436 use tower::ServiceExt;
5437
5438 let (state, mut rx) = enforce_state(
5439 &["anthropic/claude-haiku-4-5"],
5440 &["non-empty"],
5441 vec![(
5442 "anthropic/claude-haiku-4-5",
5443 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
5444 )],
5445 );
5446 let router = app(state).expect("prometheus recorder installs");
5447
5448 let req = axum::http::Request::builder()
5449 .method("POST")
5450 .uri("/v1/messages")
5451 .header("content-type", "application/json")
5452 .body(Body::from(user_body()))
5453 .unwrap();
5454 let resp = router.clone().oneshot(req).await.unwrap();
5455 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5456 rx.try_recv().expect("a trace was enqueued");
5457
5458 let metrics_req = axum::http::Request::builder()
5459 .method("GET")
5460 .uri("/metrics")
5461 .body(Body::empty())
5462 .unwrap();
5463 let metrics_resp = router.oneshot(metrics_req).await.unwrap();
5464 assert_eq!(metrics_resp.status(), axum::http::StatusCode::OK);
5465 let bytes = axum::body::to_bytes(metrics_resp.into_body(), 1 << 20)
5466 .await
5467 .unwrap();
5468 let body = String::from_utf8(bytes.to_vec()).unwrap();
5469 assert!(
5470 body.contains("firstpass_enforce_latency_ms"),
5471 "metrics body missing enforce latency histogram: {body}"
5472 );
5473 assert!(
5474 body.contains("firstpass_served_total"),
5475 "metrics body missing served counter: {body}"
5476 );
5477 }
5478
5479 #[tokio::test]
5485 async fn metrics_are_dimensioned_by_provider_rung_and_gate() {
5486 use tower::ServiceExt;
5487
5488 let (state, mut rx) = enforce_state(
5489 &["anthropic/claude-haiku-4-5"],
5490 &["non-empty"],
5491 vec![(
5492 "anthropic/claude-haiku-4-5",
5493 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
5494 )],
5495 );
5496 let router = app(state).expect("prometheus recorder installs");
5497
5498 let req = axum::http::Request::builder()
5499 .method("POST")
5500 .uri("/v1/messages")
5501 .header("content-type", "application/json")
5502 .body(Body::from(user_body()))
5503 .unwrap();
5504 assert_eq!(
5505 router.clone().oneshot(req).await.unwrap().status(),
5506 axum::http::StatusCode::OK
5507 );
5508 rx.try_recv().expect("a trace was enqueued");
5509
5510 let metrics_req = axum::http::Request::builder()
5511 .method("GET")
5512 .uri("/metrics")
5513 .body(Body::empty())
5514 .unwrap();
5515 let bytes = axum::body::to_bytes(
5516 router.oneshot(metrics_req).await.unwrap().into_body(),
5517 1 << 20,
5518 )
5519 .await
5520 .unwrap();
5521 let body = String::from_utf8(bytes.to_vec()).unwrap();
5522
5523 assert!(
5525 body.contains(r#"firstpass_attempt_total{provider="anthropic",rung="0"}"#),
5526 "attempt counter is not dimensioned by provider+rung: {body}"
5527 );
5528 assert!(
5529 body.contains("firstpass_attempt_latency_ms")
5530 && body.contains(r#"provider="anthropic""#),
5531 "attempt latency is not dimensioned by provider: {body}"
5532 );
5533 assert!(
5534 body.contains("firstpass_attempt_cost_usd_total"),
5535 "per-attempt cost series missing: {body}"
5536 );
5537 assert!(
5539 body.contains(r#"firstpass_gate_verdict_total{gate_id="non-empty",verdict="pass"}"#),
5540 "gate verdicts are not dimensioned by gate_id+verdict: {body}"
5541 );
5542 }
5543
5544 fn auth_state(require_auth: bool, keys_json: Option<String>) -> AppState {
5548 auth_state_rated(require_auth, keys_json, None)
5549 }
5550
5551 fn auth_state_rated(
5554 require_auth: bool,
5555 keys_json: Option<String>,
5556 rate_per_sec: Option<u32>,
5557 ) -> AppState {
5558 let config = ProxyConfig::from_lookup(|k| match k {
5559 "FIRSTPASS_REQUIRE_AUTH" => require_auth.then(|| "true".to_owned()),
5560 "FIRSTPASS_TENANT_KEYS_JSON" => keys_json.clone(),
5561 "FIRSTPASS_TENANT_RATE_PER_SEC" => rate_per_sec.map(|n| n.to_string()),
5562 _ => None,
5563 })
5564 .unwrap();
5565 let (traces, _rx) = mpsc::channel(64);
5566 std::mem::forget(_rx);
5569 let providers: HashMap<String, Arc<dyn Provider>> = HashMap::new();
5570 let tenant_rate_limiter = build_tenant_rate_limiter(&config);
5571 AppState {
5572 config: Arc::new(config),
5573 http: reqwest::Client::new(),
5574 providers: ProviderRegistry::from_map(providers),
5575 gate_health: Arc::new(GateHealthRegistry::new()),
5576 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
5577 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
5578 traces,
5579 adaptive: None,
5580 eprocess: None,
5581 bandit: None,
5582 promoter: None,
5583 verified_cache: None,
5584 predictor: None,
5585 tenant_rate_limiter,
5586 spill: None,
5587 }
5588 }
5589
5590 fn cap_request(auth_header: Option<&str>) -> axum::http::Request<Body> {
5591 let mut b = axum::http::Request::builder()
5592 .method("GET")
5593 .uri("/v1/capabilities");
5594 if let Some(h) = auth_header {
5595 b = b.header("authorization", h);
5596 }
5597 b.body(Body::empty()).unwrap()
5598 }
5599
5600 #[tokio::test]
5601 async fn auth_off_allows_unauthenticated_request() {
5602 use tower::ServiceExt;
5603 let router = app(auth_state(false, None)).expect("router");
5604 let resp = router.oneshot(cap_request(None)).await.unwrap();
5605 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5607 }
5608
5609 #[tokio::test]
5610 async fn auth_on_missing_key_is_401_opaque() {
5611 use tower::ServiceExt;
5612 let hash = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
5613 let keys = format!("{{\"tenant-a\": {hash:?}}}");
5614 let router = app(auth_state(true, Some(keys))).expect("router");
5615
5616 let resp = router.oneshot(cap_request(None)).await.unwrap();
5617 assert_eq!(resp.status(), axum::http::StatusCode::UNAUTHORIZED);
5618 let json = body_json(resp).await;
5619 assert_eq!(json["error"]["type"], "unauthorized");
5620 let msg = json["error"]["message"].as_str().unwrap();
5622 assert!(!msg.contains("tenant"), "no tenant oracle in body: {msg}");
5623 }
5624
5625 #[tokio::test]
5626 async fn auth_on_invalid_key_is_401() {
5627 use tower::ServiceExt;
5628 let hash = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
5629 let keys = format!("{{\"tenant-a\": {hash:?}}}");
5630 let router = app(auth_state(true, Some(keys))).expect("router");
5631
5632 let resp = router
5633 .oneshot(cap_request(Some("Bearer wrong-key")))
5634 .await
5635 .unwrap();
5636 assert_eq!(resp.status(), axum::http::StatusCode::UNAUTHORIZED);
5637 }
5638
5639 #[tokio::test]
5640 async fn auth_on_valid_key_proceeds() {
5641 use tower::ServiceExt;
5642 let hash = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
5643 let keys = format!("{{\"tenant-a\": {hash:?}}}");
5644 let router = app(auth_state(true, Some(keys))).expect("router");
5645
5646 let resp = router
5648 .oneshot(cap_request(Some("Bearer tenant-a.key-a")))
5649 .await
5650 .unwrap();
5651 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5653 }
5654
5655 fn two_tenant_state(rate_per_sec: Option<u32>) -> AppState {
5657 let hash_a = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
5658 let hash_b = crate::tenant_auth::TenantKeys::hash_key("key-b").unwrap();
5659 let keys = format!("{{\"tenant-a\": {hash_a:?}, \"tenant-b\": {hash_b:?}}}");
5660 auth_state_rated(true, Some(keys), rate_per_sec)
5661 }
5662
5663 #[tokio::test]
5664 async fn tenant_exceeding_rate_limit_gets_429_opaque() {
5665 use tower::ServiceExt;
5666 let router = app(two_tenant_state(Some(1))).expect("router");
5672
5673 let (r1, r2, r3, r4) = tokio::join!(
5674 router
5675 .clone()
5676 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
5677 router
5678 .clone()
5679 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
5680 router
5681 .clone()
5682 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
5683 router
5684 .clone()
5685 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
5686 );
5687 let responses = [r1.unwrap(), r2.unwrap(), r3.unwrap(), r4.unwrap()];
5688 let ok = responses
5689 .iter()
5690 .filter(|r| r.status() == axum::http::StatusCode::OK)
5691 .count();
5692 assert!(ok >= 1, "the burst's first request must pass");
5693 let limited: Vec<_> = responses
5694 .into_iter()
5695 .filter(|r| r.status() == axum::http::StatusCode::TOO_MANY_REQUESTS)
5696 .collect();
5697 assert!(
5698 !limited.is_empty(),
5699 "a 4-request burst against 1 req/sec must trip the limiter"
5700 );
5701
5702 let json = body_json(limited.into_iter().next().unwrap()).await;
5703 assert_eq!(json["error"]["type"], "rate_limited");
5704 let msg = json["error"]["message"].as_str().unwrap();
5706 assert!(!msg.contains('1'), "no limit value in body: {msg}");
5707 }
5708
5709 #[tokio::test]
5710 async fn rate_limit_buckets_are_independent_per_tenant() {
5711 use tower::ServiceExt;
5712 let router = app(two_tenant_state(Some(1))).expect("router");
5713
5714 let (a1, a2, a3) = tokio::join!(
5717 router
5718 .clone()
5719 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
5720 router
5721 .clone()
5722 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
5723 router
5724 .clone()
5725 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
5726 );
5727 let a_limited = [a1.unwrap(), a2.unwrap(), a3.unwrap()]
5728 .iter()
5729 .filter(|r| r.status() == axum::http::StatusCode::TOO_MANY_REQUESTS)
5730 .count();
5731 assert!(a_limited >= 1, "tenant A's burst must trip its limiter");
5732
5733 let b1 = router
5735 .clone()
5736 .oneshot(cap_request(Some("Bearer tenant-b.key-b")))
5737 .await
5738 .unwrap();
5739 assert_eq!(b1.status(), axum::http::StatusCode::OK);
5740 }
5741
5742 #[tokio::test]
5743 async fn rate_limit_unset_never_429s() {
5744 use tower::ServiceExt;
5745 let router = app(two_tenant_state(None)).expect("router");
5748 for _ in 0..20 {
5749 let resp = router
5750 .clone()
5751 .oneshot(cap_request(Some("Bearer tenant-a.key-a")))
5752 .await
5753 .unwrap();
5754 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5755 }
5756 }
5757
5758 #[test]
5761 fn u01_is_deterministic_and_in_range() {
5762 let s1 = u01(0xDEAD_BEEF_CAFE_1234_u128);
5763 let s2 = u01(0xDEAD_BEEF_CAFE_1234_u128);
5764 assert_eq!(s1, s2, "u01 must be deterministic for the same seed");
5765 assert!((0.0..1.0).contains(&s1), "u01 must return [0, 1), got {s1}");
5766
5767 let s3 = u01(0x1234_5678_9ABC_DEF0_u128);
5769 assert_ne!(s1, s3, "different seeds should give different values");
5770
5771 for i in 0u64..256 {
5773 let v = u01(i as u128);
5774 assert!((0.0..1.0).contains(&v), "seed {i}: u01={v} out of [0,1)");
5775 }
5776 }
5777
5778 #[test]
5779 fn epsilon_propensity_formula() {
5780 let epsilon = 0.2_f64;
5781 let k = 3_usize;
5782 let greedy = 1_u32;
5783
5784 let p_greedy = epsilon_propensity(greedy, greedy, epsilon, k);
5786 let expected_greedy = (1.0 - epsilon) + epsilon / k as f64;
5787 assert!(
5788 (p_greedy - expected_greedy).abs() < 1e-12,
5789 "{p_greedy} != {expected_greedy}"
5790 );
5791
5792 let p_other = epsilon_propensity(0, greedy, epsilon, k);
5794 let expected_other = epsilon / k as f64;
5795 assert!(
5796 (p_other - expected_other).abs() < 1e-12,
5797 "{p_other} != {expected_other}"
5798 );
5799
5800 for chosen in 0..k as u32 {
5802 let p = epsilon_propensity(chosen, greedy, epsilon, k);
5803 assert!(
5804 p > 0.0 && p <= 1.0,
5805 "propensity {p} out of (0,1] for chosen={chosen}"
5806 );
5807 }
5808 }
5809
5810 #[test]
5811 fn epsilon_branch_and_greedy_branch_both_occur_over_many_seeds() {
5812 let epsilon = 0.3_f64;
5814 let mut saw_explore = false;
5815 let mut saw_greedy = false;
5816 for i in 0u64..200 {
5817 let u = u01(i as u128);
5818 if u < epsilon {
5819 saw_explore = true;
5820 } else {
5821 saw_greedy = true;
5822 }
5823 if saw_explore && saw_greedy {
5824 break;
5825 }
5826 }
5827 assert!(
5828 saw_explore,
5829 "epsilon branch must fire with epsilon=0.3 over 200 seeds"
5830 );
5831 assert!(
5832 saw_greedy,
5833 "greedy branch must occur with epsilon=0.3 over 200 seeds"
5834 );
5835 }
5836
5837 #[test]
5838 fn epsilon_propensity_sums_to_one_over_all_rungs() {
5839 let epsilon = 0.15_f64;
5841 let k = 4_usize;
5842 let greedy = 2_u32;
5843 let total: f64 = (0..k as u32)
5844 .map(|r| epsilon_propensity(r, greedy, epsilon, k))
5845 .sum();
5846 assert!(
5847 (total - 1.0).abs() < 1e-12,
5848 "propensities must sum to 1, got {total}"
5849 );
5850 }
5851
5852 #[tokio::test(start_paused = true)]
5855 async fn keepalive_stream_ticks_then_emits_final_frame() {
5856 let (tx, rx) = tokio::sync::oneshot::channel::<Result<Value, ProxyError>>();
5857 let mut ticks = tokio::time::interval(SSE_KEEPALIVE_EVERY);
5858 ticks.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
5859 ticks.reset();
5860 let mut stream = KeepaliveStream {
5861 rx: Some(rx),
5862 ticks,
5863 format_message: anthropic_sse_from_message,
5864 };
5865 async fn next(
5867 stream: &mut KeepaliveStream,
5868 ) -> Option<Option<Result<Bytes, std::convert::Infallible>>> {
5869 std::future::poll_fn(|cx| {
5870 std::task::Poll::Ready(
5871 match futures_core::Stream::poll_next(std::pin::Pin::new(&mut *stream), cx) {
5872 std::task::Poll::Ready(item) => Some(item),
5873 std::task::Poll::Pending => None,
5874 },
5875 )
5876 })
5877 .await
5878 }
5879
5880 assert!(
5882 next(&mut stream).await.is_none(),
5883 "no frame before an interval"
5884 );
5885
5886 tokio::time::advance(SSE_KEEPALIVE_EVERY + Duration::from_millis(1)).await;
5888 let frame = next(&mut stream)
5889 .await
5890 .expect("keepalive due")
5891 .unwrap()
5892 .unwrap();
5893 assert!(
5894 frame.starts_with(b": "),
5895 "keepalive must be an SSE comment (ignored by every conforming parser)"
5896 );
5897
5898 let message = serde_json::json!({
5900 "id": "msg_1", "type": "message", "role": "assistant", "model": "m",
5901 "content": [{ "type": "text", "text": "done" }],
5902 "usage": { "input_tokens": 1, "output_tokens": 1 }
5903 });
5904 tx.send(Ok(message)).unwrap();
5905 let frame = next(&mut stream)
5906 .await
5907 .expect("final frame")
5908 .unwrap()
5909 .unwrap();
5910 let text = String::from_utf8(frame.to_vec()).unwrap();
5911 assert!(text.contains("event: message_start"));
5912 assert!(text.contains("event: message_stop"));
5913 let eos = next(&mut stream)
5914 .await
5915 .expect("stream must end after the final frame");
5916 assert!(eos.is_none(), "end-of-stream after the final frame");
5917 }
5918
5919 #[tokio::test]
5922 async fn streaming_enforce_serves_full_sse_sequence() {
5923 let toml = "[[price]]\nmodel = \"anthropic/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/m\"]\ngates = [\"non-empty\"]\n";
5924 let config = ProxyConfig::from_lookup(|k| match k {
5925 "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
5926 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
5927 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
5928 _ => None,
5929 })
5930 .unwrap();
5931 let mut outs = HashMap::new();
5932 outs.insert(
5933 "anthropic/m".to_owned(),
5934 Ok(model_resp("anthropic/m", "gated answer")),
5935 );
5936 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
5937 map.insert(
5938 "anthropic".to_owned(),
5939 Arc::new(MockProvider::new("anthropic", outs)),
5940 );
5941 let (traces, _rx) = mpsc::channel(64);
5942 let state = AppState {
5943 config: Arc::new(config),
5944 http: reqwest::Client::new(),
5945 providers: ProviderRegistry::from_map(map),
5946 gate_health: Arc::new(GateHealthRegistry::new()),
5947 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
5948 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
5949 traces,
5950 adaptive: None,
5951 eprocess: None,
5952 bandit: None,
5953 promoter: None,
5954 verified_cache: None,
5955 predictor: None,
5956 tenant_rate_limiter: None,
5957 spill: None,
5958 };
5959 let body = Bytes::from_static(
5960 br#"{"model":"m","stream":true,"messages":[{"role":"user","content":"hi"}]}"#,
5961 );
5962 let resp = messages(
5963 State(state),
5964 Extension(TenantId("default".to_owned())),
5965 HeaderMap::new(),
5966 body,
5967 )
5968 .await;
5969 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5970 assert!(
5971 resp.headers()
5972 .get(axum::http::header::CONTENT_TYPE)
5973 .and_then(|v| v.to_str().ok())
5974 .is_some_and(|ct| ct.starts_with("text/event-stream")),
5975 "streaming client must get SSE"
5976 );
5977 let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
5978 .await
5979 .unwrap();
5980 let text = String::from_utf8(bytes.to_vec()).unwrap();
5981 assert!(text.contains("event: message_start"));
5982 assert!(text.contains("gated answer"));
5983 assert!(text.contains("event: message_stop"));
5984 }
5985
5986 #[test]
5991 fn parse_openai_request_plain_text() {
5992 let body = br#"{"model":"gpt-4o","max_tokens":256,"messages":[{"role":"user","content":"hello"}]}"#;
5994 let req = parse_openai_request(body, false).expect("must parse");
5995 assert_eq!(req.model, "gpt-4o");
5996 assert_eq!(req.max_tokens, 256);
5997 assert_eq!(req.messages.len(), 1);
5998 assert_eq!(req.messages[0].role, "user");
5999 assert_eq!(req.messages[0].content, Value::String("hello".to_owned()));
6000 assert!(req.system.is_none());
6001 assert_eq!(req.raw, Value::Null);
6003 }
6004
6005 #[test]
6006 fn parse_openai_request_system_message() {
6007 let body = br#"{"model":"gpt-4o","messages":[{"role":"system","content":"be concise"},{"role":"user","content":"hi"}]}"#;
6008 let req = parse_openai_request(body, false).expect("must parse");
6009 assert_eq!(req.system.as_deref(), Some("be concise"));
6010 assert_eq!(req.messages.len(), 1);
6011 assert_eq!(req.messages[0].role, "user");
6012 }
6013
6014 #[test]
6015 fn parse_openai_request_tool_calls_translate_to_tool_use() {
6016 let body = br#"{
6017 "model":"gpt-4o",
6018 "messages":[
6019 {"role":"user","content":"what's the weather?"},
6020 {"role":"assistant","content":null,"tool_calls":[
6021 {"id":"call_1","type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Paris\"}"}}
6022 ]},
6023 {"role":"tool","tool_call_id":"call_1","content":"15C, cloudy"}
6024 ]
6025 }"#;
6026 let req = parse_openai_request(body, false).expect("must parse");
6027 assert_eq!(req.messages.len(), 3);
6029 let asst = &req.messages[1];
6031 assert_eq!(asst.role, "assistant");
6032 let blocks = asst.content.as_array().expect("content array");
6033 assert_eq!(blocks[0]["type"], "tool_use");
6034 assert_eq!(blocks[0]["name"], "get_weather");
6035 assert_eq!(blocks[0]["id"], "call_1");
6036 assert_eq!(blocks[0]["input"]["city"], "Paris");
6037 let tool_msg = &req.messages[2];
6039 assert_eq!(tool_msg.role, "user");
6040 let result_blocks = tool_msg.content.as_array().expect("result blocks");
6041 assert_eq!(result_blocks[0]["type"], "tool_result");
6042 assert_eq!(result_blocks[0]["tool_use_id"], "call_1");
6043 }
6044
6045 #[test]
6046 fn parse_openai_request_tools_translate_to_anthropic_format() {
6047 let body = br#"{
6048 "model":"gpt-4o",
6049 "messages":[{"role":"user","content":"use a tool"}],
6050 "tools":[{"type":"function","function":{"name":"search","description":"web search","parameters":{"type":"object","properties":{"q":{"type":"string"}}}}}]
6051 }"#;
6052 let req = parse_openai_request(body, false).expect("must parse");
6053 let tools = req.tools.as_array().expect("tools array");
6054 assert_eq!(tools.len(), 1);
6055 assert_eq!(tools[0]["name"], "search");
6056 assert_eq!(tools[0]["description"], "web search");
6057 assert_eq!(tools[0]["input_schema"]["type"], "object");
6058 }
6059
6060 #[test]
6061 fn parse_openai_request_raw_carry_preserves_full_body() {
6062 let body =
6064 br#"{"model":"gpt-4o","max_tokens":100,"messages":[{"role":"user","content":"hi"}]}"#;
6065 let req = parse_openai_request(body, true).expect("must parse");
6066 assert!(req.raw.is_object(), "raw must be the full JSON object");
6067 assert_eq!(req.raw["model"], "gpt-4o");
6068 assert_eq!(req.raw["max_tokens"], 100);
6069 assert!(req.tools.is_null(), "no tools in this request");
6071 }
6072
6073 #[test]
6074 fn parse_openai_request_http_image_returns_none() {
6075 let body = br#"{"model":"gpt-4o","messages":[{"role":"user","content":[
6077 {"type":"text","text":"describe this"},
6078 {"type":"image_url","image_url":{"url":"https://example.com/cat.png"}}
6079 ]}]}"#;
6080 let result = parse_openai_request(body, false);
6081 assert!(result.is_none(), "http image URL must fail translation");
6082 }
6083
6084 #[test]
6085 fn parse_openai_request_data_url_image_translates_to_anthropic_base64() {
6086 let body = br#"{"model":"gpt-4o","messages":[{"role":"user","content":[
6087 {"type":"text","text":"describe"},
6088 {"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgo="}}
6089 ]}]}"#;
6090 let req = parse_openai_request(body, false).expect("data URL must parse");
6091 let blocks = req.messages[0].content.as_array().expect("blocks");
6092 let img = blocks
6093 .iter()
6094 .find(|b| b["type"] == "image")
6095 .expect("image block");
6096 assert_eq!(img["source"]["type"], "base64");
6097 assert_eq!(img["source"]["media_type"], "image/png");
6098 assert_eq!(img["source"]["data"], "iVBORw0KGgo=");
6099 }
6100
6101 #[test]
6104 fn openai_response_json_renders_text_response() {
6105 let resp = ModelResponse {
6106 model: "gpt-4o".to_owned(),
6107 text: "Hello!".to_owned(),
6108 in_tokens: 10,
6109 cache_write_tokens: 0,
6110 cache_read_tokens: 0,
6111 out_tokens: 5,
6112 raw: serde_json::json!({
6113 "content": [{ "type": "text", "text": "Hello!" }]
6114 }),
6115 };
6116 let json = openai_response_json(&resp);
6117 assert_eq!(json["object"], "chat.completion");
6118 assert_eq!(json["model"], "gpt-4o");
6119 assert_eq!(json["choices"][0]["message"]["role"], "assistant");
6120 assert_eq!(json["choices"][0]["message"]["content"], "Hello!");
6121 assert_eq!(json["choices"][0]["finish_reason"], "stop");
6122 assert_eq!(json["usage"]["prompt_tokens"], 10);
6123 assert_eq!(json["usage"]["completion_tokens"], 5);
6124 }
6125
6126 #[test]
6127 fn openai_response_json_renders_tool_call() {
6128 let resp = ModelResponse {
6129 model: "gpt-4o".to_owned(),
6130 text: String::new(),
6131 in_tokens: 20,
6132 cache_write_tokens: 0,
6133 cache_read_tokens: 0,
6134 out_tokens: 15,
6135 raw: serde_json::json!({
6136 "content": [{
6137 "type": "tool_use",
6138 "id": "call_abc",
6139 "name": "search",
6140 "input": {"q": "Rust async"}
6141 }]
6142 }),
6143 };
6144 let json = openai_response_json(&resp);
6145 assert_eq!(json["choices"][0]["finish_reason"], "tool_calls");
6146 let tc = &json["choices"][0]["message"]["tool_calls"][0];
6147 assert_eq!(tc["id"], "call_abc");
6148 assert_eq!(tc["type"], "function");
6149 assert_eq!(tc["function"]["name"], "search");
6150 assert_eq!(json["choices"][0]["message"]["content"], Value::Null);
6152 }
6153
6154 #[test]
6155 fn openai_sse_from_message_plain_text_has_role_then_content_then_stop() {
6156 let resp = ModelResponse {
6157 model: "gpt-4o".to_owned(),
6158 text: "Hi there!".to_owned(),
6159 in_tokens: 5,
6160 cache_write_tokens: 0,
6161 cache_read_tokens: 0,
6162 out_tokens: 3,
6163 raw: serde_json::json!({
6164 "content": [{ "type": "text", "text": "Hi there!" }]
6165 }),
6166 };
6167 let sse = openai_sse_from_message(&openai_response_json(&resp));
6168 for line in sse.lines() {
6170 assert!(
6171 line.is_empty() || line.starts_with("data: "),
6172 "bad SSE line: {line:?}"
6173 );
6174 }
6175 let frames: Vec<&str> = sse
6176 .lines()
6177 .filter_map(|l| l.strip_prefix("data: "))
6178 .collect();
6179 assert_eq!(*frames.last().unwrap(), "[DONE]");
6181 let role_frame: Value = serde_json::from_str(frames[0]).unwrap();
6183 assert_eq!(role_frame["choices"][0]["delta"]["role"], "assistant");
6184 assert!(frames.iter().any(|f| {
6186 if *f == "[DONE]" {
6187 return false;
6188 }
6189 serde_json::from_str::<Value>(f)
6190 .ok()
6191 .is_some_and(|v| v["choices"][0]["delta"]["content"] == "Hi there!")
6192 }));
6193 assert!(frames.iter().any(|f| {
6195 if *f == "[DONE]" {
6196 return false;
6197 }
6198 serde_json::from_str::<Value>(f)
6199 .ok()
6200 .is_some_and(|v| v["choices"][0]["finish_reason"] == "stop")
6201 }));
6202 }
6203
6204 #[test]
6207 fn detects_openai_tool_calls() {
6208 let with_tool_calls = Bytes::from_static(br#"{"messages":[
6209 {"role":"user","content":"hi"},
6210 {"role":"assistant","content":null,"tool_calls":[{"id":"c1","type":"function","function":{"name":"f","arguments":"{}"}}]}
6211 ]}"#);
6212 let without = Bytes::from_static(br#"{"messages":[{"role":"user","content":"hi"}]}"#);
6213 let with_tool_msg = Bytes::from_static(
6214 br#"{"messages":[
6215 {"role":"tool","tool_call_id":"c1","content":"result"}
6216 ]}"#,
6217 );
6218 assert!(openai_messages_have_tool_calls(&with_tool_calls));
6219 assert!(!openai_messages_have_tool_calls(&without));
6220 assert!(openai_messages_have_tool_calls(&with_tool_msg));
6221 }
6222
6223 #[test]
6224 fn detects_openai_http_images() {
6225 let http_img = Bytes::from_static(
6226 br#"{"messages":[{"role":"user","content":[
6227 {"type":"image_url","image_url":{"url":"https://example.com/img.png"}}
6228 ]}]}"#,
6229 );
6230 let data_img = Bytes::from_static(
6231 br#"{"messages":[{"role":"user","content":[
6232 {"type":"image_url","image_url":{"url":"data:image/png;base64,abc"}}
6233 ]}]}"#,
6234 );
6235 let no_img = Bytes::from_static(br#"{"messages":[{"role":"user","content":"hi"}]}"#);
6236 assert!(openai_has_http_images(&http_img));
6237 assert!(!openai_has_http_images(&data_img));
6238 assert!(!openai_has_http_images(&no_img));
6239 }
6240
6241 #[test]
6242 fn enforce_can_handle_openai_inbound_all_openai_ladder() {
6243 let tools_body = Bytes::from_static(br#"{"model":"gpt-4o","messages":[{"role":"assistant","content":null,"tool_calls":[{"id":"c","type":"function","function":{"name":"f","arguments":"{}"}}]}]}"#);
6245 let f = extract_openai_features(&HeaderMap::new(), &tools_body);
6246 let ladder = vec!["openai/gpt-4o-mini".to_owned(), "openai/gpt-4o".to_owned()];
6247 let providers = crate::provider::ProviderRegistry::new("http://x", "http://x");
6248 assert!(enforce_can_handle(
6249 &f,
6250 &tools_body,
6251 true,
6252 &ladder,
6253 &providers,
6254 Dialect::Openai
6255 ));
6256 }
6257
6258 #[test]
6259 fn enforce_can_handle_openai_inbound_all_anthropic_ladder_no_http_image() {
6260 let tools_body = Bytes::from_static(br#"{"model":"gpt-4o","messages":[{"role":"assistant","content":null,"tool_calls":[{"id":"c","type":"function","function":{"name":"f","arguments":"{}"}}]}]}"#);
6262 let f = extract_openai_features(&HeaderMap::new(), &tools_body);
6263 let ladder = vec!["anthropic/claude-haiku-4-5".to_owned()];
6264 let providers = crate::provider::ProviderRegistry::new("http://x", "http://x");
6265 assert!(enforce_can_handle(
6266 &f,
6267 &tools_body,
6268 true,
6269 &ladder,
6270 &providers,
6271 Dialect::Openai,
6272 ));
6273 }
6274
6275 #[test]
6276 fn enforce_can_handle_openai_inbound_http_image_falls_back() {
6277 let img_body = Bytes::from_static(
6279 br#"{"model":"gpt-4o","messages":[{"role":"user","content":[
6280 {"type":"image_url","image_url":{"url":"https://example.com/img.png"}}
6281 ]}]}"#,
6282 );
6283 let f = extract_openai_features(&HeaderMap::new(), &img_body);
6284 let ladder = vec!["anthropic/claude-haiku-4-5".to_owned()];
6285 let providers = crate::provider::ProviderRegistry::new("http://x", "http://x");
6286 assert!(!enforce_can_handle(
6287 &f,
6288 &img_body,
6289 true,
6290 &ladder,
6291 &providers,
6292 Dialect::Openai,
6293 ));
6294 }
6295
6296 fn openai_enforce_state(mock_resp: ModelResponse) -> AppState {
6300 let toml = "[[price]]\nmodel = \"mock/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"mock/m\"]\ngates = [\"non-empty\"]\n";
6301 let config = ProxyConfig::from_lookup(|k| match k {
6302 "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
6303 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
6304 _ => None,
6305 })
6306 .unwrap();
6307 let mut outs = HashMap::new();
6308 outs.insert("mock/m".to_owned(), Ok(mock_resp));
6309 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
6310 map.insert("mock".to_owned(), Arc::new(MockProvider::new("mock", outs)));
6311 let (traces, _rx) = mpsc::channel(64);
6312 std::mem::forget(_rx);
6313 let tenant_rate_limiter = build_tenant_rate_limiter(&config);
6314 AppState {
6315 config: Arc::new(config),
6316 http: reqwest::Client::new(),
6317 providers: ProviderRegistry::from_map(map),
6318 gate_health: Arc::new(GateHealthRegistry::new()),
6319 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
6320 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
6321 traces,
6322 adaptive: None,
6323 eprocess: None,
6324 bandit: None,
6325 promoter: None,
6326 verified_cache: None,
6327 predictor: None,
6328 tenant_rate_limiter,
6329 spill: None,
6330 }
6331 }
6332
6333 #[tokio::test]
6334 async fn chat_completions_plain_text_enforce_returns_openai_shape() {
6335 let mock = model_resp("mock/m", "gated answer");
6336 let state = openai_enforce_state(mock);
6337 let body = Bytes::from_static(
6338 br#"{"model":"gpt-4o","messages":[{"role":"user","content":"hello"}]}"#,
6339 );
6340 let resp = chat_completions(
6341 State(state),
6342 Extension(TenantId("default".to_owned())),
6343 HeaderMap::new(),
6344 body,
6345 )
6346 .await;
6347 assert_eq!(resp.status(), axum::http::StatusCode::OK);
6348 let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
6349 .await
6350 .unwrap();
6351 let json: Value = serde_json::from_slice(&bytes).expect("must be JSON");
6352 assert_eq!(json["object"], "chat.completion");
6353 assert_eq!(json["choices"][0]["message"]["role"], "assistant");
6354 assert_eq!(json["choices"][0]["message"]["content"], "gated answer");
6355 assert_eq!(json["choices"][0]["finish_reason"], "stop");
6356 assert!(
6357 json["id"]
6358 .as_str()
6359 .is_some_and(|id| id.starts_with("chatcmpl-"))
6360 );
6361 }
6362
6363 #[tokio::test]
6364 async fn chat_completions_stream_true_returns_sse_with_openai_chunks() {
6365 let mock = model_resp("mock/m", "gated answer");
6366 let state = openai_enforce_state(mock);
6367 let body = Bytes::from_static(
6368 br#"{"model":"gpt-4o","stream":true,"messages":[{"role":"user","content":"hello"}]}"#,
6369 );
6370 let resp = chat_completions(
6371 State(state),
6372 Extension(TenantId("default".to_owned())),
6373 HeaderMap::new(),
6374 body,
6375 )
6376 .await;
6377 assert_eq!(resp.status(), axum::http::StatusCode::OK);
6378 assert!(
6379 resp.headers()
6380 .get(axum::http::header::CONTENT_TYPE)
6381 .and_then(|v| v.to_str().ok())
6382 .is_some_and(|ct| ct.starts_with("text/event-stream")),
6383 "stream:true must return SSE"
6384 );
6385 let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
6386 .await
6387 .unwrap();
6388 let text = String::from_utf8(bytes.to_vec()).unwrap();
6389 assert!(
6391 text.contains("chat.completion.chunk"),
6392 "must have OpenAI chunk frames"
6393 );
6394 assert!(text.contains("[DONE]"), "must end with [DONE]");
6395 assert!(
6396 text.contains("gated answer"),
6397 "content must be in the stream"
6398 );
6399 assert!(
6401 !text.contains("message_start"),
6402 "must not have Anthropic event types"
6403 );
6404 }
6405
6406 fn probe_state(
6410 sample_rate: f64,
6411 k: u32,
6412 outcomes: Vec<(&str, Result<ModelResponse, ProviderError>)>,
6413 ) -> (AppState, mpsc::Receiver<Trace>) {
6414 let toml = format!(
6415 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\nladder = [\"anthropic/claude-haiku-4-5\"]\ngates = [\"non-empty\"]\n\
6416 [escalation.probe]\nk = {k}\nsample_rate = {sample_rate}\n"
6417 );
6418 let config = ProxyConfig::from_lookup(|k_| match k_ {
6419 "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
6420 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
6421 _ => None,
6422 })
6423 .unwrap();
6424 let mut outs = HashMap::new();
6425 for (model, out) in outcomes {
6426 outs.insert(model.to_owned(), out);
6427 }
6428 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
6429 map.insert(
6430 "anthropic".to_owned(),
6431 Arc::new(MockProvider::new("anthropic", outs)),
6432 );
6433 let (traces, rx) = mpsc::channel(64);
6434 let state = AppState {
6435 config: Arc::new(config),
6436 http: reqwest::Client::new(),
6437 providers: ProviderRegistry::from_map(map),
6438 gate_health: Arc::new(GateHealthRegistry::new()),
6439 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
6440 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
6441 traces,
6442 adaptive: None,
6443 eprocess: None,
6444 bandit: None,
6445 promoter: None,
6446 verified_cache: None,
6447 predictor: None,
6448 tenant_rate_limiter: None,
6449 spill: None,
6450 };
6451 (state, rx)
6452 }
6453
6454 async fn run_enforce_get_trace(state: AppState, mut rx: mpsc::Receiver<Trace>) -> Trace {
6456 let resp = messages(
6457 State(state),
6458 Extension(TenantId("default".to_owned())),
6459 HeaderMap::new(),
6460 user_body(),
6461 )
6462 .await;
6463 assert_eq!(resp.status(), axum::http::StatusCode::OK);
6464 rx.try_recv().expect("trace must be enqueued")
6465 }
6466
6467 #[tokio::test]
6469 async fn probe_off_trace_has_no_probe_field() {
6470 let (state, rx) = enforce_state(
6471 &["anthropic/claude-haiku-4-5"],
6472 &["non-empty"],
6473 vec![(
6474 "anthropic/claude-haiku-4-5",
6475 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
6476 )],
6477 );
6478 assert!(
6479 state
6480 .config
6481 .routing
6482 .as_ref()
6483 .unwrap()
6484 .escalation
6485 .probe
6486 .is_none(),
6487 "probe must default to None"
6488 );
6489 let trace = run_enforce_get_trace(state, rx).await;
6490 assert!(
6491 trace.probe.is_none(),
6492 "probe=None config must not set trace.probe"
6493 );
6494 }
6495
6496 #[tokio::test]
6498 async fn probe_sample_rate_zero_never_fires() {
6499 let (state, rx) = probe_state(
6501 0.0,
6502 5,
6503 vec![(
6504 "anthropic/claude-haiku-4-5",
6505 Ok(model_resp("anthropic/claude-haiku-4-5", "hi")),
6506 )],
6507 );
6508 let trace = run_enforce_get_trace(state, rx).await;
6509 assert!(
6510 trace.probe.is_none(),
6511 "sample_rate=0.0 must never set trace.probe"
6512 );
6513 }
6514
6515 #[tokio::test]
6518 async fn probe_on_all_pass_sets_confident_pass() {
6519 let model = "anthropic/claude-haiku-4-5";
6521 let (state, mut rx) = probe_state(1.0, 3, vec![(model, Ok(model_resp(model, "hello")))]);
6522 let resp = messages(
6523 State(state),
6524 Extension(TenantId("default".to_owned())),
6525 HeaderMap::new(),
6526 user_body(),
6527 )
6528 .await;
6529 assert_eq!(resp.status(), axum::http::StatusCode::OK);
6530 let json = body_json(resp).await;
6532 assert_eq!(
6533 json["content"][0]["text"], "hello",
6534 "served content unchanged"
6535 );
6536
6537 let trace = rx.try_recv().expect("trace enqueued");
6538 let sig = trace.probe.expect("probe must be set when sample_rate=1.0");
6539 assert_eq!(sig.k, 3);
6540 assert_eq!(sig.gate_pass_count, 3, "all 3 samples must pass non-empty");
6541 assert_eq!(
6542 sig.regime,
6543 firstpass_core::ProbeRegime::ConfidentPass,
6544 "all-pass → ConfidentPass"
6545 );
6546 assert!(
6547 sig.probe_cost_usd > 0.0,
6548 "k model calls must cost something"
6549 );
6550 }
6551
6552 #[tokio::test]
6555 async fn probe_on_all_fail_sets_confident_fail() {
6556 let model = "anthropic/claude-haiku-4-5";
6557 let (state, mut rx) = probe_state(1.0, 3, vec![(model, Ok(model_resp(model, "")))]);
6559 let resp = messages(
6561 State(state),
6562 Extension(TenantId("default".to_owned())),
6563 HeaderMap::new(),
6564 user_body(),
6565 )
6566 .await;
6567 assert_eq!(resp.status(), axum::http::StatusCode::OK);
6568
6569 let trace = rx.try_recv().expect("trace enqueued");
6570 let sig = trace.probe.expect("probe must be set when sample_rate=1.0");
6571 assert_eq!(
6572 sig.gate_pass_count, 0,
6573 "empty response fails non-empty: all 0 pass"
6574 );
6575 assert_eq!(
6576 sig.regime,
6577 firstpass_core::ProbeRegime::ConfidentFail,
6578 "0 passes → ConfidentFail"
6579 );
6580 }
6581
6582 #[tokio::test]
6585 async fn probe_on_served_output_identical_to_probe_off() {
6586 let model = "anthropic/claude-haiku-4-5";
6587 let mk = |sample_rate: f64| {
6588 probe_state(
6589 sample_rate,
6590 2,
6591 vec![(model, Ok(model_resp(model, "gated answer")))],
6592 )
6593 };
6594
6595 let (state_off, mut rx_off) = mk(0.0);
6597 let resp_off = messages(
6598 State(state_off),
6599 Extension(TenantId("default".to_owned())),
6600 HeaderMap::new(),
6601 user_body(),
6602 )
6603 .await;
6604 let json_off = body_json(resp_off).await;
6605 let trace_off = rx_off.try_recv().unwrap();
6606
6607 let (state_on, mut rx_on) = mk(1.0);
6609 let resp_on = messages(
6610 State(state_on),
6611 Extension(TenantId("default".to_owned())),
6612 HeaderMap::new(),
6613 user_body(),
6614 )
6615 .await;
6616 let json_on = body_json(resp_on).await;
6617 let trace_on = rx_on.try_recv().unwrap();
6618
6619 assert_eq!(
6621 json_off["content"][0]["text"], json_on["content"][0]["text"],
6622 "served text must be identical regardless of probe"
6623 );
6624 assert!(
6626 (trace_off.final_.total_cost_usd - trace_on.final_.total_cost_usd).abs() < 1e-12,
6627 "total_cost_usd must not include probe cost: off={} on={}",
6628 trace_off.final_.total_cost_usd,
6629 trace_on.final_.total_cost_usd
6630 );
6631 assert!(trace_off.probe.is_none());
6633 assert!(trace_on.probe.is_some());
6634 assert!(
6636 trace_on.probe.as_ref().unwrap().probe_cost_usd > 0.0,
6637 "probe cost must be positive"
6638 );
6639 }
6640
6641 #[tokio::test]
6651 async fn probe_does_not_mutate_gate_health() {
6652 let model = "anthropic/claude-haiku-4-5";
6653 let (mut state, rx) = probe_state(1.0, 2, vec![(model, Ok(model_resp(model, "answer")))]);
6654
6655 let registry = GateHealthRegistry::new().with_budget("non-empty", 2, 0.4);
6658 registry.record("default", "non-empty", true);
6661 assert!(
6662 registry.enabled("default", "non-empty"),
6663 "gate must start enabled (window not full yet)"
6664 );
6665 state.gate_health = Arc::new(registry);
6666
6667 let trace = run_enforce_get_trace(state, rx).await;
6679 let sig = trace.probe.expect("probe must fire with sample_rate=1.0");
6680 assert_eq!(sig.k, 2);
6681 assert!(
6682 sig.gate_pass_count <= 2,
6683 "gate_pass_count must be in [0, k]"
6684 );
6685 }
6686
6687 fn predictor_state(
6689 enabled: bool,
6690 outcome: Result<ModelResponse, ProviderError>,
6691 ) -> (AppState, mpsc::Receiver<Trace>) {
6692 let toml = "[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/claude-haiku-4-5\"]\ngates = [\"non-empty\"]\n";
6693 let config = ProxyConfig::from_lookup(|k_| match k_ {
6694 "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
6695 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
6696 _ => None,
6697 })
6698 .unwrap();
6699 let mut outs = HashMap::new();
6700 outs.insert("anthropic/claude-haiku-4-5".to_owned(), outcome);
6701 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
6702 map.insert(
6703 "anthropic".to_owned(),
6704 Arc::new(MockProvider::new("anthropic", outs)),
6705 );
6706 let (traces, rx) = mpsc::channel(64);
6707 let predictor = enabled.then(|| {
6708 Arc::new(std::sync::Mutex::new(firstpass_core::PassPredictor::new(
6709 0.05, 1e-4,
6710 )))
6711 });
6712 let state = AppState {
6713 config: Arc::new(config),
6714 http: reqwest::Client::new(),
6715 providers: ProviderRegistry::from_map(map),
6716 gate_health: Arc::new(GateHealthRegistry::new()),
6717 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
6718 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
6719 traces,
6720 adaptive: None,
6721 eprocess: None,
6722 bandit: None,
6723 promoter: None,
6724 verified_cache: None,
6725 predictor,
6726 tenant_rate_limiter: None,
6727 spill: None,
6728 };
6729 (state, rx)
6730 }
6731
6732 #[tokio::test]
6733 async fn predictor_off_leaves_predicted_pass_none() {
6734 let (state, rx) =
6735 predictor_state(false, Ok(model_resp("anthropic/claude-haiku-4-5", "ok")));
6736 let trace = run_enforce_get_trace(state, rx).await;
6737 assert!(
6738 trace.predicted_pass.is_none(),
6739 "predictor off => no field (byte-identical)"
6740 );
6741 let j = serde_json::to_string(&trace).unwrap();
6743 assert!(!j.contains("predicted_pass"), "None must be omitted: {j}");
6744 }
6745
6746 #[tokio::test]
6747 async fn predictor_on_records_shadow_prediction_and_serves_identically() {
6748 let (state_off, rx_off) = predictor_state(
6751 false,
6752 Ok(model_resp("anthropic/claude-haiku-4-5", "served answer")),
6753 );
6754 let off = run_enforce_get_trace(state_off, rx_off).await;
6755
6756 let (state_on, rx_on) = predictor_state(
6757 true,
6758 Ok(model_resp("anthropic/claude-haiku-4-5", "served answer")),
6759 );
6760 let on = run_enforce_get_trace(state_on, rx_on).await;
6761
6762 assert_eq!(
6763 on.final_.served_rung, off.final_.served_rung,
6764 "served rung identical"
6765 );
6766 assert_eq!(on.attempts.len(), off.attempts.len(), "same attempts");
6767 assert_eq!(
6768 on.final_.total_cost_usd, off.final_.total_cost_usd,
6769 "predictor never adds served cost"
6770 );
6771 let p = on
6772 .predicted_pass
6773 .expect("predictor on => predicted_pass recorded");
6774 assert!(p > 0.0 && p < 1.0, "shadow prediction in (0,1): {p}");
6775 }
6776
6777 fn rollout_state(
6779 percent: f64,
6780 key: &str,
6781 outcomes: Vec<(&str, Result<ModelResponse, ProviderError>)>,
6782 ) -> (AppState, mpsc::Receiver<Trace>) {
6783 let toml = format!(
6784 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
6785 ladder = [\"anthropic/claude-haiku-4-5\"]\ngates = [\"non-empty\"]\n\
6786 [route.rollout]\npercent = {percent}\nkey = \"{key}\"\n"
6787 );
6788 let config = ProxyConfig::from_lookup(|k| match k {
6789 "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
6790 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
6791 _ => None,
6792 })
6793 .unwrap();
6794 let mut outs = HashMap::new();
6795 for (model, out) in outcomes {
6796 outs.insert(model.to_owned(), out);
6797 }
6798 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
6799 map.insert(
6800 "anthropic".to_owned(),
6801 Arc::new(MockProvider::new("anthropic", outs)),
6802 );
6803 let (traces, rx) = mpsc::channel(64);
6804 let state = AppState {
6805 config: Arc::new(config),
6806 http: reqwest::Client::new(),
6807 providers: ProviderRegistry::from_map(map),
6808 gate_health: Arc::new(GateHealthRegistry::new()),
6809 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
6810 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
6811 traces,
6812 adaptive: None,
6813 eprocess: None,
6814 bandit: None,
6815 promoter: None,
6816 verified_cache: None,
6817 predictor: None,
6818 tenant_rate_limiter: None,
6819 spill: None,
6820 };
6821 (state, rx)
6822 }
6823
6824 #[tokio::test]
6828 async fn rollout_at_zero_percent_never_enforces() {
6829 for i in 0..25 {
6830 let (state, mut rx) = rollout_state(
6831 0.0,
6832 "session",
6833 vec![(
6834 "anthropic/claude-haiku-4-5",
6835 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
6836 )],
6837 );
6838 let mut headers = HeaderMap::new();
6839 headers.insert("x-firstpass-session", format!("s{i}").parse().unwrap());
6840 let resp = messages(
6841 State(state),
6842 Extension(TenantId("default".to_owned())),
6843 headers,
6844 user_body(),
6845 )
6846 .await;
6847 let _ = resp;
6850 if let Ok(t) = rx.try_recv() {
6851 assert!(
6852 t.attempts.is_empty(),
6853 "0% rollout produced an enforced decision for session s{i}"
6854 );
6855 }
6856 }
6857 }
6858
6859 #[tokio::test]
6862 async fn rollout_at_hundred_percent_always_enforces() {
6863 for i in 0..25 {
6864 let (state, mut rx) = rollout_state(
6865 100.0,
6866 "session",
6867 vec![(
6868 "anthropic/claude-haiku-4-5",
6869 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
6870 )],
6871 );
6872 let mut headers = HeaderMap::new();
6873 headers.insert("x-firstpass-session", format!("s{i}").parse().unwrap());
6874 let resp = messages(
6875 State(state),
6876 Extension(TenantId("default".to_owned())),
6877 headers,
6878 user_body(),
6879 )
6880 .await;
6881 assert_eq!(resp.status(), axum::http::StatusCode::OK);
6882 let t = rx.try_recv().expect("trace enqueued");
6883 assert!(
6884 !t.attempts.is_empty(),
6885 "100% rollout skipped enforcement for session s{i}"
6886 );
6887 }
6888 }
6889
6890 #[tokio::test]
6898 async fn dispatch_arm_matches_the_predicted_arm_per_session() {
6899 let mut checked_enforced = 0;
6900 let mut checked_control = 0;
6901 for i in 0..24 {
6902 let session = format!("session-{i}");
6903 let (state, mut rx) = rollout_state(
6904 50.0,
6905 "session",
6906 vec![(
6907 "anthropic/claude-haiku-4-5",
6908 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
6909 )],
6910 );
6911 let expected = firstpass_core::rollout::decide(
6914 &state.config.prompt_salt,
6915 &firstpass_core::Rollout {
6916 percent: 50.0,
6917 key: firstpass_core::RolloutKey::Session,
6918 },
6919 &session,
6920 )
6921 .enforced;
6922
6923 let mut headers = HeaderMap::new();
6924 headers.insert("x-firstpass-session", session.parse().unwrap());
6925 let _ = messages(
6926 State(state),
6927 Extension(TenantId("default".to_owned())),
6928 headers,
6929 user_body(),
6930 )
6931 .await;
6932 let observed = rx
6933 .try_recv()
6934 .map(|t| !t.attempts.is_empty())
6935 .unwrap_or(false);
6936
6937 assert_eq!(
6938 observed,
6939 expected,
6940 "{session}: dispatch put it in the {} arm, bucketing predicted the {} arm",
6941 if observed { "enforced" } else { "control" },
6942 if expected { "enforced" } else { "control" }
6943 );
6944 if expected {
6945 checked_enforced += 1;
6946 } else {
6947 checked_control += 1;
6948 }
6949 }
6950 assert!(
6952 checked_enforced > 0 && checked_control > 0,
6953 "test saw only one arm ({checked_enforced} enforced, {checked_control} control)"
6954 );
6955 }
6956}