1use std::sync::Arc;
5use std::time::Instant;
6
7use axum::body::Body;
8use axum::extract::{Request, State};
9use axum::http::HeaderMap;
10use axum::middleware::Next;
11use axum::response::{IntoResponse, Response};
12use axum::routing::{get, post};
13use axum::{Extension, Json, Router};
14use bytes::Bytes;
15use firstpass_core::features::{hour_bucket, token_bucket};
16use firstpass_core::hashchain::sha256_hex;
17use firstpass_core::{
18 Attempt, DeferredVerdict, Dialect, FEATURE_VERSION, Features, FinalOutcome, GENESIS_HASH, Mode,
19 ModelRef, PolicyRef, ProbeRegime, ProbeSignal, RequestInfo, RoutingMode, Score, ServedFrom,
20 TaskKind, Trace, Verdict,
21};
22use serde::Deserialize;
23use serde_json::Value;
24use std::future::Future;
25use std::time::Duration;
26use tokio::sync::mpsc::error::TrySendError;
27use uuid::Uuid;
28
29use crate::config::ProxyConfig;
30use crate::error::ProxyError;
31use crate::gate::{GateHealthRegistry, aggregate_with_policy, resolve_gates};
32use crate::provider::{Auth, ChatMessage, ModelRequest, ModelResponse, ProviderRegistry};
33use crate::router::{EnforceCtx, EngineOutcome, route_enforce};
34use crate::store;
35use crate::tenant_auth::{TenantId, auth_middleware};
36use crate::upstream::{
37 forward_anthropic, forward_anthropic_streaming, forward_openai, forward_openai_streaming,
38};
39use firstpass_core::Route;
40use firstpass_core::trace::ShadowSignal;
41
42#[derive(Clone)]
45pub struct AppState {
46 pub config: Arc<ProxyConfig>,
48 pub http: reqwest::Client,
50 pub providers: ProviderRegistry,
52 pub gate_health: Arc<GateHealthRegistry>,
54 pub shadow_ledger: Arc<crate::shadow::ShadowLedger>,
57 pub guardrails: Arc<crate::guard::GuardrailRegistry>,
60 pub traces: store::TraceSender,
62 pub adaptive: Option<Arc<std::sync::Mutex<firstpass_core::conformal::AdaptiveConformal>>>,
66 pub bandit: Option<Arc<std::sync::Mutex<crate::bandit::StartRungBandit>>>,
71 pub predictor: Option<Arc<std::sync::Mutex<firstpass_core::PassPredictor>>>,
77 pub tenant_rate_limiter: Option<Arc<governor::DefaultKeyedRateLimiter<String>>>,
81 pub spill: Option<store::SpillHandle>,
85}
86
87#[must_use]
91pub fn build_tenant_rate_limiter(
92 config: &ProxyConfig,
93) -> Option<Arc<governor::DefaultKeyedRateLimiter<String>>> {
94 let per_sec = config.tenant_rate_per_sec?;
95 Some(Arc::new(governor::RateLimiter::keyed(
96 governor::Quota::per_second(per_sec),
97 )))
98}
99
100pub async fn tenant_rate_limit_middleware(
104 State(state): State<AppState>,
105 Extension(tenant): Extension<TenantId>,
106 req: Request,
107 next: Next,
108) -> Response {
109 if let Some(limiter) = &state.tenant_rate_limiter
110 && limiter.check_key(&tenant.0).is_err()
111 {
112 return ProxyError::RateLimited.into_response();
113 }
114 next.run(req).await
115}
116
117impl std::fmt::Debug for AppState {
118 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
119 f.debug_struct("AppState")
120 .field("config", &self.config)
121 .finish_non_exhaustive()
122 }
123}
124
125fn offer_trace(traces: &store::TraceSender, spill: Option<&store::SpillHandle>, trace: Trace) {
142 record_trace_metrics(&trace);
143 match traces.try_send(trace) {
144 Ok(()) => {}
145 Err(TrySendError::Full(t)) => {
146 if let Some(handle) = spill {
147 match store::append_to_spill(handle, &t) {
148 Ok(()) => {
149 metrics::counter!("firstpass_receipts_spilled_total").increment(1);
150 }
151 Err(e) => {
152 tracing::error!(%e, "durable mode: spill write failed; trace lost");
153 metrics::counter!("firstpass_traces_dropped_total").increment(1);
154 }
155 }
156 } else {
157 tracing::warn!("trace channel full; dropping trace (writer behind under load)");
158 metrics::counter!("firstpass_traces_dropped_total").increment(1);
159 }
160 }
161 Err(TrySendError::Closed(_)) => {
162 tracing::warn!("trace writer is gone; dropping trace");
163 }
164 }
165}
166
167fn record_trace_metrics(trace: &Trace) {
171 if trace.mode == Mode::Enforce {
172 metrics::histogram!("firstpass_enforce_latency_ms")
173 .record(trace.final_.total_latency_ms as f64);
174 if trace.final_.escalations > 0 {
175 metrics::counter!("firstpass_escalations_total")
176 .increment(u64::from(trace.final_.escalations));
177 }
178 }
179 let served_from = match trace.final_.served_from {
180 ServedFrom::Attempt => "attempt",
181 ServedFrom::BestAttempt => "best_attempt",
182 ServedFrom::Error => "error",
183 };
184 metrics::counter!("firstpass_served_total", "served_from" => served_from).increment(1);
185 if trace.final_.served_from == ServedFrom::Error {
186 metrics::counter!("firstpass_upstream_failures_total").increment(1);
187 }
188 metrics::gauge!("firstpass_cost_usd_total").increment(trace.final_.total_cost_usd);
192 metrics::gauge!("firstpass_gate_cost_usd_total").increment(trace.final_.gate_cost_usd);
193 metrics::gauge!("firstpass_baseline_usd_total")
194 .increment(trace.final_.counterfactual_baseline_usd);
195 metrics::gauge!("firstpass_savings_usd_total").increment(trace.final_.savings_usd);
196 if let Some(rung) = trace.final_.served_rung {
198 let model = trace
199 .attempts
200 .iter()
201 .find(|a| a.rung == rung)
202 .map(|a| a.model.clone())
203 .unwrap_or_else(|| "unknown".to_owned());
204 metrics::counter!(
205 "firstpass_served_rung_total",
206 "rung" => rung.to_string(),
207 "model" => model
208 )
209 .increment(1);
210 }
211}
212
213const MAX_BODY_BYTES: usize = 16 * 1024 * 1024;
217
218pub(crate) fn u01(seed: u128) -> f64 {
229 let lo = splitmix64_finalise(seed as u64);
230 let hi = splitmix64_finalise((seed >> 64) as u64);
231 ((lo ^ hi) >> 11) as f64 * (1.0_f64 / (1u64 << 53) as f64)
233}
234
235fn splitmix64_finalise(mut z: u64) -> u64 {
236 z = z.wrapping_add(0x9E37_79B9_7F4A_7C15);
237 z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
238 z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
239 z ^ (z >> 31)
240}
241
242#[must_use]
249pub(crate) fn epsilon_propensity(chosen: u32, greedy: u32, epsilon: f64, k: usize) -> f64 {
250 let greedy_term = if chosen == greedy { 1.0 - epsilon } else { 0.0 };
251 greedy_term + epsilon / k as f64
252}
253
254pub fn app(state: AppState) -> Result<Router, ProxyError> {
261 crate::metrics::install()?;
262 let max_concurrency = state.config.max_concurrency;
263
264 let business = Router::new()
273 .route("/v1/messages", post(messages))
274 .route("/v1/chat/completions", post(chat_completions))
275 .route("/v1/feedback", post(feedback))
276 .route("/v1/capabilities", get(capabilities))
277 .route("/v1/receipts", get(receipts))
280 .layer(axum::middleware::from_fn_with_state(
281 state.clone(),
282 tenant_rate_limit_middleware,
283 ))
284 .layer(axum::middleware::from_fn_with_state(
285 state.clone(),
286 auth_middleware,
287 ));
288
289 Ok(Router::new()
290 .merge(business)
291 .route("/healthz", get(healthz))
292 .route("/metrics", get(crate::metrics::handler))
293 .route("/panel", get(panel))
296 .layer(axum::extract::DefaultBodyLimit::max(MAX_BODY_BYTES))
298 .layer(tower::limit::GlobalConcurrencyLimitLayer::new(
301 max_concurrency,
302 ))
303 .with_state(state))
304}
305
306async fn healthz() -> impl IntoResponse {
308 Json(serde_json::json!({ "status": "ok" }))
309}
310
311const PANEL_HTML: &str = include_str!("panel.html");
315
316async fn receipts(
322 State(state): State<AppState>,
323 Extension(TenantId(tenant)): Extension<TenantId>,
324 axum::extract::Query(q): axum::extract::Query<ReceiptsQuery>,
325) -> Response {
326 let limit = q.limit.unwrap_or(20).clamp(1, 200);
328 let db = state.config.db_path.clone();
329 let traces =
330 match tokio::task::spawn_blocking(move || crate::store::load_tenant_traces(&db, &tenant))
331 .await
332 {
333 Ok(Ok(t)) => t,
334 _ => Vec::new(),
337 };
338 let items: Vec<serde_json::Value> = traces
339 .iter()
340 .rev()
341 .take(limit)
342 .map(|t| {
343 let f = &t.final_;
344 serde_json::json!({
345 "trace_id": t.trace_id,
346 "served_rung": f.served_rung,
347 "served_model": f.served_rung
352 .and_then(|r| t.attempts.iter().find(|a| a.rung == r))
353 .map(|a| a.model.clone()),
354 "attempts": t.attempts.iter().map(|a| serde_json::json!({
355 "rung": a.rung,
356 "model": a.model,
357 "verdict": a.verdict,
358 "cost_usd": a.cost_usd,
359 "latency_ms": a.latency_ms,
360 })).collect::<Vec<_>>(),
361 "total_cost_usd": f.total_cost_usd,
362 "baseline_usd": f.counterfactual_baseline_usd,
363 "savings_usd": f.savings_usd,
364 })
365 })
366 .collect();
367 axum::Json(serde_json::json!({ "receipts": items })).into_response()
368}
369
370#[derive(Debug, serde::Deserialize)]
372struct ReceiptsQuery {
373 limit: Option<usize>,
375}
376
377async fn panel() -> impl IntoResponse {
383 (
384 [(axum::http::header::CONTENT_TYPE, "text/html; charset=utf-8")],
385 PANEL_HTML,
386 )
387}
388
389async fn capabilities(State(state): State<AppState>) -> impl IntoResponse {
390 let (ladder, gates) = state
393 .config
394 .routing
395 .as_ref()
396 .and_then(|c| c.routes.iter().find(|r| r.mode == Mode::Enforce))
397 .map(|r| (r.ladder.clone(), r.gates.clone()))
398 .unwrap_or_default();
399 let routing_modes: Vec<serde_json::Value> = RoutingMode::ALL
400 .iter()
401 .map(|m| {
402 let p = m.preset();
403 serde_json::json!({
404 "name": m.as_str(),
405 "description": p.description,
406 "tradeoff": p.tradeoff,
407 })
408 })
409 .collect();
410 Json(serde_json::json!({
411 "service": "firstpass",
412 "version": env!("CARGO_PKG_VERSION"),
413 "feature_version": FEATURE_VERSION,
414 "modes": ["observe", "enforce"],
415 "routing_modes": routing_modes,
416 "wire_apis": ["anthropic.messages", "openai.chat_completions"],
417 "ladder": ladder,
418 "gates": gates,
419 "feedback_api": "POST /v1/feedback",
420 "offboarding": "unset ANTHROPIC_BASE_URL (or OPENAI_BASE_URL for OpenAI clients)",
421 }))
422}
423
424#[derive(Debug, Deserialize)]
426struct FeedbackRequest {
427 trace_id: String,
429 gate_id: String,
431 verdict: String,
433 #[serde(default)]
435 score: Option<f64>,
436 reporter: String,
438}
439
440async fn feedback(
445 State(state): State<AppState>,
446 Extension(TenantId(tenant)): Extension<TenantId>,
447 body: Bytes,
448) -> Response {
449 let req: FeedbackRequest = match serde_json::from_slice(&body) {
450 Ok(r) => r,
451 Err(e) => {
452 return ProxyError::BadRequest(format!("invalid feedback body: {e}")).into_response();
453 }
454 };
455 let verdict = match req.verdict.as_str() {
456 "pass" => Verdict::Pass,
457 "fail" => Verdict::Fail,
458 "abstain" => Verdict::Abstain,
459 other => {
460 return ProxyError::BadRequest(format!("unknown verdict {other:?}")).into_response();
461 }
462 };
463 let score = match req.score {
464 Some(s) => match Score::new(s) {
465 Ok(sc) => Some(sc),
466 Err(_) => {
467 return ProxyError::BadRequest(format!("score {s} out of range [0,1]"))
468 .into_response();
469 }
470 },
471 None => None,
472 };
473
474 let db = state.config.db_path.clone();
475
476 let (db_check, tenant_check, tid_check) = (db.clone(), tenant.clone(), req.trace_id.clone());
481 match tokio::task::spawn_blocking(move || {
482 store::trace_exists(&db_check, &tenant_check, &tid_check)
483 })
484 .await
485 {
486 Ok(Ok(true)) => {}
487 Ok(Ok(false)) => {
488 return ProxyError::NotFound(format!("unknown trace_id {:?}", req.trace_id))
489 .into_response();
490 }
491 Ok(Err(e)) => {
492 tracing::error!(%e, "feedback: trace_exists check failed");
493 return ProxyError::Internal(e.to_string()).into_response();
494 }
495 Err(e) => {
496 tracing::error!(%e, "feedback: trace_exists task panicked");
497 return ProxyError::Internal(e.to_string()).into_response();
498 }
499 }
500
501 let feedback_signal = match verdict {
503 Verdict::Pass => Some(true),
504 Verdict::Fail => Some(false),
505 Verdict::Abstain => None,
506 };
507 let dv = DeferredVerdict {
508 gate_id: req.gate_id,
509 verdict,
510 score,
511 reported_at: jiff::Timestamp::now(),
512 reporter: req.reporter,
513 };
514 let trace_id = req.trace_id.clone();
515 match tokio::task::spawn_blocking(move || store::append_deferred(&db, &req.trace_id, &dv)).await
516 {
517 Ok(Ok(())) => {
518 if let (Some(a), Some(correct)) = (state.adaptive.as_ref(), feedback_signal)
520 && let Ok(mut g) = a.lock()
521 {
522 g.observe_served(correct);
523 metrics::gauge!("firstpass_serve_threshold").set(g.threshold());
524 metrics::gauge!("firstpass_realized_served_failure")
525 .set(g.realized_served_failure());
526 }
527
528 let attributed_route = tokio::task::spawn_blocking({
533 let (db, ten, tid) = (
534 state.config.db_path.clone(),
535 tenant.clone(),
536 trace_id.clone(),
537 );
538 move || store::trace_route(&db, &ten, &tid)
539 })
540 .await
541 .ok()
542 .and_then(Result::ok)
543 .flatten()
544 .flatten()
545 .unwrap_or(0) as usize;
546
547 if let (Some(cfg), Some(correct)) = (
552 state.config.routing.as_ref().and_then(|r| r.guardrail),
553 feedback_signal,
554 ) {
555 let cooldown = state
556 .config
557 .routing
558 .as_ref()
559 .map_or(3_600, |r| r.guardrail_cooldown_secs);
560 let reaction = state.guardrails.record(
561 &tenant,
562 attributed_route,
563 &cfg,
564 correct,
565 jiff::Timestamp::now().as_second(),
566 cooldown,
567 );
568 match &reaction {
569 crate::guard::Reaction::Demoted(v) => {
570 metrics::counter!("firstpass_guardrail_demotions_total").increment(1);
571 tracing::error!(
572 tenant = %tenant,
573 n = v.n,
574 rate = v.rate,
575 bound = v.bound,
576 alpha = cfg.alpha,
577 "GUARDRAIL: served-failure bound exceeded target — route demoted to observe; traffic now serves as it would without Firstpass"
578 );
579 }
580 crate::guard::Reaction::Alarmed(v) => {
581 metrics::counter!("firstpass_guardrail_breaches_total").increment(1);
582 tracing::error!(
583 tenant = %tenant,
584 n = v.n,
585 rate = v.rate,
586 bound = v.bound,
587 alpha = cfg.alpha,
588 "GUARDRAIL: served-failure bound exceeded target (alarm only — routing unchanged)"
589 );
590 }
591 crate::guard::Reaction::None => {}
592 }
593 }
594 (
595 axum::http::StatusCode::ACCEPTED,
596 Json(serde_json::json!({ "status": "recorded", "trace_id": trace_id })),
597 )
598 .into_response()
599 }
600 Ok(Err(e)) => {
601 tracing::error!(%e, "feedback: append_deferred failed");
602 ProxyError::Internal(e.to_string()).into_response()
603 }
604 Err(e) => {
605 tracing::error!(%e, "feedback: append_deferred task panicked");
606 ProxyError::Internal(e.to_string()).into_response()
607 }
608 }
609}
610
611const SESSION_HEADER: &str = "x-firstpass-session";
614
615const AGENT_HEADER: &str = "x-firstpass-agent";
617const SUBAGENT_HEADER: &str = "x-firstpass-subagent";
619#[expect(
625 clippy::too_many_arguments,
626 reason = "mirrors the enforce context it evaluates; a wrapper struct would only move the list"
627)]
628fn spawn_shadow(
629 state: &AppState,
630 route: &firstpass_core::Route,
631 route_ix: usize,
632 body: Bytes,
633 auth: Auth,
634 features: Features,
635 tenant: String,
636 session_id: String,
637 api: &str,
638) {
639 let Some(shadow) = route.shadow else {
640 return;
641 };
642 if !shadow.sampled(&state.config.prompt_salt, &session_id) {
645 return;
646 }
647 let (state, route, api) = (state.clone(), route.clone(), api.to_owned());
648 tokio::spawn(async move {
649 let signal = evaluate_shadow(
650 &state, &route, shadow, route_ix, &body, auth, &features, tenant, session_id, &api, 0.0,
651 )
652 .await;
653 tracing::debug!(
655 would_pass = signal.would_pass,
656 projected_usd = signal.projected_cost_usd,
657 skipped = ?signal.skipped,
658 "shadow evaluation complete"
659 );
660 });
661}
662
663#[expect(
670 clippy::too_many_arguments,
671 reason = "mirrors the enforce context; grouping them into a struct would only move the list"
672)]
673async fn evaluate_shadow(
674 state: &AppState,
675 route: &firstpass_core::Route,
676 shadow: firstpass_core::rollout::Shadow,
677 route_ix: usize,
678 body: &Bytes,
679 auth: Auth,
680 features: &Features,
681 tenant: String,
682 session_id: String,
683 api: &str,
684 actual_cost_usd: f64,
685) -> ShadowSignal {
686 let skipped = |why: &str| ShadowSignal {
687 would_serve_rung: None,
688 would_pass: false,
689 projected_cost_usd: 0.0,
690 actual_cost_usd,
691 skipped: Some(why.to_owned()),
692 };
693
694 let now = jiff::Timestamp::now();
695 if !state
696 .shadow_ledger
697 .may_spend(&tenant, route_ix, shadow.max_usd_per_day, now)
698 {
699 return skipped("budget_exhausted");
702 }
703
704 let Some(base_request) = parse_model_request(body) else {
705 return skipped("unparseable_request");
706 };
707 let gate_defs = state
708 .config
709 .routing
710 .as_ref()
711 .map_or(&[][..], |cfg| &cfg.gate_defs);
712 let gates = resolve_gates(
713 &route.gates,
714 gate_defs,
715 &state.providers,
716 &auth,
717 &state.config.prices,
718 );
719 let (budget, max_rungs) = state
720 .config
721 .routing
722 .as_ref()
723 .map_or((None, u32::MAX), |cfg| {
724 (
725 cfg.budget.per_request_usd,
726 cfg.escalation.max_rungs_per_request,
727 )
728 });
729
730 let ctx = EnforceCtx {
731 ladder: &route.ladder,
732 gates: &gates,
733 health: &state.gate_health,
734 base_request: &base_request,
735 providers: &state.providers,
736 auth: &auth,
737 prices: &state.config.prices,
738 budget_per_request_usd: budget,
739 max_rungs,
740 speculation: 0,
743 serve_threshold: None,
744 elastic: None,
745 features: features.clone(),
746 start_rung: 0,
747 tenant_id: tenant.clone(),
748 session_id,
749 prompt_hash: prompt_hash(&state.config.prompt_salt, body),
750 api: api.to_owned(),
751 policy_id: "shadow".to_owned(),
752 };
753
754 let (outcome, trace) = route_enforce(ctx).await;
755 let spent = trace.final_.total_cost_usd;
756 state.shadow_ledger.debit(&tenant, route_ix, spent, now);
757
758 let would_pass = matches!(outcome, EngineOutcome::Served(_));
761 let would_serve_rung = if would_pass {
762 trace.final_.served_rung
763 } else {
764 None
765 };
766 ShadowSignal {
767 would_serve_rung,
768 would_pass,
769 projected_cost_usd: spent,
770 actual_cost_usd,
771 skipped: None,
772 }
773}
774
775const MODE_PROFILE_HEADER: &str = "x-firstpass-mode";
778
779fn resolve_mode(headers: &HeaderMap, route: &Route, config: &ProxyConfig) -> RoutingMode {
788 if let Some(val) = header_str(headers, MODE_PROFILE_HEADER) {
790 match val.trim().to_ascii_lowercase().as_str() {
791 "observe" => return RoutingMode::Observe,
792 "cost" => return RoutingMode::Cost,
793 "balanced" => return RoutingMode::Balanced,
794 "quality" => return RoutingMode::Quality,
795 "latency" => return RoutingMode::Latency,
796 "max" => return RoutingMode::Max,
797 other => {
798 tracing::warn!(
799 value = other,
800 "unknown x-firstpass-mode value; ignoring \
801 (valid: observe|cost|balanced|quality|latency|max)"
802 );
803 }
804 }
805 }
806 if let Some(m) = route.routing_mode {
808 return m;
809 }
810 config.default_routing_mode
812}
813
814async fn messages(
819 State(state): State<AppState>,
820 Extension(TenantId(tenant)): Extension<TenantId>,
821 headers: HeaderMap,
822 body: Bytes,
823) -> Response {
824 let session_header = header_str(&headers, SESSION_HEADER);
825
826 if let Some(routing) = state.config.routing.as_ref() {
829 let features = extract_features(&headers, &body);
830 if let Some(route) = routing
831 .route_for(&features)
832 .filter(|r| r.mode == Mode::Enforce && !r.ladder.is_empty())
833 {
834 let route_ix = routing
837 .routes
838 .iter()
839 .position(|r| std::ptr::eq(r, route))
840 .unwrap_or(0);
841 let route = route.clone();
844 let routing_mode = resolve_mode(&headers, &route, &state.config);
846 if routing_mode == RoutingMode::Observe {
848 return observe_passthrough(state, headers, body, session_header, tenant).await;
849 }
850 if state
854 .guardrails
855 .is_demoted(&tenant, route_ix, jiff::Timestamp::now().as_second())
856 {
857 return observe_passthrough(state, headers, body, session_header, tenant).await;
858 }
859 if let Some(rollout) = route.rollout.as_ref() {
864 let key_value = match rollout.key {
865 firstpass_core::RolloutKey::Session => session_header
866 .clone()
867 .unwrap_or_else(|| firstpass_core::rollout::request_identity(&body)),
868 firstpass_core::RolloutKey::Request => {
874 firstpass_core::rollout::request_identity(&body)
875 }
876 firstpass_core::RolloutKey::Tenant => tenant.to_string(),
877 };
878 let decision =
879 firstpass_core::rollout::decide(&state.config.prompt_salt, rollout, &key_value);
880 if !decision.enforced {
881 let shadow_ctx = route.shadow.map(|_| {
883 (
884 body.clone(),
885 Auth::from_headers(&headers),
886 features.clone(),
887 tenant.clone(),
888 session_header
889 .clone()
890 .unwrap_or_else(|| Uuid::now_v7().to_string()),
891 )
892 });
893 let resp =
894 observe_passthrough(state.clone(), headers, body, session_header, tenant)
895 .await;
896 if let Some((b, auth, feats, ten, sess)) = shadow_ctx {
899 spawn_shadow(
900 &state,
901 &route,
902 route_ix,
903 b,
904 auth,
905 feats,
906 ten,
907 sess,
908 "anthropic",
909 );
910 }
911 return resp;
912 }
913 }
914 if enforce_can_handle(
915 &features,
916 &body,
917 routing.escalation.enforce_structured,
918 &route.ladder,
919 &state.providers,
920 Dialect::Anthropic,
921 ) {
922 return handle_enforce(
923 &state,
924 &headers,
925 &body,
926 features,
927 &route,
928 route_ix,
929 session_header,
930 tenant,
931 routing_mode,
932 )
933 .await;
934 }
935 tracing::info!(
939 "enforce route matched but structured request can't be routed faithfully (flag/ladder); serving via observe passthrough"
940 );
941 }
942 }
943 observe_passthrough(state, headers, body, session_header, tenant).await
944}
945
946fn enforce_can_handle(
960 features: &Features,
961 body: &[u8],
962 enforce_structured: bool,
963 ladder: &[String],
964 providers: &crate::provider::ProviderRegistry,
965 inbound: Dialect,
966) -> bool {
967 let structured = features.tool_count > 0
968 || features.has_images
969 || match inbound {
970 Dialect::Anthropic => messages_have_tool_blocks(body),
971 Dialect::Openai => openai_messages_have_tool_calls(body),
972 Dialect::Gemini => false,
973 };
974 if !structured {
975 return true;
976 }
977 if !enforce_structured {
978 return false;
979 }
980 let all_verbatim = ladder.iter().all(|rung| {
982 let provider_id = rung.split('/').next().unwrap_or_default();
983 providers
984 .get(provider_id)
985 .is_some_and(|p| p.carries_structured_verbatim(inbound))
986 });
987 if all_verbatim {
988 return true;
989 }
990 if inbound == Dialect::Openai && !openai_has_http_images(body) {
994 let all_anthropic = ladder.iter().all(|rung| {
995 let pid = rung.split('/').next().unwrap_or_default();
996 providers
997 .get(pid)
998 .is_some_and(|p| p.carries_structured_verbatim(Dialect::Anthropic))
999 });
1000 if all_anthropic {
1001 return true;
1002 }
1003 }
1004 false
1005}
1006
1007fn messages_have_tool_blocks(body: &[u8]) -> bool {
1010 serde_json::from_slice::<Value>(body)
1011 .ok()
1012 .and_then(|json| {
1013 json.get("messages")
1014 .and_then(Value::as_array)
1015 .map(|messages| messages.iter().any(message_has_tool_block))
1016 })
1017 .unwrap_or(false)
1018}
1019
1020fn message_has_tool_block(message: &Value) -> bool {
1022 message
1023 .get("content")
1024 .and_then(Value::as_array)
1025 .is_some_and(|blocks| {
1026 blocks.iter().any(|block| {
1027 matches!(
1028 block.get("type").and_then(Value::as_str),
1029 Some("tool_use" | "tool_result")
1030 )
1031 })
1032 })
1033}
1034
1035fn is_stream_request(body: &[u8]) -> bool {
1037 serde_json::from_slice::<Value>(body)
1038 .ok()
1039 .and_then(|json| json.get("stream").and_then(Value::as_bool))
1040 .unwrap_or(false)
1041}
1042
1043fn header_str(headers: &HeaderMap, name: &str) -> Option<String> {
1045 headers
1046 .get(name)
1047 .and_then(|v| v.to_str().ok())
1048 .map(str::to_owned)
1049}
1050
1051fn extract_features(headers: &HeaderMap, body: &[u8]) -> Features {
1054 let (_model, tool_count, has_images) = request_features(body);
1055 let mut f = Features::new(TaskKind::Other);
1056 f.agent = header_str(headers, AGENT_HEADER);
1057 f.subagent = header_str(headers, SUBAGENT_HEADER);
1058 f.tool_count = tool_count;
1059 f.has_images = has_images;
1060 f.prompt_token_bucket = token_bucket(body.len() as u64);
1063 f.hour_bucket = hour_bucket(jiff::Timestamp::now());
1064 f
1065}
1066
1067#[allow(clippy::too_many_arguments)]
1070async fn handle_enforce(
1071 state: &AppState,
1072 headers: &HeaderMap,
1073 body: &Bytes,
1074 features: Features,
1075 route: &Route,
1076 route_ix: usize,
1079 session_header: Option<String>,
1080 tenant: String,
1081 routing_mode: RoutingMode,
1082) -> Response {
1083 if is_stream_request(body) {
1091 let (tx, rx) = tokio::sync::oneshot::channel::<Result<Value, ProxyError>>();
1092 let (state_c, headers_c, body_c, route_c) =
1093 (state.clone(), headers.clone(), body.clone(), route.clone());
1094 tokio::spawn(async move {
1095 let out = enforce_pipeline(
1096 &state_c,
1097 &headers_c,
1098 &body_c,
1099 features,
1100 &route_c,
1101 route_ix,
1102 session_header,
1103 tenant,
1104 routing_mode,
1105 )
1106 .await;
1107 let _ = tx.send(out);
1108 });
1109 return sse_keepalive_response(rx, anthropic_sse_from_message);
1110 }
1111 match enforce_pipeline(
1112 state,
1113 headers,
1114 body,
1115 features,
1116 route,
1117 route_ix,
1118 session_header,
1119 tenant,
1120 routing_mode,
1121 )
1122 .await
1123 {
1124 Ok(message) => (axum::http::StatusCode::OK, Json(message)).into_response(),
1125 Err(e) => e.into_response(),
1126 }
1127}
1128
1129#[allow(clippy::too_many_arguments)] async fn enforce_pipeline_inner(
1134 state: &AppState,
1135 body: &Bytes,
1136 base_request: ModelRequest,
1137 auth: Auth,
1138 features: Features,
1139 route: &Route,
1140 session_header: Option<String>,
1141 tenant: String,
1142 api: &str,
1143 routing_mode: RoutingMode,
1144 route_ix: usize,
1145) -> Result<ModelResponse, ProxyError> {
1146 let gate_defs = state
1147 .config
1148 .routing
1149 .as_ref()
1150 .map_or(&[][..], |cfg| &cfg.gate_defs);
1151 let gates = resolve_gates(
1152 &route.gates,
1153 gate_defs,
1154 &state.providers,
1155 &auth,
1156 &state.config.prices,
1157 );
1158 let session_id = session_header.unwrap_or_else(|| Uuid::now_v7().to_string());
1159 let (budget, max_rungs, speculation, serve_threshold, elastic) =
1160 match state.config.routing.as_ref() {
1161 Some(cfg) => (
1162 cfg.budget.per_request_usd,
1163 cfg.escalation.max_rungs_per_request,
1164 cfg.escalation.speculation,
1165 cfg.escalation.serve_threshold,
1166 cfg.escalation.elastic.as_ref(),
1167 ),
1168 None => (None, 3, 0, None, None),
1169 };
1170 let serve_threshold = state
1173 .adaptive
1174 .as_ref()
1175 .and_then(|a| a.lock().ok().map(|g| g.threshold()))
1176 .or(serve_threshold);
1177
1178 let preset = routing_mode.preset();
1181 let max_rungs = if let Some(delta) = preset.max_rungs_delta {
1182 (max_rungs as i32 + delta).max(1) as u32
1183 } else {
1184 max_rungs
1185 };
1186 let speculation = preset.speculation.unwrap_or(speculation);
1187
1188 let bandit_ctx = crate::bandit::ContextBucket::from_features(&features);
1192
1193 let (greedy_rung, base_policy_id, ts_propensity) = {
1197 let (chosen, ts_p) = state
1198 .bandit
1199 .as_ref()
1200 .and_then(|b| b.lock().ok())
1201 .map(|mut b| {
1202 b.choose_start_with_propensity(&bandit_ctx, &route.ladder, &state.config.prices)
1203 })
1204 .unwrap_or((0, None));
1205 let policy = if ts_p.is_some() {
1206 "bandit@v2-ts".to_owned()
1207 } else if chosen > 0 {
1208 "bandit@v1".to_owned()
1209 } else {
1210 "static-ladder@v0".to_owned()
1211 };
1212 (chosen, policy, ts_p)
1213 };
1214
1215 let exploration_epsilon = state
1220 .config
1221 .routing
1222 .as_ref()
1223 .and_then(|cfg| cfg.escalation.exploration.as_ref())
1224 .map(|e| e.epsilon);
1225
1226 let (start_rung, policy_id, explore_flag, propensity) = if ts_propensity.is_some() {
1227 if exploration_epsilon.is_some() {
1230 tracing::warn!(
1231 "bandit.algorithm = thompson already logs propensities; \
1232 [escalation.exploration] epsilon is ignored"
1233 );
1234 }
1235 (greedy_rung, base_policy_id, false, ts_propensity)
1236 } else if let Some(epsilon) = exploration_epsilon {
1237 let k = route.ladder.len().max(1);
1238 let u = u01(Uuid::now_v7().as_u128());
1240 let (chosen, eps_branch) = if u < epsilon {
1241 let idx = ((u / epsilon) * k as f64) as u32;
1243 (idx.min(k as u32 - 1), true)
1244 } else {
1245 (greedy_rung, false)
1246 };
1247 let p = epsilon_propensity(chosen, greedy_rung, epsilon, k);
1248 (chosen, format!("{base_policy_id}+eps"), eps_branch, Some(p))
1249 } else {
1250 (greedy_rung, base_policy_id, false, None)
1251 };
1252
1253 let start_rung = if preset.start_at_top {
1256 route.ladder.len().saturating_sub(1) as u32
1257 } else {
1258 start_rung
1259 };
1260
1261 let speculation = match state
1267 .config
1268 .routing
1269 .as_ref()
1270 .and_then(|cfg| cfg.escalation.speculation_band)
1271 {
1272 Some([lo, hi]) if speculation > 0 => {
1273 let estimate = state
1274 .bandit
1275 .as_ref()
1276 .and_then(|b| b.lock().ok())
1277 .and_then(|b| b.pass_estimate(&bandit_ctx, start_rung));
1278 match estimate {
1279 Some(p) if p < lo || p > hi => {
1280 metrics::counter!("firstpass_speculation_skipped_total").increment(1);
1281 0
1282 }
1283 _ => speculation,
1284 }
1285 }
1286 _ => speculation,
1287 };
1288
1289 if state.bandit.is_some() {
1291 metrics::counter!(
1292 "firstpass_bandit_start_rung",
1293 "rung" => start_rung.to_string()
1294 )
1295 .increment(1);
1296 }
1297
1298 let ctx = EnforceCtx {
1299 ladder: &route.ladder,
1300 gates: &gates,
1301 health: &state.gate_health,
1302 base_request: &base_request,
1303 providers: &state.providers,
1304 auth: &auth,
1305 prices: &state.config.prices,
1306 budget_per_request_usd: budget,
1307 max_rungs,
1308 speculation,
1309 serve_threshold,
1310 elastic,
1311 features,
1312 start_rung,
1313 tenant_id: tenant,
1316 session_id,
1317 prompt_hash: prompt_hash(&state.config.prompt_salt, body),
1318 api: api.to_owned(),
1319 policy_id,
1320 };
1321
1322 let (outcome, mut trace) = route_enforce(ctx).await;
1323
1324 trace.route_ix = Some(u32::try_from(route_ix).unwrap_or(0));
1328 trace.policy.explore = explore_flag;
1331 trace.policy.propensity = propensity;
1332 if routing_mode != RoutingMode::Balanced {
1335 trace.policy.mode_profile = Some(routing_mode.as_str().to_owned());
1336 }
1337
1338 if let Some(bandit) = state.bandit.as_ref()
1342 && let Ok(mut b) = bandit.lock()
1343 {
1344 for attempt in &trace.attempts {
1345 b.observe(&bandit_ctx, attempt.rung, attempt.verdict);
1346 }
1347 }
1348
1349 if let Some(predictor) = state.predictor.as_ref()
1355 && let Ok(mut p) = predictor.lock()
1356 {
1357 let predicted = p.predict(&trace.request.features, start_rung);
1359 for attempt in &trace.attempts {
1360 match attempt.verdict {
1361 Verdict::Pass => p.update(&trace.request.features, attempt.rung, true),
1362 Verdict::Fail => p.update(&trace.request.features, attempt.rung, false),
1363 Verdict::Abstain => {} }
1365 }
1366 trace.predicted_pass = Some(predicted);
1367 metrics::histogram!("firstpass_predictor_pass_prob").record(predicted);
1368 }
1369
1370 if let Some(probe_cfg) = state
1377 .config
1378 .routing
1379 .as_ref()
1380 .and_then(|c| c.escalation.probe)
1381 && u01(Uuid::now_v7().as_u128()) < probe_cfg.sample_rate
1382 {
1383 let probe_rung = (start_rung as usize).min(route.ladder.len().saturating_sub(1));
1385 if let Some(probe_model_str) = route.ladder.get(probe_rung).cloned() {
1386 let probe_provider = ModelRef::parse(&probe_model_str)
1387 .ok()
1388 .and_then(|m| state.providers.get(&m.provider));
1389
1390 if let Some(probe_provider) = probe_provider {
1391 let mut join_set = tokio::task::JoinSet::new();
1394 for _ in 0..probe_cfg.k {
1395 let mut probe_req = base_request.clone();
1396 probe_req.model = probe_model_str.clone();
1397 let probe_auth = auth.clone();
1398 let prov = probe_provider.clone();
1399 join_set.spawn(async move { prov.complete(&probe_req, &probe_auth).await });
1400 }
1401
1402 let fail_closed_owned: std::collections::HashSet<String> = gates
1405 .iter()
1406 .filter(|g| g.abstain_fails_closed())
1407 .map(|g| g.id().to_owned())
1408 .collect();
1409
1410 let mut gate_pass_count = 0u32;
1411 let mut probe_cost_usd = 0.0f64;
1412
1413 while let Some(task_result) = join_set.join_next().await {
1414 let Ok(Ok(probe_resp)) = task_result else {
1415 continue; };
1417 probe_cost_usd += state
1418 .config
1419 .prices
1420 .cost_usd(
1421 &probe_model_str,
1422 probe_resp.in_tokens,
1423 probe_resp.out_tokens,
1424 )
1425 .unwrap_or(0.0);
1426
1427 let mut probe_gate_req = base_request.clone();
1428 probe_gate_req.model = probe_model_str.clone();
1429
1430 let mut probe_gate_results = Vec::with_capacity(gates.len());
1433 for g in &gates {
1434 if !state.gate_health.enabled(&trace.tenant_id, g.id()) {
1436 continue;
1437 }
1438 let r = g.evaluate(&probe_gate_req, &probe_resp).await;
1439 probe_gate_results.push(r);
1441 }
1442
1443 let fail_closed_refs: std::collections::HashSet<&str> =
1444 fail_closed_owned.iter().map(|s| s.as_str()).collect();
1445 let verdict = aggregate_with_policy(&probe_gate_results, &fail_closed_refs);
1446 let passes = match serve_threshold {
1449 None => verdict == Verdict::Pass,
1450 Some(t) => crate::calibrate::gate_score(&probe_gate_results, verdict) >= t,
1451 };
1452 if passes {
1453 gate_pass_count += 1;
1454 }
1455 }
1456
1457 let regime = ProbeRegime::classify(gate_pass_count, probe_cfg.k);
1458 let regime_label = match regime {
1459 ProbeRegime::ConfidentPass => "confident_pass",
1460 ProbeRegime::ConfidentFail => "confident_fail",
1461 ProbeRegime::Ambiguous => "ambiguous",
1462 };
1463 metrics::counter!(
1464 "firstpass_probe_regime_total",
1465 "regime" => regime_label
1466 )
1467 .increment(1);
1468 metrics::gauge!("firstpass_probe_cost_usd_total").increment(probe_cost_usd);
1469 trace.probe = Some(ProbeSignal {
1470 k: probe_cfg.k,
1471 gate_pass_count,
1472 regime,
1473 probe_cost_usd,
1474 });
1475 }
1476 }
1477 }
1478 offer_trace(&state.traces, state.spill.as_ref(), trace);
1482
1483 match outcome {
1484 EngineOutcome::Served(resp) => Ok(resp),
1485 EngineOutcome::Failed(msg) => Err(ProxyError::Engine(msg)),
1486 }
1487}
1488
1489#[allow(clippy::too_many_arguments)]
1492async fn enforce_pipeline(
1493 state: &AppState,
1494 headers: &HeaderMap,
1495 body: &Bytes,
1496 features: Features,
1497 route: &Route,
1498 route_ix: usize,
1500 session_header: Option<String>,
1501 tenant: String,
1502 routing_mode: RoutingMode,
1503) -> Result<Value, ProxyError> {
1504 let Some(base_request) = parse_model_request(body) else {
1505 return Err(ProxyError::BadRequest(
1506 "request body is not a valid Anthropic Messages request".to_owned(),
1507 ));
1508 };
1509 let auth = Auth::from_headers(headers);
1510 let resp = enforce_pipeline_inner(
1511 state,
1512 body,
1513 base_request,
1514 auth,
1515 features,
1516 route,
1517 session_header,
1518 tenant,
1519 "anthropic.messages",
1520 routing_mode,
1521 route_ix,
1522 )
1523 .await?;
1524 Ok(anthropic_response_json(&resp))
1525}
1526
1527#[allow(clippy::too_many_arguments)]
1530async fn enforce_pipeline_openai(
1531 state: &AppState,
1532 headers: &HeaderMap,
1533 body: &Bytes,
1534 features: Features,
1535 route: &Route,
1536 route_ix: usize,
1538 session_header: Option<String>,
1539 tenant: String,
1540 routing_mode: RoutingMode,
1541) -> Result<Value, ProxyError> {
1542 let providers = &state.providers;
1544 let all_openai = route.ladder.iter().all(|rung| {
1545 let pid = rung.split('/').next().unwrap_or_default();
1546 providers
1547 .get(pid)
1548 .is_some_and(|p| p.carries_structured_verbatim(Dialect::Openai))
1549 });
1550 let Some(base_request) = parse_openai_request(body, all_openai) else {
1551 return Err(ProxyError::BadRequest(
1552 "request body is not a valid OpenAI Chat Completions request".to_owned(),
1553 ));
1554 };
1555 let auth = Auth::from_headers(headers);
1556 let resp = enforce_pipeline_inner(
1557 state,
1558 body,
1559 base_request,
1560 auth,
1561 features,
1562 route,
1563 session_header,
1564 tenant,
1565 "openai.chat_completions",
1566 routing_mode,
1567 route_ix,
1568 )
1569 .await?;
1570 Ok(openai_response_json(&resp))
1571}
1572
1573const SSE_KEEPALIVE_EVERY: Duration = Duration::from_secs(5);
1575
1576fn sse_keepalive_response(
1585 rx: tokio::sync::oneshot::Receiver<Result<Value, ProxyError>>,
1586 format_message: fn(&Value) -> String,
1587) -> Response {
1588 let mut ticks = tokio::time::interval(SSE_KEEPALIVE_EVERY);
1589 ticks.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
1590 ticks.reset(); let stream = KeepaliveStream {
1592 rx: Some(rx),
1593 ticks,
1594 format_message,
1595 };
1596 (
1597 axum::http::StatusCode::OK,
1598 [(
1599 axum::http::header::CONTENT_TYPE,
1600 "text/event-stream; charset=utf-8",
1601 )],
1602 axum::body::Body::from_stream(stream),
1603 )
1604 .into_response()
1605}
1606
1607struct KeepaliveStream {
1610 rx: Option<tokio::sync::oneshot::Receiver<Result<Value, ProxyError>>>,
1612 ticks: tokio::time::Interval,
1613 format_message: fn(&Value) -> String,
1615}
1616
1617impl futures_core::Stream for KeepaliveStream {
1618 type Item = Result<Bytes, std::convert::Infallible>;
1619
1620 fn poll_next(
1621 mut self: std::pin::Pin<&mut Self>,
1622 cx: &mut std::task::Context<'_>,
1623 ) -> std::task::Poll<Option<Self::Item>> {
1624 use std::task::Poll;
1625 let Some(rx) = self.rx.as_mut() else {
1626 return Poll::Ready(None); };
1628 if let Poll::Ready(out) = std::pin::Pin::new(rx).poll(cx) {
1629 let fmt = self.format_message;
1630 let frame = match out {
1631 Ok(Ok(message)) => fmt(&message),
1632 Ok(Err(e)) => sse_error_event(&e),
1633 Err(_) => sse_error_event(&ProxyError::Internal(
1634 "enforce pipeline task dropped".to_owned(),
1635 )),
1636 };
1637 self.rx = None;
1638 return Poll::Ready(Some(Ok(Bytes::from(frame))));
1639 }
1640 if self.ticks.poll_tick(cx).is_ready() {
1641 return Poll::Ready(Some(Ok(Bytes::from_static(b": firstpass routing\n\n"))));
1642 }
1643 Poll::Pending
1644 }
1645}
1646
1647fn sse_error_event(e: &ProxyError) -> String {
1650 let mut out = String::new();
1651 sse_event(
1652 &mut out,
1653 "error",
1654 &serde_json::json!({
1655 "type": "error",
1656 "error": { "type": "api_error", "message": e.client_message() }
1657 }),
1658 );
1659 out
1660}
1661
1662fn parse_model_request(body: &[u8]) -> Option<ModelRequest> {
1671 let json: Value = serde_json::from_slice(body).ok()?;
1672 let raw = json.clone();
1673 let messages_json = json.get("messages")?.as_array()?;
1674 let messages = messages_json
1675 .iter()
1676 .map(|m| ChatMessage {
1677 role: m
1678 .get("role")
1679 .and_then(Value::as_str)
1680 .unwrap_or("user")
1681 .to_owned(),
1682 content: m
1683 .get("content")
1684 .cloned()
1685 .unwrap_or_else(|| Value::String(String::new())),
1686 })
1687 .collect();
1688 let system = json
1689 .get("system")
1690 .and_then(Value::as_str)
1691 .map(str::to_owned);
1692 let max_tokens = json
1693 .get("max_tokens")
1694 .and_then(Value::as_u64)
1695 .and_then(|n| u32::try_from(n).ok())
1696 .unwrap_or(1024);
1697 let tools = json.get("tools").cloned().unwrap_or(Value::Null);
1698 Some(ModelRequest {
1699 model: json
1700 .get("model")
1701 .and_then(Value::as_str)
1702 .unwrap_or_default()
1703 .to_owned(),
1704 system,
1705 messages,
1706 max_tokens,
1707 tools,
1708 raw,
1709 })
1710}
1711
1712fn anthropic_response_json(resp: &ModelResponse) -> Value {
1722 let content = resp
1723 .raw
1724 .get("content")
1725 .filter(|c| c.is_array())
1726 .cloned()
1727 .unwrap_or_else(|| serde_json::json!([{ "type": "text", "text": resp.text }]));
1728 serde_json::json!({
1729 "id": format!("msg_{}", Uuid::now_v7()),
1730 "type": "message",
1731 "role": "assistant",
1732 "model": resp.model,
1733 "content": content,
1734 "usage": { "input_tokens": resp.in_tokens, "output_tokens": resp.out_tokens },
1735 })
1736}
1737
1738fn sse_event(out: &mut String, event: &str, data: &Value) {
1740 out.push_str("event: ");
1741 out.push_str(event);
1742 out.push_str("\ndata: ");
1743 out.push_str(&data.to_string());
1744 out.push_str("\n\n");
1745}
1746
1747fn anthropic_sse_from_message(message: &Value) -> String {
1754 let mut out = String::new();
1755
1756 let mut start_msg = message.clone();
1758 start_msg["content"] = Value::Array(Vec::new());
1759 sse_event(
1760 &mut out,
1761 "message_start",
1762 &serde_json::json!({ "type": "message_start", "message": start_msg }),
1763 );
1764
1765 let empty = Vec::new();
1766 let blocks = message
1767 .get("content")
1768 .and_then(Value::as_array)
1769 .unwrap_or(&empty);
1770 for (i, block) in blocks.iter().enumerate() {
1771 match block.get("type").and_then(Value::as_str) {
1772 Some("tool_use") => {
1773 let mut shell = block.clone();
1775 shell["input"] = serde_json::json!({});
1776 sse_event(
1777 &mut out,
1778 "content_block_start",
1779 &serde_json::json!({ "type": "content_block_start", "index": i, "content_block": shell }),
1780 );
1781 let input_json = block
1782 .get("input")
1783 .map_or_else(|| "{}".to_owned(), std::string::ToString::to_string);
1784 sse_event(
1785 &mut out,
1786 "content_block_delta",
1787 &serde_json::json!({ "type": "content_block_delta", "index": i,
1788 "delta": { "type": "input_json_delta", "partial_json": input_json } }),
1789 );
1790 }
1791 _ => {
1792 let text = block.get("text").and_then(Value::as_str).unwrap_or("");
1794 sse_event(
1795 &mut out,
1796 "content_block_start",
1797 &serde_json::json!({ "type": "content_block_start", "index": i,
1798 "content_block": { "type": "text", "text": "" } }),
1799 );
1800 sse_event(
1801 &mut out,
1802 "content_block_delta",
1803 &serde_json::json!({ "type": "content_block_delta", "index": i,
1804 "delta": { "type": "text_delta", "text": text } }),
1805 );
1806 }
1807 }
1808 sse_event(
1809 &mut out,
1810 "content_block_stop",
1811 &serde_json::json!({ "type": "content_block_stop", "index": i }),
1812 );
1813 }
1814
1815 let out_tokens = message
1816 .pointer("/usage/output_tokens")
1817 .cloned()
1818 .unwrap_or_else(|| Value::from(0));
1819 sse_event(
1820 &mut out,
1821 "message_delta",
1822 &serde_json::json!({ "type": "message_delta", "delta": { "stop_reason": "end_turn" },
1823 "usage": { "output_tokens": out_tokens } }),
1824 );
1825 sse_event(
1826 &mut out,
1827 "message_stop",
1828 &serde_json::json!({ "type": "message_stop" }),
1829 );
1830 out
1831}
1832
1833async fn observe_passthrough(
1835 state: AppState,
1836 headers: HeaderMap,
1837 body: Bytes,
1838 session_header: Option<String>,
1839 tenant: String,
1840) -> Response {
1841 if is_stream_request(&body) {
1843 return observe_stream(state, headers, body, session_header, tenant).await;
1844 }
1845 let start = Instant::now();
1846 let result = forward_anthropic(
1847 &state.http,
1848 &state.config.upstream_anthropic,
1849 &headers,
1850 body.clone(),
1851 )
1852 .await;
1853 let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
1854
1855 match result {
1856 Ok((status, resp_headers, resp_body)) => {
1857 spawn_trace(
1861 &state,
1862 body,
1863 Some(resp_body.clone()),
1864 latency_ms,
1865 session_header,
1866 tenant,
1867 );
1868 (status, resp_headers, resp_body).into_response()
1869 }
1870 Err(err) => {
1871 spawn_trace(&state, body, None, latency_ms, session_header, tenant);
1872 err.into_response()
1873 }
1874 }
1875}
1876
1877async fn observe_stream(
1886 state: AppState,
1887 headers: HeaderMap,
1888 body: Bytes,
1889 session_header: Option<String>,
1890 tenant: String,
1891) -> Response {
1892 let start = Instant::now();
1893 let result = forward_anthropic_streaming(
1894 &state.http,
1895 &state.config.upstream_anthropic,
1896 &headers,
1897 body.clone(),
1898 )
1899 .await;
1900 let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
1901
1902 match result {
1903 Ok((status, resp_headers, response)) => {
1904 spawn_stream_trace(&state, body, latency_ms, session_header, tenant);
1905 let stream_body = Body::from_stream(response.bytes_stream());
1906 (status, resp_headers, stream_body).into_response()
1907 }
1908 Err(err) => {
1909 spawn_trace(&state, body, None, latency_ms, session_header, tenant);
1910 err.into_response()
1911 }
1912 }
1913}
1914
1915fn spawn_stream_trace(
1917 state: &AppState,
1918 req_body: Bytes,
1919 latency_ms: u64,
1920 session_header: Option<String>,
1921 tenant: String,
1922) {
1923 let config = state.config.clone();
1924 let traces = state.traces.clone();
1925 let spill = state.spill.clone();
1926 tokio::spawn(async move {
1927 let mut trace =
1928 build_stream_trace(&config, &req_body, latency_ms, session_header.as_deref());
1929 trace.tenant_id = tenant;
1931 offer_trace(&traces, spill.as_ref(), trace);
1932 });
1933}
1934
1935fn spawn_trace(
1940 state: &AppState,
1941 req_body: Bytes,
1942 resp_body: Option<Bytes>,
1943 latency_ms: u64,
1944 session_header: Option<String>,
1945 tenant: String,
1946) {
1947 let config = state.config.clone();
1948 let traces = state.traces.clone();
1949 let spill = state.spill.clone();
1950 tokio::spawn(async move {
1951 let mut trace = match resp_body {
1952 Some(resp) => build_trace(
1953 &config,
1954 &req_body,
1955 &resp,
1956 latency_ms,
1957 session_header.as_deref(),
1958 ),
1959 None => build_error_trace(&config, &req_body, latency_ms, session_header.as_deref()),
1960 };
1961 trace.tenant_id = tenant;
1963 offer_trace(&traces, spill.as_ref(), trace);
1964 });
1965}
1966
1967fn session_id(session_header: Option<&str>, trace_id: Uuid) -> String {
1969 session_header
1970 .map(str::to_owned)
1971 .unwrap_or_else(|| trace_id.to_string())
1972}
1973
1974fn prompt_hash(salt: &str, body: &[u8]) -> String {
1977 let mut salted = Vec::with_capacity(salt.len() + body.len());
1978 salted.extend_from_slice(salt.as_bytes());
1979 salted.extend_from_slice(body);
1980 sha256_hex(&salted)
1981}
1982
1983fn request_features(body: &[u8]) -> (Option<String>, u32, bool) {
1987 let Ok(json) = serde_json::from_slice::<Value>(body) else {
1988 return (None, 0, false);
1989 };
1990 let model = json.get("model").and_then(Value::as_str).map(str::to_owned);
1991 let tool_count = json
1992 .get("tools")
1993 .and_then(Value::as_array)
1994 .map_or(0, |tools| u32::try_from(tools.len()).unwrap_or(u32::MAX));
1995 let has_images = json
1996 .get("messages")
1997 .and_then(Value::as_array)
1998 .is_some_and(|messages| messages.iter().any(message_has_image));
1999 (model, tool_count, has_images)
2000}
2001
2002fn message_has_image(message: &Value) -> bool {
2004 message
2005 .get("content")
2006 .and_then(Value::as_array)
2007 .is_some_and(|blocks| {
2008 blocks
2009 .iter()
2010 .any(|block| block.get("type").and_then(Value::as_str) == Some("image"))
2011 })
2012}
2013
2014fn response_usage(body: &[u8]) -> (Option<String>, u64, u64) {
2017 let Ok(json) = serde_json::from_slice::<Value>(body) else {
2018 return (None, 0, 0);
2019 };
2020 let model = json.get("model").and_then(Value::as_str).map(str::to_owned);
2021 let in_tokens = json
2022 .pointer("/usage/input_tokens")
2023 .and_then(Value::as_u64)
2024 .unwrap_or(0);
2025 let out_tokens = json
2026 .pointer("/usage/output_tokens")
2027 .and_then(Value::as_u64)
2028 .unwrap_or(0);
2029 (model, in_tokens, out_tokens)
2030}
2031
2032fn build_trace(
2034 config: &ProxyConfig,
2035 req_body: &Bytes,
2036 resp_body: &Bytes,
2037 latency_ms: u64,
2038 session_header: Option<&str>,
2039) -> Trace {
2040 let (req_model, tool_count, has_images) = request_features(req_body);
2041 let (resp_model, in_tokens, out_tokens) = response_usage(resp_body);
2042 let model = resp_model
2043 .or(req_model)
2044 .unwrap_or_else(|| "unknown".to_owned());
2045
2046 let cost_usd = config
2047 .prices
2048 .cost_usd(&format!("anthropic/{model}"), in_tokens, out_tokens)
2049 .unwrap_or(0.0);
2050
2051 let attempt = Attempt {
2052 rung: 0,
2053 model,
2054 provider: "anthropic".to_owned(),
2055 in_tokens,
2056 out_tokens,
2057 cost_usd,
2058 latency_ms,
2059 gates: Vec::new(),
2060 verdict: Verdict::Pass,
2061 };
2062
2063 let mut trace = base_trace(config, req_body, latency_ms, session_header);
2064 trace.request.features.prompt_token_bucket = token_bucket(in_tokens);
2065 trace.request.features.tool_count = tool_count;
2066 trace.request.features.has_images = has_images;
2067 trace.attempts.push(attempt);
2068 trace.final_ = FinalOutcome {
2069 served_rung: Some(0),
2070 served_from: ServedFrom::Attempt,
2071 total_cost_usd: cost_usd,
2072 gate_cost_usd: 0.0,
2073 total_latency_ms: latency_ms,
2074 escalations: 0,
2075 counterfactual_baseline_usd: cost_usd,
2076 savings_usd: 0.0,
2077 };
2078 trace.recompute_savings();
2079 trace
2080}
2081
2082fn build_stream_trace(
2086 config: &ProxyConfig,
2087 req_body: &Bytes,
2088 latency_ms: u64,
2089 session_header: Option<&str>,
2090) -> Trace {
2091 let (req_model, tool_count, has_images) = request_features(req_body);
2092 let model = req_model.unwrap_or_else(|| "unknown".to_owned());
2093
2094 let attempt = Attempt {
2095 rung: 0,
2096 model,
2097 provider: "anthropic".to_owned(),
2098 in_tokens: 0,
2099 out_tokens: 0,
2100 cost_usd: 0.0,
2101 latency_ms,
2102 gates: Vec::new(),
2103 verdict: Verdict::Pass,
2104 };
2105
2106 let mut trace = base_trace(config, req_body, latency_ms, session_header);
2107 trace.request.features.tool_count = tool_count;
2108 trace.request.features.has_images = has_images;
2109 trace.attempts.push(attempt);
2110 trace.final_ = FinalOutcome {
2111 served_rung: Some(0),
2112 served_from: ServedFrom::Attempt,
2113 total_cost_usd: 0.0,
2114 gate_cost_usd: 0.0,
2115 total_latency_ms: latency_ms,
2116 escalations: 0,
2117 counterfactual_baseline_usd: 0.0,
2118 savings_usd: 0.0,
2119 };
2120 trace.recompute_savings();
2121 trace
2122}
2123
2124fn build_error_trace(
2128 config: &ProxyConfig,
2129 req_body: &Bytes,
2130 latency_ms: u64,
2131 session_header: Option<&str>,
2132) -> Trace {
2133 let (_, tool_count, has_images) = request_features(req_body);
2134 let mut trace = base_trace(config, req_body, latency_ms, session_header);
2135 trace.request.features.tool_count = tool_count;
2136 trace.request.features.has_images = has_images;
2137 trace.final_ = FinalOutcome {
2138 served_rung: None,
2139 served_from: ServedFrom::Error,
2140 total_cost_usd: 0.0,
2141 gate_cost_usd: 0.0,
2142 total_latency_ms: latency_ms,
2143 escalations: 0,
2144 counterfactual_baseline_usd: 0.0,
2145 savings_usd: 0.0,
2146 };
2147 trace.recompute_savings();
2148 trace
2149}
2150
2151fn base_trace(
2154 config: &ProxyConfig,
2155 req_body: &Bytes,
2156 latency_ms: u64,
2157 session_header: Option<&str>,
2158) -> Trace {
2159 let trace_id = Uuid::now_v7();
2160 let mut features = Features::new(TaskKind::Other);
2161 features.hour_bucket = hour_bucket(jiff::Timestamp::now());
2162
2163 Trace {
2164 trace_id,
2165 prev_hash: GENESIS_HASH.to_owned(),
2166 tenant_id: config.tenant_id.clone(),
2167 session_id: session_id(session_header, trace_id),
2168 ts: jiff::Timestamp::now(),
2169 mode: Mode::Observe,
2170 policy: PolicyRef {
2171 id: "observe-passthrough@v0".to_owned(),
2172 explore: false,
2173 propensity: None,
2174 mode_profile: None,
2175 },
2176 request: RequestInfo {
2177 api: "anthropic.messages".to_owned(),
2178 prompt_hash: prompt_hash(&config.prompt_salt, req_body),
2179 features,
2180 },
2181 attempts: Vec::new(),
2182 deferred: Vec::new(),
2183 final_: FinalOutcome {
2184 served_rung: None,
2185 served_from: ServedFrom::Error,
2186 total_cost_usd: 0.0,
2187 gate_cost_usd: 0.0,
2188 total_latency_ms: latency_ms,
2189 escalations: 0,
2190 counterfactual_baseline_usd: 0.0,
2191 savings_usd: 0.0,
2192 },
2193 probe: None,
2194 rollout: None,
2195 shadow: None,
2196 route_ix: None,
2197 predicted_pass: None,
2198 elastic: None,
2199 }
2200}
2201
2202fn openai_messages_have_tool_calls(body: &[u8]) -> bool {
2207 serde_json::from_slice::<Value>(body)
2208 .ok()
2209 .and_then(|json| {
2210 json.get("messages").and_then(Value::as_array).map(|msgs| {
2211 msgs.iter().any(|m| {
2212 m.get("tool_calls").is_some()
2213 || m.get("role").and_then(Value::as_str) == Some("tool")
2214 })
2215 })
2216 })
2217 .unwrap_or(false)
2218}
2219
2220fn openai_has_http_images(body: &[u8]) -> bool {
2224 serde_json::from_slice::<Value>(body)
2225 .ok()
2226 .and_then(|json| {
2227 json.get("messages").and_then(Value::as_array).map(|msgs| {
2228 msgs.iter().any(|m| {
2229 m.get("content")
2230 .and_then(Value::as_array)
2231 .is_some_and(|parts| {
2232 parts.iter().any(|p| {
2233 p.get("type").and_then(Value::as_str) == Some("image_url")
2234 && p.pointer("/image_url/url")
2235 .and_then(Value::as_str)
2236 .is_some_and(|u| {
2237 u.starts_with("http://") || u.starts_with("https://")
2238 })
2239 })
2240 })
2241 })
2242 })
2243 })
2244 .unwrap_or(false)
2245}
2246
2247fn openai_messages_have_images(body: &[u8]) -> bool {
2250 serde_json::from_slice::<Value>(body)
2251 .ok()
2252 .and_then(|json| {
2253 json.get("messages").and_then(Value::as_array).map(|msgs| {
2254 msgs.iter().any(|m| {
2255 m.get("content")
2256 .and_then(Value::as_array)
2257 .is_some_and(|parts| {
2258 parts
2259 .iter()
2260 .any(|p| p.get("type").and_then(Value::as_str) == Some("image_url"))
2261 })
2262 })
2263 })
2264 })
2265 .unwrap_or(false)
2266}
2267
2268fn extract_openai_features(headers: &HeaderMap, body: &[u8]) -> Features {
2271 let Ok(json) = serde_json::from_slice::<Value>(body) else {
2272 let mut f = Features::new(TaskKind::Other);
2273 f.hour_bucket = hour_bucket(jiff::Timestamp::now());
2274 return f;
2275 };
2276 let tool_count = json
2277 .get("tools")
2278 .and_then(Value::as_array)
2279 .map_or(0, |tools| u32::try_from(tools.len()).unwrap_or(u32::MAX));
2280 let has_images = openai_messages_have_images(body);
2281 let mut f = Features::new(TaskKind::Other);
2282 f.agent = header_str(headers, AGENT_HEADER);
2283 f.subagent = header_str(headers, SUBAGENT_HEADER);
2284 f.tool_count = tool_count;
2285 f.has_images = has_images;
2286 f.prompt_token_bucket = token_bucket(body.len() as u64);
2287 f.hour_bucket = hour_bucket(jiff::Timestamp::now());
2288 f
2289}
2290
2291fn parse_data_url(url: &str) -> Option<(&str, &str)> {
2295 let rest = url.strip_prefix("data:")?;
2296 let (meta, data) = rest.split_once(',')?;
2297 let media_type = meta.strip_suffix(";base64")?;
2298 Some((media_type, data))
2299}
2300
2301fn translate_openai_user_content(content: &Value) -> Option<Value> {
2304 match content {
2305 Value::String(_) => Some(content.clone()),
2307 Value::Array(parts) => {
2308 let mut blocks: Vec<Value> = Vec::with_capacity(parts.len());
2309 for part in parts {
2310 match part.get("type").and_then(Value::as_str) {
2311 Some("text") => {
2312 let text = part.get("text").and_then(Value::as_str).unwrap_or("");
2313 blocks.push(serde_json::json!({ "type": "text", "text": text }));
2314 }
2315 Some("image_url") => {
2316 let url = part.pointer("/image_url/url").and_then(Value::as_str)?;
2317 if url.starts_with("http://") || url.starts_with("https://") {
2318 return None; }
2320 let (media_type, data) = parse_data_url(url)?;
2322 blocks.push(serde_json::json!({
2323 "type": "image",
2324 "source": { "type": "base64", "media_type": media_type, "data": data }
2325 }));
2326 }
2327 _ => {} }
2329 }
2330 Some(Value::Array(blocks))
2331 }
2332 _ => Some(Value::String(String::new())),
2333 }
2334}
2335
2336fn translate_openai_tools(tools: &Value) -> Value {
2340 let Some(arr) = tools.as_array() else {
2341 return Value::Null;
2342 };
2343 let converted: Vec<Value> = arr
2344 .iter()
2345 .map(|tool| {
2346 let func = tool.get("function").unwrap_or(&Value::Null);
2347 let mut out = serde_json::json!({
2348 "name": func.get("name").cloned().unwrap_or(Value::String(String::new())),
2349 "input_schema": func.get("parameters").cloned()
2350 .unwrap_or_else(|| serde_json::json!({ "type": "object" })),
2351 });
2352 if let Some(desc) = func.get("description") {
2353 out["description"] = desc.clone();
2354 }
2355 out
2356 })
2357 .collect();
2358 Value::Array(converted)
2359}
2360
2361fn translate_openai_tool_choice(tc: &Value) -> Value {
2363 match tc {
2364 Value::String(s) => match s.as_str() {
2365 "auto" => serde_json::json!({ "type": "auto" }),
2366 "required" => serde_json::json!({ "type": "any" }),
2367 _ => serde_json::json!({ "type": "auto" }),
2369 },
2370 Value::Object(_) => {
2371 if tc.get("type").and_then(Value::as_str) == Some("function") {
2373 let name = tc.pointer("/function/name").cloned().unwrap_or(Value::Null);
2374 serde_json::json!({ "type": "tool", "name": name })
2375 } else {
2376 serde_json::json!({ "type": "auto" })
2377 }
2378 }
2379 _ => serde_json::json!({ "type": "auto" }),
2380 }
2381}
2382
2383pub fn parse_openai_request(body: &[u8], carry_raw: bool) -> Option<ModelRequest> {
2396 let json: Value = serde_json::from_slice(body).ok()?;
2397 let raw = if carry_raw { json.clone() } else { Value::Null };
2398
2399 let messages_json = json.get("messages")?.as_array()?;
2400
2401 let mut system: Option<String> = None;
2402 let mut messages: Vec<ChatMessage> = Vec::with_capacity(messages_json.len());
2403 let mut tools = Value::Null;
2404 let mut tool_choice_override: Option<Value> = None;
2405
2406 for msg in messages_json {
2407 let role = msg.get("role").and_then(Value::as_str).unwrap_or("user");
2408 match role {
2409 "system" => {
2410 if let Some(s) = msg.get("content").and_then(Value::as_str) {
2415 system = Some(s.to_owned());
2416 }
2417 }
2418 "user" => {
2419 let content_val = msg.get("content").unwrap_or(&Value::Null);
2420 let translated = translate_openai_user_content(content_val)?;
2421 messages.push(ChatMessage {
2422 role: "user".to_owned(),
2423 content: translated,
2424 });
2425 }
2426 "assistant" => {
2427 if let Some(tc_arr) = msg.get("tool_calls").and_then(Value::as_array) {
2428 let mut blocks: Vec<Value> = Vec::new();
2430 if let Some(text) = msg.get("content").and_then(Value::as_str)
2432 && !text.is_empty()
2433 {
2434 blocks.push(serde_json::json!({ "type": "text", "text": text }));
2435 }
2436 for tc in tc_arr {
2437 let id = tc.get("id").and_then(Value::as_str).unwrap_or("");
2438 let func = tc.get("function").unwrap_or(&Value::Null);
2439 let name = func.get("name").and_then(Value::as_str).unwrap_or("");
2440 let args_str = func
2441 .get("arguments")
2442 .and_then(Value::as_str)
2443 .unwrap_or("{}");
2444 let input: Value = serde_json::from_str(args_str)
2445 .unwrap_or_else(|_| serde_json::json!({}));
2446 blocks.push(serde_json::json!({
2447 "type": "tool_use",
2448 "id": id,
2449 "name": name,
2450 "input": input,
2451 }));
2452 }
2453 messages.push(ChatMessage {
2454 role: "assistant".to_owned(),
2455 content: Value::Array(blocks),
2456 });
2457 } else {
2458 let content = msg
2460 .get("content")
2461 .cloned()
2462 .unwrap_or_else(|| Value::String(String::new()));
2463 messages.push(ChatMessage {
2464 role: "assistant".to_owned(),
2465 content,
2466 });
2467 }
2468 }
2469 "tool" => {
2470 let tool_call_id = msg
2472 .get("tool_call_id")
2473 .and_then(Value::as_str)
2474 .unwrap_or("");
2475 let content = msg
2476 .get("content")
2477 .cloned()
2478 .unwrap_or_else(|| Value::String(String::new()));
2479 let result_block = serde_json::json!({
2480 "type": "tool_result",
2481 "tool_use_id": tool_call_id,
2482 "content": content,
2483 });
2484 messages.push(ChatMessage {
2485 role: "user".to_owned(),
2486 content: Value::Array(vec![result_block]),
2487 });
2488 }
2489 _ => {} }
2491 }
2492
2493 if !carry_raw {
2495 if let Some(t) = json.get("tools") {
2496 tools = translate_openai_tools(t);
2497 }
2498 if let Some(tc) = json.get("tool_choice") {
2499 tool_choice_override = Some(translate_openai_tool_choice(tc));
2500 }
2501 } else {
2502 tools = json.get("tools").cloned().unwrap_or(Value::Null);
2503 }
2504
2505 let max_tokens = json
2506 .get("max_tokens")
2507 .and_then(Value::as_u64)
2508 .and_then(|n| u32::try_from(n).ok())
2509 .unwrap_or(1024);
2510
2511 let model = json
2512 .get("model")
2513 .and_then(Value::as_str)
2514 .unwrap_or_default()
2515 .to_owned();
2516
2517 let _ = tool_choice_override; Some(ModelRequest {
2528 model,
2529 system,
2530 messages,
2531 max_tokens,
2532 tools,
2533 raw,
2534 })
2535}
2536
2537fn extract_openai_content_and_tools(raw: &Value, text: &str) -> (Value, Option<Value>) {
2544 if let Some(blocks) = raw.get("content").and_then(Value::as_array) {
2546 let mut text_parts: Vec<&str> = Vec::new();
2547 let mut tool_calls: Vec<Value> = Vec::new();
2548 for block in blocks {
2549 match block.get("type").and_then(Value::as_str) {
2550 Some("text") => {
2551 if let Some(t) = block.get("text").and_then(Value::as_str) {
2552 text_parts.push(t);
2553 }
2554 }
2555 Some("tool_use") => {
2556 let id = block.get("id").and_then(Value::as_str).unwrap_or("");
2557 let name = block.get("name").and_then(Value::as_str).unwrap_or("");
2558 let input_str = block
2559 .get("input")
2560 .map_or_else(|| "{}".to_owned(), std::string::ToString::to_string);
2561 tool_calls.push(serde_json::json!({
2562 "id": id,
2563 "type": "function",
2564 "function": { "name": name, "arguments": input_str },
2565 }));
2566 }
2567 _ => {}
2568 }
2569 }
2570 let content_text = if tool_calls.is_empty() || !text_parts.is_empty() {
2571 Value::String(text_parts.join(""))
2572 } else {
2573 Value::Null };
2575 let tc = if tool_calls.is_empty() {
2576 None
2577 } else {
2578 Some(Value::Array(tool_calls))
2579 };
2580 return (content_text, tc);
2581 }
2582
2583 if let Some(msg) = raw.pointer("/choices/0/message") {
2585 let content = msg
2586 .get("content")
2587 .cloned()
2588 .unwrap_or(Value::String(text.to_owned()));
2589 let tc = msg.get("tool_calls").cloned();
2590 return (content, tc);
2591 }
2592
2593 (Value::String(text.to_owned()), None)
2595}
2596
2597fn openai_response_json(resp: &ModelResponse) -> Value {
2600 let (content_text, tool_calls) = extract_openai_content_and_tools(&resp.raw, &resp.text);
2601 let finish_reason = if tool_calls.is_some() {
2602 "tool_calls"
2603 } else {
2604 "stop"
2605 };
2606 let mut message = serde_json::json!({
2607 "role": "assistant",
2608 "content": content_text,
2609 });
2610 if let Some(tc) = tool_calls {
2611 message["tool_calls"] = tc;
2612 }
2613 serde_json::json!({
2614 "id": format!("chatcmpl-{}", Uuid::now_v7()),
2615 "object": "chat.completion",
2616 "created": jiff::Timestamp::now().as_second(),
2617 "model": resp.model,
2618 "choices": [{
2619 "index": 0,
2620 "message": message,
2621 "finish_reason": finish_reason,
2622 }],
2623 "usage": {
2624 "prompt_tokens": resp.in_tokens,
2625 "completion_tokens": resp.out_tokens,
2626 "total_tokens": resp.in_tokens + resp.out_tokens,
2627 }
2628 })
2629}
2630
2631fn openai_sse_from_message(message: &Value) -> String {
2635 let id = message
2636 .get("id")
2637 .and_then(Value::as_str)
2638 .unwrap_or("chatcmpl-unknown")
2639 .to_owned();
2640 let created = message.get("created").cloned().unwrap_or(Value::from(0));
2641 let model = message
2642 .get("model")
2643 .and_then(Value::as_str)
2644 .unwrap_or("unknown");
2645
2646 let choices = message.get("choices").and_then(Value::as_array);
2647 let msg = choices
2648 .and_then(|c| c.first())
2649 .and_then(|c| c.get("message"));
2650 let content = msg
2651 .and_then(|m| m.get("content"))
2652 .cloned()
2653 .unwrap_or(Value::Null);
2654 let tool_calls = msg.and_then(|m| m.get("tool_calls")).cloned();
2655 let finish_reason = choices
2656 .and_then(|c| c.first())
2657 .and_then(|c| c.get("finish_reason"))
2658 .cloned()
2659 .unwrap_or_else(|| Value::String("stop".to_owned()));
2660
2661 let mut out = String::new();
2662 let chunk = |delta: Value| {
2663 serde_json::json!({
2664 "id": id,
2665 "object": "chat.completion.chunk",
2666 "created": created,
2667 "model": model,
2668 "choices": [{ "index": 0, "delta": delta, "finish_reason": Value::Null }]
2669 })
2670 };
2671
2672 let role_chunk = chunk(serde_json::json!({ "role": "assistant", "content": "" }));
2674 out.push_str("data: ");
2675 out.push_str(&role_chunk.to_string());
2676 out.push_str("\n\n");
2677
2678 if let Value::String(text) = &content
2680 && !text.is_empty()
2681 {
2682 let content_chunk = chunk(serde_json::json!({ "content": text }));
2683 out.push_str("data: ");
2684 out.push_str(&content_chunk.to_string());
2685 out.push_str("\n\n");
2686 }
2687
2688 if let Some(tc) = tool_calls {
2690 let tc_chunk = chunk(serde_json::json!({ "tool_calls": tc }));
2691 out.push_str("data: ");
2692 out.push_str(&tc_chunk.to_string());
2693 out.push_str("\n\n");
2694 }
2695
2696 let finish_chunk = serde_json::json!({
2698 "id": id,
2699 "object": "chat.completion.chunk",
2700 "created": created,
2701 "model": model,
2702 "choices": [{ "index": 0, "delta": {}, "finish_reason": finish_reason }]
2703 });
2704 out.push_str("data: ");
2705 out.push_str(&finish_chunk.to_string());
2706 out.push_str("\n\n");
2707
2708 out.push_str("data: [DONE]\n\n");
2709 out
2710}
2711
2712#[allow(clippy::too_many_arguments)]
2717async fn handle_enforce_openai(
2718 state: &AppState,
2719 headers: &HeaderMap,
2720 body: &Bytes,
2721 features: Features,
2722 route: &Route,
2723 route_ix: usize,
2726 session_header: Option<String>,
2727 tenant: String,
2728 routing_mode: RoutingMode,
2729) -> Response {
2730 if is_stream_request(body) {
2731 let (tx, rx) = tokio::sync::oneshot::channel::<Result<Value, ProxyError>>();
2732 let (state_c, headers_c, body_c, route_c) =
2733 (state.clone(), headers.clone(), body.clone(), route.clone());
2734 tokio::spawn(async move {
2735 let out = enforce_pipeline_openai(
2736 &state_c,
2737 &headers_c,
2738 &body_c,
2739 features,
2740 &route_c,
2741 route_ix,
2742 session_header,
2743 tenant,
2744 routing_mode,
2745 )
2746 .await;
2747 let _ = tx.send(out);
2748 });
2749 return sse_keepalive_response(rx, openai_sse_from_message);
2750 }
2751 match enforce_pipeline_openai(
2752 state,
2753 headers,
2754 body,
2755 features,
2756 route,
2757 route_ix,
2758 session_header,
2759 tenant,
2760 routing_mode,
2761 )
2762 .await
2763 {
2764 Ok(message) => (axum::http::StatusCode::OK, Json(message)).into_response(),
2765 Err(e) => e.into_response(),
2766 }
2767}
2768
2769async fn observe_passthrough_openai(
2775 state: AppState,
2776 headers: HeaderMap,
2777 body: Bytes,
2778 session_header: Option<String>,
2779 tenant: String,
2780) -> Response {
2781 if is_stream_request(&body) {
2782 return observe_stream_openai(state, headers, body, session_header, tenant).await;
2783 }
2784 let start = Instant::now();
2785 let result = forward_openai(
2786 &state.http,
2787 &state.config.upstream_openai,
2788 &headers,
2789 body.clone(),
2790 )
2791 .await;
2792 let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
2793 match result {
2794 Ok((status, resp_headers, resp_body)) => {
2795 spawn_trace(
2796 &state,
2797 body,
2798 Some(resp_body.clone()),
2799 latency_ms,
2800 session_header,
2801 tenant,
2802 );
2803 (status, resp_headers, resp_body).into_response()
2804 }
2805 Err(err) => {
2806 spawn_trace(&state, body, None, latency_ms, session_header, tenant);
2807 err.into_response()
2808 }
2809 }
2810}
2811
2812async fn observe_stream_openai(
2814 state: AppState,
2815 headers: HeaderMap,
2816 body: Bytes,
2817 session_header: Option<String>,
2818 tenant: String,
2819) -> Response {
2820 let start = Instant::now();
2821 let result = forward_openai_streaming(
2822 &state.http,
2823 &state.config.upstream_openai,
2824 &headers,
2825 body.clone(),
2826 )
2827 .await;
2828 let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
2829 match result {
2830 Ok((status, resp_headers, response)) => {
2831 spawn_stream_trace(&state, body, latency_ms, session_header, tenant);
2832 let stream_body = Body::from_stream(response.bytes_stream());
2833 (status, resp_headers, stream_body).into_response()
2834 }
2835 Err(err) => {
2836 spawn_trace(&state, body, None, latency_ms, session_header, tenant);
2837 err.into_response()
2838 }
2839 }
2840}
2841
2842async fn chat_completions(
2847 State(state): State<AppState>,
2848 Extension(TenantId(tenant)): Extension<TenantId>,
2849 headers: HeaderMap,
2850 body: Bytes,
2851) -> Response {
2852 let session_header = header_str(&headers, SESSION_HEADER);
2853
2854 if let Some(routing) = state.config.routing.as_ref() {
2855 let features = extract_openai_features(&headers, &body);
2856 if let Some(route) = routing
2857 .route_for(&features)
2858 .filter(|r| r.mode == Mode::Enforce && !r.ladder.is_empty())
2859 {
2860 let route_ix = routing
2862 .routes
2863 .iter()
2864 .position(|r| std::ptr::eq(r, route))
2865 .unwrap_or(0);
2866 let route = route.clone();
2867 let routing_mode = resolve_mode(&headers, &route, &state.config);
2869 if routing_mode == RoutingMode::Observe {
2871 return observe_passthrough_openai(state, headers, body, session_header, tenant)
2872 .await;
2873 }
2874 if enforce_can_handle(
2875 &features,
2876 &body,
2877 routing.escalation.enforce_structured,
2878 &route.ladder,
2879 &state.providers,
2880 Dialect::Openai,
2881 ) {
2882 return handle_enforce_openai(
2883 &state,
2884 &headers,
2885 &body,
2886 features,
2887 &route,
2888 route_ix,
2889 session_header,
2890 tenant,
2891 routing_mode,
2892 )
2893 .await;
2894 }
2895 tracing::info!(
2896 "enforce route matched but OpenAI structured request can't be routed faithfully (flag/ladder); serving via observe passthrough"
2897 );
2898 }
2899 }
2900 observe_passthrough_openai(state, headers, body, session_header, tenant).await
2901}
2902
2903#[cfg(test)]
2904mod tests {
2905 use bytes::Bytes;
2906
2907 use super::*;
2908
2909 fn test_config() -> ProxyConfig {
2910 ProxyConfig::from_lookup(|_| None).unwrap()
2911 }
2912
2913 #[test]
2914 fn build_trace_maps_request_and_response_fields() {
2915 let config = test_config();
2916 let req = Bytes::from_static(
2917 br#"{"model":"claude-haiku-4-5","tools":[{"name":"a"}],"messages":[]}"#,
2918 );
2919 let resp = Bytes::from_static(
2920 br#"{"model":"claude-haiku-4-5","usage":{"input_tokens":1200,"output_tokens":300}}"#,
2921 );
2922
2923 let trace = build_trace(&config, &req, &resp, 42, Some("sess-1"));
2924
2925 assert_eq!(trace.request.api, "anthropic.messages");
2926 assert_eq!(trace.session_id, "sess-1");
2927 assert_eq!(trace.attempts.len(), 1);
2928 let attempt = &trace.attempts[0];
2929 assert_eq!(attempt.model, "claude-haiku-4-5");
2930 assert_eq!(attempt.provider, "anthropic");
2931 assert_eq!(attempt.in_tokens, 1200);
2932 assert_eq!(attempt.out_tokens, 300);
2933 assert!(attempt.cost_usd > 0.0);
2934 assert_eq!(trace.request.features.tool_count, 1);
2935 assert!(!trace.request.features.has_images);
2936 assert_eq!(trace.final_.served_rung, Some(0));
2937 }
2938
2939 #[test]
2940 fn build_trace_falls_back_to_trace_id_session_when_header_absent() {
2941 let config = test_config();
2942 let req = Bytes::from_static(b"{}");
2943 let resp = Bytes::from_static(b"{}");
2944
2945 let trace = build_trace(&config, &req, &resp, 1, None);
2946
2947 assert_eq!(trace.session_id, trace.trace_id.to_string());
2948 }
2949
2950 #[test]
2951 fn build_error_trace_has_no_attempts_and_served_from_error() {
2952 let config = test_config();
2953 let req = Bytes::from_static(br#"{"model":"claude-haiku-4-5"}"#);
2954
2955 let trace = build_error_trace(&config, &req, 7, None);
2956
2957 assert!(trace.attempts.is_empty());
2958 assert_eq!(trace.final_.served_from, ServedFrom::Error);
2959 assert_eq!(trace.final_.served_rung, None);
2960 }
2961
2962 #[test]
2963 fn message_with_image_block_sets_has_images() {
2964 let req = br#"{"messages":[{"role":"user","content":[{"type":"image"}]}]}"#;
2965 let (_, _, has_images) = request_features(req);
2966 assert!(has_images);
2967 }
2968
2969 #[test]
2970 fn prompt_hash_never_contains_raw_prompt_text() {
2971 let hash = prompt_hash("salt", b"super secret prompt");
2972 assert!(!hash.contains("secret"));
2973 assert_eq!(hash.len(), 64);
2974 }
2975
2976 #[test]
2977 fn parse_model_request_preserves_content_verbatim_and_projects_text() {
2978 let body = br#"{"model":"m","system":"sys","max_tokens":50,
2979 "messages":[{"role":"user","content":[{"type":"text","text":"a"},{"type":"text","text":"b"}]},
2980 {"role":"assistant","content":"c"}]}"#;
2981 let req = parse_model_request(body).unwrap();
2982 assert_eq!(req.system.as_deref(), Some("sys"));
2983 assert_eq!(req.max_tokens, 50);
2984 assert_eq!(req.messages.len(), 2);
2985 assert_eq!(
2987 req.messages[0].content,
2988 serde_json::json!([{"type":"text","text":"a"},{"type":"text","text":"b"}])
2989 );
2990 assert_eq!(req.messages[1].content, Value::String("c".to_owned()));
2992 assert_eq!(req.messages[0].text_view(), "a\nb");
2994 assert_eq!(req.messages[1].text_view(), "c");
2995 }
2996
2997 #[test]
2998 fn tool_and_image_blocks_survive_the_request_round_trip() {
2999 let body = br#"{"model":"m","max_tokens":50,"messages":[
3001 {"role":"assistant","content":[{"type":"tool_use","id":"t1","name":"calc","input":{"x":1}}]},
3002 {"role":"user","content":[
3003 {"type":"tool_result","tool_use_id":"t1","content":"2"},
3004 {"type":"image","source":{"type":"base64","media_type":"image/png","data":"AA=="}}
3005 ]}]}"#;
3006 let req = parse_model_request(body).unwrap();
3007 let round_tripped = serde_json::to_value(&req.messages).unwrap();
3008 assert_eq!(
3009 round_tripped,
3010 serde_json::json!([
3011 {"role":"assistant","content":[{"type":"tool_use","id":"t1","name":"calc","input":{"x":1}}]},
3012 {"role":"user","content":[
3013 {"type":"tool_result","tool_use_id":"t1","content":"2"},
3014 {"type":"image","source":{"type":"base64","media_type":"image/png","data":"AA=="}}
3015 ]}
3016 ])
3017 );
3018 }
3019
3020 #[test]
3021 fn text_message_serializes_byte_identical_to_a_plain_string() {
3022 let m = ChatMessage::text("user", "hello");
3024 assert_eq!(
3025 serde_json::to_string(&m).unwrap(),
3026 r#"{"role":"user","content":"hello"}"#
3027 );
3028 }
3029
3030 #[test]
3031 fn parse_model_request_rejects_non_message_bodies() {
3032 assert!(parse_model_request(b"not json").is_none());
3033 assert!(parse_model_request(br#"{"no":"messages"}"#).is_none());
3034 }
3035
3036 use crate::provider::{MockProvider, ModelResponse, Provider, ProviderError, ProviderRegistry};
3039 use axum::extract::State;
3040 use std::collections::HashMap;
3041 use std::sync::Arc;
3042 use tokio::sync::mpsc;
3043
3044 fn model_resp(model: &str, text: &str) -> ModelResponse {
3045 ModelResponse {
3046 model: model.to_owned(),
3047 text: text.to_owned(),
3048 in_tokens: 1000,
3049 out_tokens: 400,
3050 raw: serde_json::Value::Null,
3051 }
3052 }
3053
3054 fn enforce_state(
3057 ladder: &[&str],
3058 gates: &[&str],
3059 outcomes: Vec<(&str, Result<ModelResponse, ProviderError>)>,
3060 ) -> (AppState, mpsc::Receiver<Trace>) {
3061 let toml = format!(
3062 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\nladder = [{}]\ngates = [{}]\n",
3063 ladder
3064 .iter()
3065 .map(|m| format!("\"{m}\""))
3066 .collect::<Vec<_>>()
3067 .join(", "),
3068 gates
3069 .iter()
3070 .map(|g| format!("\"{g}\""))
3071 .collect::<Vec<_>>()
3072 .join(", "),
3073 );
3074 let config = ProxyConfig::from_lookup(|k| match k {
3075 "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
3076 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
3077 _ => None,
3078 })
3079 .unwrap();
3080
3081 let mut outs = HashMap::new();
3082 for (model, out) in outcomes {
3083 outs.insert(model.to_owned(), out);
3084 }
3085 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
3086 map.insert(
3087 "anthropic".to_owned(),
3088 Arc::new(MockProvider::new("anthropic", outs)),
3089 );
3090 let providers = ProviderRegistry::from_map(map);
3091
3092 let (traces, rx) = mpsc::channel(64);
3093 let state = AppState {
3094 config: Arc::new(config),
3095 http: reqwest::Client::new(),
3096 providers,
3097 gate_health: Arc::new(GateHealthRegistry::new()),
3098 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
3099 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
3100 traces,
3101 adaptive: None,
3102 bandit: None,
3103 predictor: None,
3104 tenant_rate_limiter: None,
3105 spill: None,
3106 };
3107 (state, rx)
3108 }
3109
3110 fn user_body() -> Bytes {
3111 Bytes::from_static(
3112 br#"{"model":"ignored","max_tokens":64,"messages":[{"role":"user","content":"hi"}]}"#,
3113 )
3114 }
3115
3116 async fn body_json(resp: Response) -> Value {
3117 let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
3118 .await
3119 .unwrap();
3120 serde_json::from_slice(&bytes).unwrap()
3121 }
3122
3123 #[tokio::test]
3124 async fn mode_profile_stamped_on_trace_when_non_balanced() {
3125 let (state, mut rx) = enforce_state(
3126 &["anthropic/claude-haiku-4-5"],
3127 &["non-empty"],
3128 vec![(
3129 "anthropic/claude-haiku-4-5",
3130 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
3131 )],
3132 );
3133 let mut headers = HeaderMap::new();
3134 headers.insert("x-firstpass-mode", "quality".parse().unwrap());
3135 let resp = messages(
3136 State(state),
3137 Extension(TenantId("default".to_owned())),
3138 headers,
3139 user_body(),
3140 )
3141 .await;
3142 assert_eq!(resp.status(), axum::http::StatusCode::OK);
3143 let trace = rx.try_recv().expect("trace enqueued");
3144 assert_eq!(
3145 trace.policy.mode_profile.as_deref(),
3146 Some("quality"),
3147 "mode_profile must be stamped when quality mode is active"
3148 );
3149 }
3150
3151 #[tokio::test]
3152 async fn mode_profile_absent_from_trace_when_balanced() {
3153 let (state, mut rx) = enforce_state(
3154 &["anthropic/claude-haiku-4-5"],
3155 &["non-empty"],
3156 vec![(
3157 "anthropic/claude-haiku-4-5",
3158 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
3159 )],
3160 );
3161 let resp = messages(
3163 State(state),
3164 Extension(TenantId("default".to_owned())),
3165 HeaderMap::new(),
3166 user_body(),
3167 )
3168 .await;
3169 assert_eq!(resp.status(), axum::http::StatusCode::OK);
3170 let trace = rx.try_recv().expect("trace enqueued");
3171 assert!(
3172 trace.policy.mode_profile.is_none(),
3173 "mode_profile must be absent when Balanced (byte-identical invariant)"
3174 );
3175 }
3176
3177 #[tokio::test]
3178 async fn enforce_serves_first_pass_and_returns_anthropic_shape() {
3179 let (state, mut rx) = enforce_state(
3180 &["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"],
3181 &["non-empty"],
3182 vec![(
3183 "anthropic/claude-haiku-4-5",
3184 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
3185 )],
3186 );
3187 let resp = messages(
3188 State(state),
3189 Extension(TenantId("default".to_owned())),
3190 HeaderMap::new(),
3191 user_body(),
3192 )
3193 .await;
3194 assert_eq!(resp.status(), axum::http::StatusCode::OK);
3195 let json = body_json(resp).await;
3196 assert_eq!(json["type"], "message");
3197 assert_eq!(json["content"][0]["text"], "hello");
3198 assert_eq!(json["model"], "anthropic/claude-haiku-4-5");
3199
3200 let trace = rx.try_recv().expect("a trace was enqueued");
3201 assert_eq!(trace.mode, Mode::Enforce);
3202 assert_eq!(trace.final_.served_rung, Some(0));
3203 assert_eq!(trace.attempts.len(), 1);
3204 }
3205
3206 #[tokio::test]
3207 async fn enforce_escalates_then_serves_and_traces_two_attempts() {
3208 let (state, mut rx) = enforce_state(
3209 &["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"],
3210 &["non-empty"],
3211 vec![
3212 (
3213 "anthropic/claude-haiku-4-5",
3214 Ok(model_resp("anthropic/claude-haiku-4-5", " ")),
3215 ), (
3217 "anthropic/claude-sonnet-5",
3218 Ok(model_resp("anthropic/claude-sonnet-5", "answer")),
3219 ),
3220 ],
3221 );
3222 let resp = messages(
3223 State(state),
3224 Extension(TenantId("default".to_owned())),
3225 HeaderMap::new(),
3226 user_body(),
3227 )
3228 .await;
3229 let json = body_json(resp).await;
3230 assert_eq!(json["content"][0]["text"], "answer");
3231
3232 let trace = rx.try_recv().expect("trace enqueued");
3233 assert_eq!(trace.attempts.len(), 2);
3234 assert_eq!(trace.final_.escalations, 1);
3235 assert_eq!(trace.final_.served_rung, Some(1));
3236 }
3237
3238 #[tokio::test]
3239 async fn enforce_all_rungs_error_returns_502() {
3240 let (state, mut rx) = enforce_state(
3241 &["anthropic/claude-haiku-4-5"],
3242 &["non-empty"],
3243 vec![(
3244 "anthropic/claude-haiku-4-5",
3245 Err(ProviderError::Transport("down".into())),
3246 )],
3247 );
3248 let resp = messages(
3249 State(state),
3250 Extension(TenantId("default".to_owned())),
3251 HeaderMap::new(),
3252 user_body(),
3253 )
3254 .await;
3255 assert_eq!(resp.status(), axum::http::StatusCode::BAD_GATEWAY);
3256 assert!(rx.try_recv().is_ok());
3258 }
3259
3260 #[tokio::test]
3261 async fn no_routing_config_falls_through_to_observe_not_enforce() {
3262 let config = ProxyConfig::from_lookup(|k| match k {
3266 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
3267 _ => None,
3268 })
3269 .unwrap();
3270 let (traces, _rx) = mpsc::channel(64);
3271 let state = AppState {
3272 config: Arc::new(config),
3273 http: reqwest::Client::new(),
3274 providers: ProviderRegistry::new("http://127.0.0.1:1", "http://127.0.0.1:1"),
3275 gate_health: Arc::new(GateHealthRegistry::new()),
3276 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
3277 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
3278 traces,
3279 adaptive: None,
3280 bandit: None,
3281 predictor: None,
3282 tenant_rate_limiter: None,
3283 spill: None,
3284 };
3285 let resp = messages(
3286 State(state),
3287 Extension(TenantId("default".to_owned())),
3288 HeaderMap::new(),
3289 user_body(),
3290 )
3291 .await;
3292 assert_ne!(resp.status(), axum::http::StatusCode::OK);
3294 }
3295
3296 fn bare_enforce_route() -> Route {
3299 use firstpass_core::config::{Match, Mode};
3300 Route {
3301 match_: Match::default(),
3302 mode: Mode::Enforce,
3303 ladder: vec!["anthropic/claude-haiku-4-5".to_owned()],
3304 gates: vec![],
3305 deferred_gates: vec![],
3306 routing_mode: None,
3307 rollout: None,
3308 shadow: None,
3309 }
3310 }
3311
3312 #[test]
3313 fn balanced_preset_application_is_noop() {
3314 let base_max_rungs = 3u32;
3316 let base_speculation = 2u32;
3317 let preset = RoutingMode::Balanced.preset();
3318 let max_rungs = if let Some(d) = preset.max_rungs_delta {
3319 (base_max_rungs as i32 + d).max(1) as u32
3320 } else {
3321 base_max_rungs
3322 };
3323 let speculation = preset.speculation.unwrap_or(base_speculation);
3324 let start_at_top = preset.start_at_top;
3325 assert_eq!(max_rungs, 3, "Balanced must not change max_rungs");
3326 assert_eq!(speculation, 2, "Balanced must not change speculation");
3327 assert!(!start_at_top, "Balanced must not set start_at_top");
3328 }
3329
3330 #[test]
3331 fn resolve_mode_header_wins_over_route_and_global() {
3332 let mut headers = HeaderMap::new();
3333 headers.insert("x-firstpass-mode", "cost".parse().unwrap());
3334 let mut route = bare_enforce_route();
3335 route.routing_mode = Some(RoutingMode::Quality); let mut config = test_config();
3337 config.default_routing_mode = RoutingMode::Max; assert_eq!(
3339 resolve_mode(&headers, &route, &config),
3340 RoutingMode::Cost,
3341 "header must win"
3342 );
3343 }
3344
3345 #[test]
3346 fn resolve_mode_route_wins_over_global_when_no_header() {
3347 let mut route = bare_enforce_route();
3348 route.routing_mode = Some(RoutingMode::Latency);
3349 let mut config = test_config();
3350 config.default_routing_mode = RoutingMode::Max;
3351 assert_eq!(
3352 resolve_mode(&HeaderMap::new(), &route, &config),
3353 RoutingMode::Latency,
3354 "route must beat global default"
3355 );
3356 }
3357
3358 #[test]
3359 fn resolve_mode_global_when_header_and_route_absent() {
3360 let mut config = test_config();
3361 config.default_routing_mode = RoutingMode::Quality;
3362 assert_eq!(
3363 resolve_mode(&HeaderMap::new(), &bare_enforce_route(), &config),
3364 RoutingMode::Quality
3365 );
3366 }
3367
3368 #[test]
3369 fn resolve_mode_unknown_header_falls_through_to_route() {
3370 let mut headers = HeaderMap::new();
3371 headers.insert("x-firstpass-mode", "turbo-mode".parse().unwrap());
3373 let mut route = bare_enforce_route();
3374 route.routing_mode = Some(RoutingMode::Cost);
3375 let config = test_config();
3376 assert_eq!(
3377 resolve_mode(&headers, &route, &config),
3378 RoutingMode::Cost,
3379 "unknown header value must fall through to route"
3380 );
3381 }
3382
3383 #[test]
3384 fn resolve_mode_header_case_insensitive() {
3385 let mut headers = HeaderMap::new();
3386 headers.insert("x-firstpass-mode", "QUALITY".parse().unwrap());
3387 assert_eq!(
3388 resolve_mode(&headers, &bare_enforce_route(), &test_config()),
3389 RoutingMode::Quality
3390 );
3391 }
3392
3393 #[test]
3394 fn resolve_mode_no_mode_set_returns_balanced() {
3395 assert_eq!(
3397 resolve_mode(&HeaderMap::new(), &bare_enforce_route(), &test_config()),
3398 RoutingMode::Balanced
3399 );
3400 }
3401
3402 #[test]
3403 fn capabilities_json_includes_routing_modes() {
3404 let modes: Vec<&'static str> = RoutingMode::ALL.iter().map(|m| m.as_str()).collect();
3405 assert!(modes.contains(&"balanced"));
3406 assert!(modes.contains(&"cost"));
3407 assert!(modes.contains(&"quality"));
3408 assert!(modes.contains(&"latency"));
3409 assert!(modes.contains(&"max"));
3410 assert!(modes.contains(&"observe"));
3411 }
3412
3413 #[test]
3414 fn detects_stream_requests() {
3415 assert!(is_stream_request(br#"{"stream": true}"#));
3416 assert!(!is_stream_request(br#"{"stream": false}"#));
3417 assert!(!is_stream_request(br#"{"model":"m"}"#));
3418 assert!(!is_stream_request(b"not json"));
3419 }
3420
3421 #[test]
3422 fn detects_tool_blocks_in_messages() {
3423 let with =
3424 br#"{"messages":[{"role":"user","content":[{"type":"tool_result","content":"42"}]}]}"#;
3425 let without = br#"{"messages":[{"role":"user","content":"hi"}]}"#;
3426 assert!(messages_have_tool_blocks(with));
3427 assert!(!messages_have_tool_blocks(without));
3428 }
3429
3430 #[test]
3431 fn enforce_only_handles_plain_text() {
3432 let plain =
3433 Bytes::from_static(br#"{"model":"m","messages":[{"role":"user","content":"hi"}]}"#);
3434 let tools = Bytes::from_static(
3435 br#"{"model":"m","tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
3436 );
3437 let f_plain = extract_features(&HeaderMap::new(), &plain);
3438 let f_tools = extract_features(&HeaderMap::new(), &tools);
3439 let anthropic_ladder = vec!["anthropic/claude-haiku-4-5".to_owned()];
3440 let providers = test_registry();
3441 assert!(enforce_can_handle(
3443 &f_plain,
3444 &plain,
3445 false,
3446 &anthropic_ladder,
3447 &providers,
3448 Dialect::Anthropic,
3449 ));
3450 assert!(!enforce_can_handle(
3451 &f_tools,
3452 &tools,
3453 false,
3454 &anthropic_ladder,
3455 &providers,
3456 Dialect::Anthropic,
3457 ));
3458 }
3459
3460 #[test]
3461 fn structured_enforce_routes_tools_and_streaming() {
3462 let tools = Bytes::from_static(
3465 br#"{"model":"m","tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
3466 );
3467 let streaming_tools = Bytes::from_static(
3468 br#"{"model":"m","stream":true,"tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
3469 );
3470 let f = extract_features(&HeaderMap::new(), &tools);
3471 let anthropic_ladder = vec![
3472 "anthropic/claude-haiku-4-5".to_owned(),
3473 "anthropic/claude-sonnet-5".to_owned(),
3474 ];
3475 let providers = test_registry();
3476 assert!(enforce_can_handle(
3477 &f,
3478 &tools,
3479 true,
3480 &anthropic_ladder,
3481 &providers,
3482 Dialect::Anthropic,
3483 ));
3484 assert!(enforce_can_handle(
3485 &f,
3486 &streaming_tools,
3487 true,
3488 &anthropic_ladder,
3489 &providers,
3490 Dialect::Anthropic,
3491 ));
3492 }
3493
3494 fn test_registry() -> crate::provider::ProviderRegistry {
3496 crate::provider::ProviderRegistry::new("http://localhost", "http://localhost")
3497 }
3498
3499 #[test]
3500 fn fidelity_guard_blocks_structured_on_non_verbatim_ladder() {
3501 let tools = Bytes::from_static(
3505 br#"{"model":"m","tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
3506 );
3507 let plain =
3508 Bytes::from_static(br#"{"model":"m","messages":[{"role":"user","content":"hi"}]}"#);
3509 let f_tools = extract_features(&HeaderMap::new(), &tools);
3510 let f_plain = extract_features(&HeaderMap::new(), &plain);
3511 let providers = test_registry();
3512 let mixed_ladder = vec![
3513 "openai/gpt-4.1-mini".to_owned(),
3514 "anthropic/claude-sonnet-5".to_owned(),
3515 ];
3516 assert!(!enforce_can_handle(
3517 &f_tools,
3518 &tools,
3519 true,
3520 &mixed_ladder,
3521 &providers,
3522 Dialect::Anthropic,
3523 ));
3524 assert!(enforce_can_handle(
3525 &f_plain,
3526 &plain,
3527 true,
3528 &mixed_ladder,
3529 &providers,
3530 Dialect::Anthropic,
3531 ));
3532 }
3533
3534 #[test]
3535 fn enforce_sse_reemission_preserves_text_and_tool_use() {
3536 let resp = ModelResponse {
3539 model: "anthropic/claude-haiku-4-5".to_owned(),
3540 text: "let me check".to_owned(),
3541 in_tokens: 5,
3542 out_tokens: 7,
3543 raw: serde_json::json!({
3544 "content": [
3545 { "type": "text", "text": "let me check" },
3546 { "type": "tool_use", "id": "tu_1", "name": "get_weather", "input": { "city": "Paris" } }
3547 ]
3548 }),
3549 };
3550 let sse = anthropic_sse_from_message(&anthropic_response_json(&resp));
3551
3552 let frames: Vec<Value> = sse
3554 .lines()
3555 .filter_map(|l| l.strip_prefix("data: "))
3556 .map(|d| serde_json::from_str::<Value>(d).expect("each SSE data frame is valid JSON"))
3557 .collect();
3558
3559 assert_eq!(frames.first().unwrap()["type"], "message_start");
3561 assert_eq!(frames.last().unwrap()["type"], "message_stop");
3562 assert!(frames.iter().any(|f| f["delta"]["type"] == "text_delta"
3564 && f["delta"]["text"] == "let me check"));
3565 assert!(
3568 frames
3569 .iter()
3570 .any(|f| f["content_block"]["type"] == "tool_use"
3571 && f["content_block"]["name"] == "get_weather"
3572 && f["content_block"]["id"] == "tu_1")
3573 );
3574 assert!(
3575 frames
3576 .iter()
3577 .any(|f| f["delta"]["type"] == "input_json_delta"
3578 && f["delta"]["partial_json"] == r#"{"city":"Paris"}"#)
3579 );
3580 }
3581
3582 #[tokio::test]
3587 async fn enforce_falls_back_to_observe_for_tool_requests() {
3588 let toml = "[[price]]\nmodel = \"anthropic/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/m\"]\ngates = [\"non-empty\"]\n";
3589 let config = ProxyConfig::from_lookup(|k| match k {
3590 "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
3591 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
3592 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
3593 _ => None,
3594 })
3595 .unwrap();
3596 let mut outs = HashMap::new();
3597 outs.insert(
3598 "anthropic/m".to_owned(),
3599 Ok(model_resp("anthropic/m", "hello")),
3600 );
3601 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
3602 map.insert(
3603 "anthropic".to_owned(),
3604 Arc::new(MockProvider::new("anthropic", outs)),
3605 );
3606 let (traces, _rx) = mpsc::channel(64);
3607 let state = AppState {
3608 config: Arc::new(config),
3609 http: reqwest::Client::new(),
3610 providers: ProviderRegistry::from_map(map),
3611 gate_health: Arc::new(GateHealthRegistry::new()),
3612 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
3613 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
3614 traces,
3615 adaptive: None,
3616 bandit: None,
3617 predictor: None,
3618 tenant_rate_limiter: None,
3619 spill: None,
3620 };
3621
3622 let plain =
3624 Bytes::from_static(br#"{"model":"m","messages":[{"role":"user","content":"hi"}]}"#);
3625 let resp = messages(
3626 State(state.clone()),
3627 Extension(TenantId("default".to_owned())),
3628 HeaderMap::new(),
3629 plain,
3630 )
3631 .await;
3632 assert_eq!(
3633 resp.status(),
3634 axum::http::StatusCode::OK,
3635 "plain text should enforce"
3636 );
3637
3638 let tools = Bytes::from_static(
3641 br#"{"model":"m","tools":[{"name":"get_weather"}],"messages":[{"role":"user","content":"hi"}]}"#,
3642 );
3643 let resp = messages(
3644 State(state.clone()),
3645 Extension(TenantId("default".to_owned())),
3646 HeaderMap::new(),
3647 tools.clone(),
3648 )
3649 .await;
3650 assert_eq!(
3651 resp.status(),
3652 axum::http::StatusCode::OK,
3653 "tool request must route through enforce by default (ADR 0005 default-on)"
3654 );
3655
3656 let toml_off = "[[price]]\nmodel = \"anthropic/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/m\"]\ngates = [\"non-empty\"]\n[escalation]\nenforce_structured = false\n";
3659 let config_off = ProxyConfig::from_lookup(|k| match k {
3660 "FIRSTPASS_CONFIG_TOML" => Some(toml_off.to_owned()),
3661 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
3662 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
3663 _ => None,
3664 })
3665 .unwrap();
3666 let state_off = AppState {
3667 config: Arc::new(config_off),
3668 ..state.clone()
3669 };
3670 let resp = messages(
3671 State(state_off),
3672 Extension(TenantId("default".to_owned())),
3673 HeaderMap::new(),
3674 tools,
3675 )
3676 .await;
3677 assert_ne!(
3678 resp.status(),
3679 axum::http::StatusCode::OK,
3680 "with enforce_structured = false a tool request must fall back to observe"
3681 );
3682
3683 let toolres = Bytes::from_static(
3685 br#"{"model":"m","messages":[{"role":"user","content":[{"type":"tool_result","tool_use_id":"x","content":"42"}]}]}"#,
3686 );
3687 let resp = messages(
3688 State(state),
3689 Extension(TenantId("default".to_owned())),
3690 HeaderMap::new(),
3691 toolres,
3692 )
3693 .await;
3694 assert_eq!(
3695 resp.status(),
3696 axum::http::StatusCode::OK,
3697 "tool_result blocks route through enforce by default too (verbatim carry)"
3698 );
3699 }
3700
3701 #[tokio::test]
3705 async fn panel_is_public_markup_but_receipts_stay_behind_auth() {
3706 use tower::ServiceExt;
3707 let (state, _rx) = enforce_state(
3708 &["anthropic/claude-haiku-4-5"],
3709 &["non-empty"],
3710 vec![(
3711 "anthropic/claude-haiku-4-5",
3712 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
3713 )],
3714 );
3715 let router = app(state).expect("prometheus recorder installs");
3716
3717 let page = router
3718 .clone()
3719 .oneshot(
3720 axum::http::Request::builder()
3721 .uri("/panel")
3722 .body(Body::empty())
3723 .unwrap(),
3724 )
3725 .await
3726 .unwrap();
3727 assert_eq!(page.status(), axum::http::StatusCode::OK);
3728 let html = axum::body::to_bytes(page.into_body(), 1 << 20)
3729 .await
3730 .unwrap();
3731 let html = String::from_utf8_lossy(&html);
3732 assert!(
3733 html.contains("firstpass") && html.contains("/v1/receipts"),
3734 "the page must fetch its data rather than embed it"
3735 );
3736 assert!(
3739 !html.contains("http://") && !html.contains("https://"),
3740 "the panel must not reference any external origin"
3741 );
3742 assert!(html.contains("const esc ="), "the escaper must exist");
3747 for raw in [
3748 "${a.model}",
3749 "${a.rung}",
3750 "${a.verdict}",
3751 "${r.trace_id}",
3752 "${r.served_rung}",
3753 ] {
3754 assert!(
3755 !html.contains(raw),
3756 "receipt field interpolated unescaped into innerHTML: {raw}"
3757 );
3758 }
3759
3760 let data = router
3761 .oneshot(
3762 axum::http::Request::builder()
3763 .uri("/v1/receipts?limit=5")
3764 .body(Body::empty())
3765 .unwrap(),
3766 )
3767 .await
3768 .unwrap();
3769 assert_eq!(data.status(), axum::http::StatusCode::OK);
3773 let body = axum::body::to_bytes(data.into_body(), 1 << 20)
3774 .await
3775 .unwrap();
3776 let v: Value = serde_json::from_slice(&body).unwrap();
3777 assert!(v.get("receipts").and_then(Value::as_array).is_some());
3778 }
3779
3780 async fn feedback_state() -> (AppState, std::path::PathBuf, String) {
3784 let db = std::env::temp_dir().join(format!("firstpass-feedback-{}.db", Uuid::now_v7()));
3785 let (tx, handle) = crate::store::open(&db).unwrap();
3786
3787 let mut trace = build_error_trace(
3788 &ProxyConfig::from_lookup(|_| None).unwrap(),
3789 &Bytes::from_static(b"{}"),
3790 5,
3791 Some("sess-fb"),
3792 );
3793 trace.attempts.push(Attempt {
3794 rung: 0,
3795 model: "anthropic/claude-haiku-4-5".into(),
3796 provider: "anthropic".into(),
3797 in_tokens: 10,
3798 out_tokens: 5,
3799 cost_usd: 0.001,
3800 latency_ms: 5,
3801 gates: vec![],
3802 verdict: Verdict::Pass,
3803 });
3804 let trace_id = trace.trace_id.to_string();
3805 tx.try_send(trace).unwrap();
3806 drop(tx);
3807 handle.await.unwrap();
3808
3809 let db_str = db.to_string_lossy().into_owned();
3810 let config = ProxyConfig::from_lookup(move |k| match k {
3811 "FIRSTPASS_DB" => Some(db_str.clone()),
3812 _ => None,
3813 })
3814 .unwrap();
3815 let (traces, _rx) = mpsc::channel(64);
3816 let state = AppState {
3817 config: Arc::new(config),
3818 http: reqwest::Client::new(),
3819 providers: ProviderRegistry::new("http://127.0.0.1:1", "http://127.0.0.1:1"),
3820 gate_health: Arc::new(GateHealthRegistry::new()),
3821 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
3822 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
3823 traces,
3824 adaptive: None,
3825 bandit: None,
3826 predictor: None,
3827 tenant_rate_limiter: None,
3828 spill: None,
3829 };
3830 (state, db, trace_id)
3831 }
3832
3833 #[tokio::test]
3834 async fn feedback_nudges_the_adaptive_threshold() {
3835 use firstpass_core::conformal::AdaptiveConformal;
3836 let (mut state, _db, trace_id) = feedback_state().await;
3837 let aci = Arc::new(std::sync::Mutex::new(AdaptiveConformal::new(0.1, 0.2, 0.5)));
3838 state.adaptive = Some(aci.clone());
3839 let before = aci.lock().unwrap().threshold();
3840
3841 let fail = Bytes::from(
3843 serde_json::json!({ "trace_id": trace_id, "gate_id": "tests", "verdict": "fail", "reporter": "ci" })
3844 .to_string(),
3845 );
3846 assert_eq!(
3847 feedback(
3848 State(state.clone()),
3849 Extension(TenantId("default".to_owned())),
3850 fail
3851 )
3852 .await
3853 .status(),
3854 axum::http::StatusCode::ACCEPTED
3855 );
3856 let after_fail = aci.lock().unwrap().threshold();
3857 assert!(
3858 after_fail > before,
3859 "served fail should raise the live threshold: {before} -> {after_fail}"
3860 );
3861
3862 let pass = Bytes::from(
3864 serde_json::json!({ "trace_id": trace_id, "gate_id": "tests", "verdict": "pass", "reporter": "ci" })
3865 .to_string(),
3866 );
3867 let _ = feedback(
3868 State(state),
3869 Extension(TenantId("default".to_owned())),
3870 pass,
3871 )
3872 .await;
3873 assert!(aci.lock().unwrap().threshold() < after_fail);
3874 }
3875
3876 #[tokio::test]
3877 async fn feedback_records_a_deferred_verdict_without_breaking_the_chain() {
3878 let (state, db, trace_id) = feedback_state().await;
3879 let body = Bytes::from(
3880 serde_json::json!({
3881 "trace_id": trace_id,
3882 "gate_id": "tests",
3883 "verdict": "pass",
3884 "score": 1.0,
3885 "reporter": "ci",
3886 })
3887 .to_string(),
3888 );
3889 let resp = feedback(
3890 State(state),
3891 Extension(TenantId("default".to_owned())),
3892 body,
3893 )
3894 .await;
3895 assert_eq!(resp.status(), axum::http::StatusCode::ACCEPTED);
3896
3897 let view = crate::store::load_trace_view(&db, "default", &trace_id)
3899 .unwrap()
3900 .unwrap();
3901 assert_eq!(view.deferred.len(), 1);
3902 assert_eq!(view.deferred[0].gate_id, "tests");
3903 let traces = crate::store::load_all_traces(&db).unwrap();
3905 firstpass_core::verify_chain(&traces, GENESIS_HASH).unwrap();
3906
3907 let _ = std::fs::remove_file(&db);
3908 }
3909
3910 #[tokio::test]
3911 async fn feedback_for_unknown_trace_is_404() {
3912 let (state, db, _trace_id) = feedback_state().await;
3913 let body = Bytes::from(
3914 serde_json::json!({
3915 "trace_id": "does-not-exist",
3916 "gate_id": "tests",
3917 "verdict": "pass",
3918 "reporter": "ci",
3919 })
3920 .to_string(),
3921 );
3922 let resp = feedback(
3923 State(state),
3924 Extension(TenantId("default".to_owned())),
3925 body,
3926 )
3927 .await;
3928 assert_eq!(resp.status(), axum::http::StatusCode::NOT_FOUND);
3929 let _ = std::fs::remove_file(&db);
3930 }
3931
3932 #[tokio::test]
3935 async fn feedback_across_tenants_is_404_not_403() {
3936 let (state, db, trace_id) = feedback_state().await;
3937 let body = Bytes::from(
3938 serde_json::json!({
3939 "trace_id": trace_id,
3940 "gate_id": "tests",
3941 "verdict": "pass",
3942 "score": 1.0,
3943 "reporter": "attacker",
3944 })
3945 .to_string(),
3946 );
3947 let resp = feedback(
3949 State(state),
3950 Extension(TenantId("tenant-b".to_owned())),
3951 body,
3952 )
3953 .await;
3954 assert_eq!(
3955 resp.status(),
3956 axum::http::StatusCode::NOT_FOUND,
3957 "cross-tenant feedback must look exactly like a missing trace"
3958 );
3959 let _ = std::fs::remove_file(&db);
3960 }
3961
3962 #[tokio::test]
3963 async fn feedback_rejects_bad_verdict_and_score() {
3964 let (state, db, trace_id) = feedback_state().await;
3965 let bad_verdict = Bytes::from(
3966 serde_json::json!({ "trace_id": trace_id, "gate_id": "g", "verdict": "maybe", "reporter": "x" })
3967 .to_string(),
3968 );
3969 assert_eq!(
3970 feedback(
3971 State(state.clone()),
3972 Extension(TenantId("default".to_owned())),
3973 bad_verdict
3974 )
3975 .await
3976 .status(),
3977 axum::http::StatusCode::BAD_REQUEST
3978 );
3979 let bad_score = Bytes::from(
3980 serde_json::json!({ "trace_id": trace_id, "gate_id": "g", "verdict": "pass", "score": 9.0, "reporter": "x" })
3981 .to_string(),
3982 );
3983 assert_eq!(
3984 feedback(
3985 State(state),
3986 Extension(TenantId("default".to_owned())),
3987 bad_score
3988 )
3989 .await
3990 .status(),
3991 axum::http::StatusCode::BAD_REQUEST
3992 );
3993 let _ = std::fs::remove_file(&db);
3994 }
3995
3996 #[tokio::test]
3997 async fn metrics_endpoint_renders_after_a_real_request() {
3998 use tower::ServiceExt;
3999
4000 let (state, mut rx) = enforce_state(
4001 &["anthropic/claude-haiku-4-5"],
4002 &["non-empty"],
4003 vec![(
4004 "anthropic/claude-haiku-4-5",
4005 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
4006 )],
4007 );
4008 let router = app(state).expect("prometheus recorder installs");
4009
4010 let req = axum::http::Request::builder()
4011 .method("POST")
4012 .uri("/v1/messages")
4013 .header("content-type", "application/json")
4014 .body(Body::from(user_body()))
4015 .unwrap();
4016 let resp = router.clone().oneshot(req).await.unwrap();
4017 assert_eq!(resp.status(), axum::http::StatusCode::OK);
4018 rx.try_recv().expect("a trace was enqueued");
4019
4020 let metrics_req = axum::http::Request::builder()
4021 .method("GET")
4022 .uri("/metrics")
4023 .body(Body::empty())
4024 .unwrap();
4025 let metrics_resp = router.oneshot(metrics_req).await.unwrap();
4026 assert_eq!(metrics_resp.status(), axum::http::StatusCode::OK);
4027 let bytes = axum::body::to_bytes(metrics_resp.into_body(), 1 << 20)
4028 .await
4029 .unwrap();
4030 let body = String::from_utf8(bytes.to_vec()).unwrap();
4031 assert!(
4032 body.contains("firstpass_enforce_latency_ms"),
4033 "metrics body missing enforce latency histogram: {body}"
4034 );
4035 assert!(
4036 body.contains("firstpass_served_total"),
4037 "metrics body missing served counter: {body}"
4038 );
4039 }
4040
4041 fn auth_state(require_auth: bool, keys_json: Option<String>) -> AppState {
4045 auth_state_rated(require_auth, keys_json, None)
4046 }
4047
4048 fn auth_state_rated(
4051 require_auth: bool,
4052 keys_json: Option<String>,
4053 rate_per_sec: Option<u32>,
4054 ) -> AppState {
4055 let config = ProxyConfig::from_lookup(|k| match k {
4056 "FIRSTPASS_REQUIRE_AUTH" => require_auth.then(|| "true".to_owned()),
4057 "FIRSTPASS_TENANT_KEYS_JSON" => keys_json.clone(),
4058 "FIRSTPASS_TENANT_RATE_PER_SEC" => rate_per_sec.map(|n| n.to_string()),
4059 _ => None,
4060 })
4061 .unwrap();
4062 let (traces, _rx) = mpsc::channel(64);
4063 std::mem::forget(_rx);
4066 let providers: HashMap<String, Arc<dyn Provider>> = HashMap::new();
4067 let tenant_rate_limiter = build_tenant_rate_limiter(&config);
4068 AppState {
4069 config: Arc::new(config),
4070 http: reqwest::Client::new(),
4071 providers: ProviderRegistry::from_map(providers),
4072 gate_health: Arc::new(GateHealthRegistry::new()),
4073 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
4074 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
4075 traces,
4076 adaptive: None,
4077 bandit: None,
4078 predictor: None,
4079 tenant_rate_limiter,
4080 spill: None,
4081 }
4082 }
4083
4084 fn cap_request(auth_header: Option<&str>) -> axum::http::Request<Body> {
4085 let mut b = axum::http::Request::builder()
4086 .method("GET")
4087 .uri("/v1/capabilities");
4088 if let Some(h) = auth_header {
4089 b = b.header("authorization", h);
4090 }
4091 b.body(Body::empty()).unwrap()
4092 }
4093
4094 #[tokio::test]
4095 async fn auth_off_allows_unauthenticated_request() {
4096 use tower::ServiceExt;
4097 let router = app(auth_state(false, None)).expect("router");
4098 let resp = router.oneshot(cap_request(None)).await.unwrap();
4099 assert_eq!(resp.status(), axum::http::StatusCode::OK);
4101 }
4102
4103 #[tokio::test]
4104 async fn auth_on_missing_key_is_401_opaque() {
4105 use tower::ServiceExt;
4106 let hash = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
4107 let keys = format!("{{\"tenant-a\": {hash:?}}}");
4108 let router = app(auth_state(true, Some(keys))).expect("router");
4109
4110 let resp = router.oneshot(cap_request(None)).await.unwrap();
4111 assert_eq!(resp.status(), axum::http::StatusCode::UNAUTHORIZED);
4112 let json = body_json(resp).await;
4113 assert_eq!(json["error"]["type"], "unauthorized");
4114 let msg = json["error"]["message"].as_str().unwrap();
4116 assert!(!msg.contains("tenant"), "no tenant oracle in body: {msg}");
4117 }
4118
4119 #[tokio::test]
4120 async fn auth_on_invalid_key_is_401() {
4121 use tower::ServiceExt;
4122 let hash = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
4123 let keys = format!("{{\"tenant-a\": {hash:?}}}");
4124 let router = app(auth_state(true, Some(keys))).expect("router");
4125
4126 let resp = router
4127 .oneshot(cap_request(Some("Bearer wrong-key")))
4128 .await
4129 .unwrap();
4130 assert_eq!(resp.status(), axum::http::StatusCode::UNAUTHORIZED);
4131 }
4132
4133 #[tokio::test]
4134 async fn auth_on_valid_key_proceeds() {
4135 use tower::ServiceExt;
4136 let hash = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
4137 let keys = format!("{{\"tenant-a\": {hash:?}}}");
4138 let router = app(auth_state(true, Some(keys))).expect("router");
4139
4140 let resp = router
4142 .oneshot(cap_request(Some("Bearer tenant-a.key-a")))
4143 .await
4144 .unwrap();
4145 assert_eq!(resp.status(), axum::http::StatusCode::OK);
4147 }
4148
4149 fn two_tenant_state(rate_per_sec: Option<u32>) -> AppState {
4151 let hash_a = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
4152 let hash_b = crate::tenant_auth::TenantKeys::hash_key("key-b").unwrap();
4153 let keys = format!("{{\"tenant-a\": {hash_a:?}, \"tenant-b\": {hash_b:?}}}");
4154 auth_state_rated(true, Some(keys), rate_per_sec)
4155 }
4156
4157 #[tokio::test]
4158 async fn tenant_exceeding_rate_limit_gets_429_opaque() {
4159 use tower::ServiceExt;
4160 let router = app(two_tenant_state(Some(1))).expect("router");
4166
4167 let (r1, r2, r3, r4) = tokio::join!(
4168 router
4169 .clone()
4170 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
4171 router
4172 .clone()
4173 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
4174 router
4175 .clone()
4176 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
4177 router
4178 .clone()
4179 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
4180 );
4181 let responses = [r1.unwrap(), r2.unwrap(), r3.unwrap(), r4.unwrap()];
4182 let ok = responses
4183 .iter()
4184 .filter(|r| r.status() == axum::http::StatusCode::OK)
4185 .count();
4186 assert!(ok >= 1, "the burst's first request must pass");
4187 let limited: Vec<_> = responses
4188 .into_iter()
4189 .filter(|r| r.status() == axum::http::StatusCode::TOO_MANY_REQUESTS)
4190 .collect();
4191 assert!(
4192 !limited.is_empty(),
4193 "a 4-request burst against 1 req/sec must trip the limiter"
4194 );
4195
4196 let json = body_json(limited.into_iter().next().unwrap()).await;
4197 assert_eq!(json["error"]["type"], "rate_limited");
4198 let msg = json["error"]["message"].as_str().unwrap();
4200 assert!(!msg.contains('1'), "no limit value in body: {msg}");
4201 }
4202
4203 #[tokio::test]
4204 async fn rate_limit_buckets_are_independent_per_tenant() {
4205 use tower::ServiceExt;
4206 let router = app(two_tenant_state(Some(1))).expect("router");
4207
4208 let (a1, a2, a3) = tokio::join!(
4211 router
4212 .clone()
4213 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
4214 router
4215 .clone()
4216 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
4217 router
4218 .clone()
4219 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
4220 );
4221 let a_limited = [a1.unwrap(), a2.unwrap(), a3.unwrap()]
4222 .iter()
4223 .filter(|r| r.status() == axum::http::StatusCode::TOO_MANY_REQUESTS)
4224 .count();
4225 assert!(a_limited >= 1, "tenant A's burst must trip its limiter");
4226
4227 let b1 = router
4229 .clone()
4230 .oneshot(cap_request(Some("Bearer tenant-b.key-b")))
4231 .await
4232 .unwrap();
4233 assert_eq!(b1.status(), axum::http::StatusCode::OK);
4234 }
4235
4236 #[tokio::test]
4237 async fn rate_limit_unset_never_429s() {
4238 use tower::ServiceExt;
4239 let router = app(two_tenant_state(None)).expect("router");
4242 for _ in 0..20 {
4243 let resp = router
4244 .clone()
4245 .oneshot(cap_request(Some("Bearer tenant-a.key-a")))
4246 .await
4247 .unwrap();
4248 assert_eq!(resp.status(), axum::http::StatusCode::OK);
4249 }
4250 }
4251
4252 #[test]
4255 fn u01_is_deterministic_and_in_range() {
4256 let s1 = u01(0xDEAD_BEEF_CAFE_1234_u128);
4257 let s2 = u01(0xDEAD_BEEF_CAFE_1234_u128);
4258 assert_eq!(s1, s2, "u01 must be deterministic for the same seed");
4259 assert!((0.0..1.0).contains(&s1), "u01 must return [0, 1), got {s1}");
4260
4261 let s3 = u01(0x1234_5678_9ABC_DEF0_u128);
4263 assert_ne!(s1, s3, "different seeds should give different values");
4264
4265 for i in 0u64..256 {
4267 let v = u01(i as u128);
4268 assert!((0.0..1.0).contains(&v), "seed {i}: u01={v} out of [0,1)");
4269 }
4270 }
4271
4272 #[test]
4273 fn epsilon_propensity_formula() {
4274 let epsilon = 0.2_f64;
4275 let k = 3_usize;
4276 let greedy = 1_u32;
4277
4278 let p_greedy = epsilon_propensity(greedy, greedy, epsilon, k);
4280 let expected_greedy = (1.0 - epsilon) + epsilon / k as f64;
4281 assert!(
4282 (p_greedy - expected_greedy).abs() < 1e-12,
4283 "{p_greedy} != {expected_greedy}"
4284 );
4285
4286 let p_other = epsilon_propensity(0, greedy, epsilon, k);
4288 let expected_other = epsilon / k as f64;
4289 assert!(
4290 (p_other - expected_other).abs() < 1e-12,
4291 "{p_other} != {expected_other}"
4292 );
4293
4294 for chosen in 0..k as u32 {
4296 let p = epsilon_propensity(chosen, greedy, epsilon, k);
4297 assert!(
4298 p > 0.0 && p <= 1.0,
4299 "propensity {p} out of (0,1] for chosen={chosen}"
4300 );
4301 }
4302 }
4303
4304 #[test]
4305 fn epsilon_branch_and_greedy_branch_both_occur_over_many_seeds() {
4306 let epsilon = 0.3_f64;
4308 let mut saw_explore = false;
4309 let mut saw_greedy = false;
4310 for i in 0u64..200 {
4311 let u = u01(i as u128);
4312 if u < epsilon {
4313 saw_explore = true;
4314 } else {
4315 saw_greedy = true;
4316 }
4317 if saw_explore && saw_greedy {
4318 break;
4319 }
4320 }
4321 assert!(
4322 saw_explore,
4323 "epsilon branch must fire with epsilon=0.3 over 200 seeds"
4324 );
4325 assert!(
4326 saw_greedy,
4327 "greedy branch must occur with epsilon=0.3 over 200 seeds"
4328 );
4329 }
4330
4331 #[test]
4332 fn epsilon_propensity_sums_to_one_over_all_rungs() {
4333 let epsilon = 0.15_f64;
4335 let k = 4_usize;
4336 let greedy = 2_u32;
4337 let total: f64 = (0..k as u32)
4338 .map(|r| epsilon_propensity(r, greedy, epsilon, k))
4339 .sum();
4340 assert!(
4341 (total - 1.0).abs() < 1e-12,
4342 "propensities must sum to 1, got {total}"
4343 );
4344 }
4345
4346 #[tokio::test(start_paused = true)]
4349 async fn keepalive_stream_ticks_then_emits_final_frame() {
4350 let (tx, rx) = tokio::sync::oneshot::channel::<Result<Value, ProxyError>>();
4351 let mut ticks = tokio::time::interval(SSE_KEEPALIVE_EVERY);
4352 ticks.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
4353 ticks.reset();
4354 let mut stream = KeepaliveStream {
4355 rx: Some(rx),
4356 ticks,
4357 format_message: anthropic_sse_from_message,
4358 };
4359 async fn next(
4361 stream: &mut KeepaliveStream,
4362 ) -> Option<Option<Result<Bytes, std::convert::Infallible>>> {
4363 std::future::poll_fn(|cx| {
4364 std::task::Poll::Ready(
4365 match futures_core::Stream::poll_next(std::pin::Pin::new(&mut *stream), cx) {
4366 std::task::Poll::Ready(item) => Some(item),
4367 std::task::Poll::Pending => None,
4368 },
4369 )
4370 })
4371 .await
4372 }
4373
4374 assert!(
4376 next(&mut stream).await.is_none(),
4377 "no frame before an interval"
4378 );
4379
4380 tokio::time::advance(SSE_KEEPALIVE_EVERY + Duration::from_millis(1)).await;
4382 let frame = next(&mut stream)
4383 .await
4384 .expect("keepalive due")
4385 .unwrap()
4386 .unwrap();
4387 assert!(
4388 frame.starts_with(b": "),
4389 "keepalive must be an SSE comment (ignored by every conforming parser)"
4390 );
4391
4392 let message = serde_json::json!({
4394 "id": "msg_1", "type": "message", "role": "assistant", "model": "m",
4395 "content": [{ "type": "text", "text": "done" }],
4396 "usage": { "input_tokens": 1, "output_tokens": 1 }
4397 });
4398 tx.send(Ok(message)).unwrap();
4399 let frame = next(&mut stream)
4400 .await
4401 .expect("final frame")
4402 .unwrap()
4403 .unwrap();
4404 let text = String::from_utf8(frame.to_vec()).unwrap();
4405 assert!(text.contains("event: message_start"));
4406 assert!(text.contains("event: message_stop"));
4407 let eos = next(&mut stream)
4408 .await
4409 .expect("stream must end after the final frame");
4410 assert!(eos.is_none(), "end-of-stream after the final frame");
4411 }
4412
4413 #[tokio::test]
4416 async fn streaming_enforce_serves_full_sse_sequence() {
4417 let toml = "[[price]]\nmodel = \"anthropic/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/m\"]\ngates = [\"non-empty\"]\n";
4418 let config = ProxyConfig::from_lookup(|k| match k {
4419 "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
4420 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
4421 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
4422 _ => None,
4423 })
4424 .unwrap();
4425 let mut outs = HashMap::new();
4426 outs.insert(
4427 "anthropic/m".to_owned(),
4428 Ok(model_resp("anthropic/m", "gated answer")),
4429 );
4430 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
4431 map.insert(
4432 "anthropic".to_owned(),
4433 Arc::new(MockProvider::new("anthropic", outs)),
4434 );
4435 let (traces, _rx) = mpsc::channel(64);
4436 let state = AppState {
4437 config: Arc::new(config),
4438 http: reqwest::Client::new(),
4439 providers: ProviderRegistry::from_map(map),
4440 gate_health: Arc::new(GateHealthRegistry::new()),
4441 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
4442 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
4443 traces,
4444 adaptive: None,
4445 bandit: None,
4446 predictor: None,
4447 tenant_rate_limiter: None,
4448 spill: None,
4449 };
4450 let body = Bytes::from_static(
4451 br#"{"model":"m","stream":true,"messages":[{"role":"user","content":"hi"}]}"#,
4452 );
4453 let resp = messages(
4454 State(state),
4455 Extension(TenantId("default".to_owned())),
4456 HeaderMap::new(),
4457 body,
4458 )
4459 .await;
4460 assert_eq!(resp.status(), axum::http::StatusCode::OK);
4461 assert!(
4462 resp.headers()
4463 .get(axum::http::header::CONTENT_TYPE)
4464 .and_then(|v| v.to_str().ok())
4465 .is_some_and(|ct| ct.starts_with("text/event-stream")),
4466 "streaming client must get SSE"
4467 );
4468 let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
4469 .await
4470 .unwrap();
4471 let text = String::from_utf8(bytes.to_vec()).unwrap();
4472 assert!(text.contains("event: message_start"));
4473 assert!(text.contains("gated answer"));
4474 assert!(text.contains("event: message_stop"));
4475 }
4476
4477 #[test]
4482 fn parse_openai_request_plain_text() {
4483 let body = br#"{"model":"gpt-4o","max_tokens":256,"messages":[{"role":"user","content":"hello"}]}"#;
4485 let req = parse_openai_request(body, false).expect("must parse");
4486 assert_eq!(req.model, "gpt-4o");
4487 assert_eq!(req.max_tokens, 256);
4488 assert_eq!(req.messages.len(), 1);
4489 assert_eq!(req.messages[0].role, "user");
4490 assert_eq!(req.messages[0].content, Value::String("hello".to_owned()));
4491 assert!(req.system.is_none());
4492 assert_eq!(req.raw, Value::Null);
4494 }
4495
4496 #[test]
4497 fn parse_openai_request_system_message() {
4498 let body = br#"{"model":"gpt-4o","messages":[{"role":"system","content":"be concise"},{"role":"user","content":"hi"}]}"#;
4499 let req = parse_openai_request(body, false).expect("must parse");
4500 assert_eq!(req.system.as_deref(), Some("be concise"));
4501 assert_eq!(req.messages.len(), 1);
4502 assert_eq!(req.messages[0].role, "user");
4503 }
4504
4505 #[test]
4506 fn parse_openai_request_tool_calls_translate_to_tool_use() {
4507 let body = br#"{
4508 "model":"gpt-4o",
4509 "messages":[
4510 {"role":"user","content":"what's the weather?"},
4511 {"role":"assistant","content":null,"tool_calls":[
4512 {"id":"call_1","type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Paris\"}"}}
4513 ]},
4514 {"role":"tool","tool_call_id":"call_1","content":"15C, cloudy"}
4515 ]
4516 }"#;
4517 let req = parse_openai_request(body, false).expect("must parse");
4518 assert_eq!(req.messages.len(), 3);
4520 let asst = &req.messages[1];
4522 assert_eq!(asst.role, "assistant");
4523 let blocks = asst.content.as_array().expect("content array");
4524 assert_eq!(blocks[0]["type"], "tool_use");
4525 assert_eq!(blocks[0]["name"], "get_weather");
4526 assert_eq!(blocks[0]["id"], "call_1");
4527 assert_eq!(blocks[0]["input"]["city"], "Paris");
4528 let tool_msg = &req.messages[2];
4530 assert_eq!(tool_msg.role, "user");
4531 let result_blocks = tool_msg.content.as_array().expect("result blocks");
4532 assert_eq!(result_blocks[0]["type"], "tool_result");
4533 assert_eq!(result_blocks[0]["tool_use_id"], "call_1");
4534 }
4535
4536 #[test]
4537 fn parse_openai_request_tools_translate_to_anthropic_format() {
4538 let body = br#"{
4539 "model":"gpt-4o",
4540 "messages":[{"role":"user","content":"use a tool"}],
4541 "tools":[{"type":"function","function":{"name":"search","description":"web search","parameters":{"type":"object","properties":{"q":{"type":"string"}}}}}]
4542 }"#;
4543 let req = parse_openai_request(body, false).expect("must parse");
4544 let tools = req.tools.as_array().expect("tools array");
4545 assert_eq!(tools.len(), 1);
4546 assert_eq!(tools[0]["name"], "search");
4547 assert_eq!(tools[0]["description"], "web search");
4548 assert_eq!(tools[0]["input_schema"]["type"], "object");
4549 }
4550
4551 #[test]
4552 fn parse_openai_request_raw_carry_preserves_full_body() {
4553 let body =
4555 br#"{"model":"gpt-4o","max_tokens":100,"messages":[{"role":"user","content":"hi"}]}"#;
4556 let req = parse_openai_request(body, true).expect("must parse");
4557 assert!(req.raw.is_object(), "raw must be the full JSON object");
4558 assert_eq!(req.raw["model"], "gpt-4o");
4559 assert_eq!(req.raw["max_tokens"], 100);
4560 assert!(req.tools.is_null(), "no tools in this request");
4562 }
4563
4564 #[test]
4565 fn parse_openai_request_http_image_returns_none() {
4566 let body = br#"{"model":"gpt-4o","messages":[{"role":"user","content":[
4568 {"type":"text","text":"describe this"},
4569 {"type":"image_url","image_url":{"url":"https://example.com/cat.png"}}
4570 ]}]}"#;
4571 let result = parse_openai_request(body, false);
4572 assert!(result.is_none(), "http image URL must fail translation");
4573 }
4574
4575 #[test]
4576 fn parse_openai_request_data_url_image_translates_to_anthropic_base64() {
4577 let body = br#"{"model":"gpt-4o","messages":[{"role":"user","content":[
4578 {"type":"text","text":"describe"},
4579 {"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgo="}}
4580 ]}]}"#;
4581 let req = parse_openai_request(body, false).expect("data URL must parse");
4582 let blocks = req.messages[0].content.as_array().expect("blocks");
4583 let img = blocks
4584 .iter()
4585 .find(|b| b["type"] == "image")
4586 .expect("image block");
4587 assert_eq!(img["source"]["type"], "base64");
4588 assert_eq!(img["source"]["media_type"], "image/png");
4589 assert_eq!(img["source"]["data"], "iVBORw0KGgo=");
4590 }
4591
4592 #[test]
4595 fn openai_response_json_renders_text_response() {
4596 let resp = ModelResponse {
4597 model: "gpt-4o".to_owned(),
4598 text: "Hello!".to_owned(),
4599 in_tokens: 10,
4600 out_tokens: 5,
4601 raw: serde_json::json!({
4602 "content": [{ "type": "text", "text": "Hello!" }]
4603 }),
4604 };
4605 let json = openai_response_json(&resp);
4606 assert_eq!(json["object"], "chat.completion");
4607 assert_eq!(json["model"], "gpt-4o");
4608 assert_eq!(json["choices"][0]["message"]["role"], "assistant");
4609 assert_eq!(json["choices"][0]["message"]["content"], "Hello!");
4610 assert_eq!(json["choices"][0]["finish_reason"], "stop");
4611 assert_eq!(json["usage"]["prompt_tokens"], 10);
4612 assert_eq!(json["usage"]["completion_tokens"], 5);
4613 }
4614
4615 #[test]
4616 fn openai_response_json_renders_tool_call() {
4617 let resp = ModelResponse {
4618 model: "gpt-4o".to_owned(),
4619 text: String::new(),
4620 in_tokens: 20,
4621 out_tokens: 15,
4622 raw: serde_json::json!({
4623 "content": [{
4624 "type": "tool_use",
4625 "id": "call_abc",
4626 "name": "search",
4627 "input": {"q": "Rust async"}
4628 }]
4629 }),
4630 };
4631 let json = openai_response_json(&resp);
4632 assert_eq!(json["choices"][0]["finish_reason"], "tool_calls");
4633 let tc = &json["choices"][0]["message"]["tool_calls"][0];
4634 assert_eq!(tc["id"], "call_abc");
4635 assert_eq!(tc["type"], "function");
4636 assert_eq!(tc["function"]["name"], "search");
4637 assert_eq!(json["choices"][0]["message"]["content"], Value::Null);
4639 }
4640
4641 #[test]
4642 fn openai_sse_from_message_plain_text_has_role_then_content_then_stop() {
4643 let resp = ModelResponse {
4644 model: "gpt-4o".to_owned(),
4645 text: "Hi there!".to_owned(),
4646 in_tokens: 5,
4647 out_tokens: 3,
4648 raw: serde_json::json!({
4649 "content": [{ "type": "text", "text": "Hi there!" }]
4650 }),
4651 };
4652 let sse = openai_sse_from_message(&openai_response_json(&resp));
4653 for line in sse.lines() {
4655 assert!(
4656 line.is_empty() || line.starts_with("data: "),
4657 "bad SSE line: {line:?}"
4658 );
4659 }
4660 let frames: Vec<&str> = sse
4661 .lines()
4662 .filter_map(|l| l.strip_prefix("data: "))
4663 .collect();
4664 assert_eq!(*frames.last().unwrap(), "[DONE]");
4666 let role_frame: Value = serde_json::from_str(frames[0]).unwrap();
4668 assert_eq!(role_frame["choices"][0]["delta"]["role"], "assistant");
4669 assert!(frames.iter().any(|f| {
4671 if *f == "[DONE]" {
4672 return false;
4673 }
4674 serde_json::from_str::<Value>(f)
4675 .ok()
4676 .is_some_and(|v| v["choices"][0]["delta"]["content"] == "Hi there!")
4677 }));
4678 assert!(frames.iter().any(|f| {
4680 if *f == "[DONE]" {
4681 return false;
4682 }
4683 serde_json::from_str::<Value>(f)
4684 .ok()
4685 .is_some_and(|v| v["choices"][0]["finish_reason"] == "stop")
4686 }));
4687 }
4688
4689 #[test]
4692 fn detects_openai_tool_calls() {
4693 let with_tool_calls = Bytes::from_static(br#"{"messages":[
4694 {"role":"user","content":"hi"},
4695 {"role":"assistant","content":null,"tool_calls":[{"id":"c1","type":"function","function":{"name":"f","arguments":"{}"}}]}
4696 ]}"#);
4697 let without = Bytes::from_static(br#"{"messages":[{"role":"user","content":"hi"}]}"#);
4698 let with_tool_msg = Bytes::from_static(
4699 br#"{"messages":[
4700 {"role":"tool","tool_call_id":"c1","content":"result"}
4701 ]}"#,
4702 );
4703 assert!(openai_messages_have_tool_calls(&with_tool_calls));
4704 assert!(!openai_messages_have_tool_calls(&without));
4705 assert!(openai_messages_have_tool_calls(&with_tool_msg));
4706 }
4707
4708 #[test]
4709 fn detects_openai_http_images() {
4710 let http_img = Bytes::from_static(
4711 br#"{"messages":[{"role":"user","content":[
4712 {"type":"image_url","image_url":{"url":"https://example.com/img.png"}}
4713 ]}]}"#,
4714 );
4715 let data_img = Bytes::from_static(
4716 br#"{"messages":[{"role":"user","content":[
4717 {"type":"image_url","image_url":{"url":"data:image/png;base64,abc"}}
4718 ]}]}"#,
4719 );
4720 let no_img = Bytes::from_static(br#"{"messages":[{"role":"user","content":"hi"}]}"#);
4721 assert!(openai_has_http_images(&http_img));
4722 assert!(!openai_has_http_images(&data_img));
4723 assert!(!openai_has_http_images(&no_img));
4724 }
4725
4726 #[test]
4727 fn enforce_can_handle_openai_inbound_all_openai_ladder() {
4728 let tools_body = Bytes::from_static(br#"{"model":"gpt-4o","messages":[{"role":"assistant","content":null,"tool_calls":[{"id":"c","type":"function","function":{"name":"f","arguments":"{}"}}]}]}"#);
4730 let f = extract_openai_features(&HeaderMap::new(), &tools_body);
4731 let ladder = vec!["openai/gpt-4o-mini".to_owned(), "openai/gpt-4o".to_owned()];
4732 let providers = crate::provider::ProviderRegistry::new("http://x", "http://x");
4733 assert!(enforce_can_handle(
4734 &f,
4735 &tools_body,
4736 true,
4737 &ladder,
4738 &providers,
4739 Dialect::Openai
4740 ));
4741 }
4742
4743 #[test]
4744 fn enforce_can_handle_openai_inbound_all_anthropic_ladder_no_http_image() {
4745 let tools_body = Bytes::from_static(br#"{"model":"gpt-4o","messages":[{"role":"assistant","content":null,"tool_calls":[{"id":"c","type":"function","function":{"name":"f","arguments":"{}"}}]}]}"#);
4747 let f = extract_openai_features(&HeaderMap::new(), &tools_body);
4748 let ladder = vec!["anthropic/claude-haiku-4-5".to_owned()];
4749 let providers = crate::provider::ProviderRegistry::new("http://x", "http://x");
4750 assert!(enforce_can_handle(
4751 &f,
4752 &tools_body,
4753 true,
4754 &ladder,
4755 &providers,
4756 Dialect::Openai,
4757 ));
4758 }
4759
4760 #[test]
4761 fn enforce_can_handle_openai_inbound_http_image_falls_back() {
4762 let img_body = Bytes::from_static(
4764 br#"{"model":"gpt-4o","messages":[{"role":"user","content":[
4765 {"type":"image_url","image_url":{"url":"https://example.com/img.png"}}
4766 ]}]}"#,
4767 );
4768 let f = extract_openai_features(&HeaderMap::new(), &img_body);
4769 let ladder = vec!["anthropic/claude-haiku-4-5".to_owned()];
4770 let providers = crate::provider::ProviderRegistry::new("http://x", "http://x");
4771 assert!(!enforce_can_handle(
4772 &f,
4773 &img_body,
4774 true,
4775 &ladder,
4776 &providers,
4777 Dialect::Openai,
4778 ));
4779 }
4780
4781 fn openai_enforce_state(mock_resp: ModelResponse) -> AppState {
4785 let toml = "[[price]]\nmodel = \"mock/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"mock/m\"]\ngates = [\"non-empty\"]\n";
4786 let config = ProxyConfig::from_lookup(|k| match k {
4787 "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
4788 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
4789 _ => None,
4790 })
4791 .unwrap();
4792 let mut outs = HashMap::new();
4793 outs.insert("mock/m".to_owned(), Ok(mock_resp));
4794 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
4795 map.insert("mock".to_owned(), Arc::new(MockProvider::new("mock", outs)));
4796 let (traces, _rx) = mpsc::channel(64);
4797 std::mem::forget(_rx);
4798 let tenant_rate_limiter = build_tenant_rate_limiter(&config);
4799 AppState {
4800 config: Arc::new(config),
4801 http: reqwest::Client::new(),
4802 providers: ProviderRegistry::from_map(map),
4803 gate_health: Arc::new(GateHealthRegistry::new()),
4804 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
4805 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
4806 traces,
4807 adaptive: None,
4808 bandit: None,
4809 predictor: None,
4810 tenant_rate_limiter,
4811 spill: None,
4812 }
4813 }
4814
4815 #[tokio::test]
4816 async fn chat_completions_plain_text_enforce_returns_openai_shape() {
4817 let mock = model_resp("mock/m", "gated answer");
4818 let state = openai_enforce_state(mock);
4819 let body = Bytes::from_static(
4820 br#"{"model":"gpt-4o","messages":[{"role":"user","content":"hello"}]}"#,
4821 );
4822 let resp = chat_completions(
4823 State(state),
4824 Extension(TenantId("default".to_owned())),
4825 HeaderMap::new(),
4826 body,
4827 )
4828 .await;
4829 assert_eq!(resp.status(), axum::http::StatusCode::OK);
4830 let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
4831 .await
4832 .unwrap();
4833 let json: Value = serde_json::from_slice(&bytes).expect("must be JSON");
4834 assert_eq!(json["object"], "chat.completion");
4835 assert_eq!(json["choices"][0]["message"]["role"], "assistant");
4836 assert_eq!(json["choices"][0]["message"]["content"], "gated answer");
4837 assert_eq!(json["choices"][0]["finish_reason"], "stop");
4838 assert!(
4839 json["id"]
4840 .as_str()
4841 .is_some_and(|id| id.starts_with("chatcmpl-"))
4842 );
4843 }
4844
4845 #[tokio::test]
4846 async fn chat_completions_stream_true_returns_sse_with_openai_chunks() {
4847 let mock = model_resp("mock/m", "gated answer");
4848 let state = openai_enforce_state(mock);
4849 let body = Bytes::from_static(
4850 br#"{"model":"gpt-4o","stream":true,"messages":[{"role":"user","content":"hello"}]}"#,
4851 );
4852 let resp = chat_completions(
4853 State(state),
4854 Extension(TenantId("default".to_owned())),
4855 HeaderMap::new(),
4856 body,
4857 )
4858 .await;
4859 assert_eq!(resp.status(), axum::http::StatusCode::OK);
4860 assert!(
4861 resp.headers()
4862 .get(axum::http::header::CONTENT_TYPE)
4863 .and_then(|v| v.to_str().ok())
4864 .is_some_and(|ct| ct.starts_with("text/event-stream")),
4865 "stream:true must return SSE"
4866 );
4867 let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
4868 .await
4869 .unwrap();
4870 let text = String::from_utf8(bytes.to_vec()).unwrap();
4871 assert!(
4873 text.contains("chat.completion.chunk"),
4874 "must have OpenAI chunk frames"
4875 );
4876 assert!(text.contains("[DONE]"), "must end with [DONE]");
4877 assert!(
4878 text.contains("gated answer"),
4879 "content must be in the stream"
4880 );
4881 assert!(
4883 !text.contains("message_start"),
4884 "must not have Anthropic event types"
4885 );
4886 }
4887
4888 fn probe_state(
4892 sample_rate: f64,
4893 k: u32,
4894 outcomes: Vec<(&str, Result<ModelResponse, ProviderError>)>,
4895 ) -> (AppState, mpsc::Receiver<Trace>) {
4896 let toml = format!(
4897 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\nladder = [\"anthropic/claude-haiku-4-5\"]\ngates = [\"non-empty\"]\n\
4898 [escalation.probe]\nk = {k}\nsample_rate = {sample_rate}\n"
4899 );
4900 let config = ProxyConfig::from_lookup(|k_| match k_ {
4901 "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
4902 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
4903 _ => None,
4904 })
4905 .unwrap();
4906 let mut outs = HashMap::new();
4907 for (model, out) in outcomes {
4908 outs.insert(model.to_owned(), out);
4909 }
4910 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
4911 map.insert(
4912 "anthropic".to_owned(),
4913 Arc::new(MockProvider::new("anthropic", outs)),
4914 );
4915 let (traces, rx) = mpsc::channel(64);
4916 let state = AppState {
4917 config: Arc::new(config),
4918 http: reqwest::Client::new(),
4919 providers: ProviderRegistry::from_map(map),
4920 gate_health: Arc::new(GateHealthRegistry::new()),
4921 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
4922 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
4923 traces,
4924 adaptive: None,
4925 bandit: None,
4926 predictor: None,
4927 tenant_rate_limiter: None,
4928 spill: None,
4929 };
4930 (state, rx)
4931 }
4932
4933 async fn run_enforce_get_trace(state: AppState, mut rx: mpsc::Receiver<Trace>) -> Trace {
4935 let resp = messages(
4936 State(state),
4937 Extension(TenantId("default".to_owned())),
4938 HeaderMap::new(),
4939 user_body(),
4940 )
4941 .await;
4942 assert_eq!(resp.status(), axum::http::StatusCode::OK);
4943 rx.try_recv().expect("trace must be enqueued")
4944 }
4945
4946 #[tokio::test]
4948 async fn probe_off_trace_has_no_probe_field() {
4949 let (state, rx) = enforce_state(
4950 &["anthropic/claude-haiku-4-5"],
4951 &["non-empty"],
4952 vec![(
4953 "anthropic/claude-haiku-4-5",
4954 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
4955 )],
4956 );
4957 assert!(
4958 state
4959 .config
4960 .routing
4961 .as_ref()
4962 .unwrap()
4963 .escalation
4964 .probe
4965 .is_none(),
4966 "probe must default to None"
4967 );
4968 let trace = run_enforce_get_trace(state, rx).await;
4969 assert!(
4970 trace.probe.is_none(),
4971 "probe=None config must not set trace.probe"
4972 );
4973 }
4974
4975 #[tokio::test]
4977 async fn probe_sample_rate_zero_never_fires() {
4978 let (state, rx) = probe_state(
4980 0.0,
4981 5,
4982 vec![(
4983 "anthropic/claude-haiku-4-5",
4984 Ok(model_resp("anthropic/claude-haiku-4-5", "hi")),
4985 )],
4986 );
4987 let trace = run_enforce_get_trace(state, rx).await;
4988 assert!(
4989 trace.probe.is_none(),
4990 "sample_rate=0.0 must never set trace.probe"
4991 );
4992 }
4993
4994 #[tokio::test]
4997 async fn probe_on_all_pass_sets_confident_pass() {
4998 let model = "anthropic/claude-haiku-4-5";
5000 let (state, mut rx) = probe_state(1.0, 3, vec![(model, Ok(model_resp(model, "hello")))]);
5001 let resp = messages(
5002 State(state),
5003 Extension(TenantId("default".to_owned())),
5004 HeaderMap::new(),
5005 user_body(),
5006 )
5007 .await;
5008 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5009 let json = body_json(resp).await;
5011 assert_eq!(
5012 json["content"][0]["text"], "hello",
5013 "served content unchanged"
5014 );
5015
5016 let trace = rx.try_recv().expect("trace enqueued");
5017 let sig = trace.probe.expect("probe must be set when sample_rate=1.0");
5018 assert_eq!(sig.k, 3);
5019 assert_eq!(sig.gate_pass_count, 3, "all 3 samples must pass non-empty");
5020 assert_eq!(
5021 sig.regime,
5022 firstpass_core::ProbeRegime::ConfidentPass,
5023 "all-pass → ConfidentPass"
5024 );
5025 assert!(
5026 sig.probe_cost_usd > 0.0,
5027 "k model calls must cost something"
5028 );
5029 }
5030
5031 #[tokio::test]
5034 async fn probe_on_all_fail_sets_confident_fail() {
5035 let model = "anthropic/claude-haiku-4-5";
5036 let (state, mut rx) = probe_state(1.0, 3, vec![(model, Ok(model_resp(model, "")))]);
5038 let resp = messages(
5040 State(state),
5041 Extension(TenantId("default".to_owned())),
5042 HeaderMap::new(),
5043 user_body(),
5044 )
5045 .await;
5046 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5047
5048 let trace = rx.try_recv().expect("trace enqueued");
5049 let sig = trace.probe.expect("probe must be set when sample_rate=1.0");
5050 assert_eq!(
5051 sig.gate_pass_count, 0,
5052 "empty response fails non-empty: all 0 pass"
5053 );
5054 assert_eq!(
5055 sig.regime,
5056 firstpass_core::ProbeRegime::ConfidentFail,
5057 "0 passes → ConfidentFail"
5058 );
5059 }
5060
5061 #[tokio::test]
5064 async fn probe_on_served_output_identical_to_probe_off() {
5065 let model = "anthropic/claude-haiku-4-5";
5066 let mk = |sample_rate: f64| {
5067 probe_state(
5068 sample_rate,
5069 2,
5070 vec![(model, Ok(model_resp(model, "gated answer")))],
5071 )
5072 };
5073
5074 let (state_off, mut rx_off) = mk(0.0);
5076 let resp_off = messages(
5077 State(state_off),
5078 Extension(TenantId("default".to_owned())),
5079 HeaderMap::new(),
5080 user_body(),
5081 )
5082 .await;
5083 let json_off = body_json(resp_off).await;
5084 let trace_off = rx_off.try_recv().unwrap();
5085
5086 let (state_on, mut rx_on) = mk(1.0);
5088 let resp_on = messages(
5089 State(state_on),
5090 Extension(TenantId("default".to_owned())),
5091 HeaderMap::new(),
5092 user_body(),
5093 )
5094 .await;
5095 let json_on = body_json(resp_on).await;
5096 let trace_on = rx_on.try_recv().unwrap();
5097
5098 assert_eq!(
5100 json_off["content"][0]["text"], json_on["content"][0]["text"],
5101 "served text must be identical regardless of probe"
5102 );
5103 assert!(
5105 (trace_off.final_.total_cost_usd - trace_on.final_.total_cost_usd).abs() < 1e-12,
5106 "total_cost_usd must not include probe cost: off={} on={}",
5107 trace_off.final_.total_cost_usd,
5108 trace_on.final_.total_cost_usd
5109 );
5110 assert!(trace_off.probe.is_none());
5112 assert!(trace_on.probe.is_some());
5113 assert!(
5115 trace_on.probe.as_ref().unwrap().probe_cost_usd > 0.0,
5116 "probe cost must be positive"
5117 );
5118 }
5119
5120 #[tokio::test]
5130 async fn probe_does_not_mutate_gate_health() {
5131 let model = "anthropic/claude-haiku-4-5";
5132 let (mut state, rx) = probe_state(1.0, 2, vec![(model, Ok(model_resp(model, "answer")))]);
5133
5134 let registry = GateHealthRegistry::new().with_budget("non-empty", 2, 0.4);
5137 registry.record("default", "non-empty", true);
5140 assert!(
5141 registry.enabled("default", "non-empty"),
5142 "gate must start enabled (window not full yet)"
5143 );
5144 state.gate_health = Arc::new(registry);
5145
5146 let trace = run_enforce_get_trace(state, rx).await;
5158 let sig = trace.probe.expect("probe must fire with sample_rate=1.0");
5159 assert_eq!(sig.k, 2);
5160 assert!(
5161 sig.gate_pass_count <= 2,
5162 "gate_pass_count must be in [0, k]"
5163 );
5164 }
5165
5166 fn predictor_state(
5168 enabled: bool,
5169 outcome: Result<ModelResponse, ProviderError>,
5170 ) -> (AppState, mpsc::Receiver<Trace>) {
5171 let toml = "[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/claude-haiku-4-5\"]\ngates = [\"non-empty\"]\n";
5172 let config = ProxyConfig::from_lookup(|k_| match k_ {
5173 "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
5174 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
5175 _ => None,
5176 })
5177 .unwrap();
5178 let mut outs = HashMap::new();
5179 outs.insert("anthropic/claude-haiku-4-5".to_owned(), outcome);
5180 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
5181 map.insert(
5182 "anthropic".to_owned(),
5183 Arc::new(MockProvider::new("anthropic", outs)),
5184 );
5185 let (traces, rx) = mpsc::channel(64);
5186 let predictor = enabled.then(|| {
5187 Arc::new(std::sync::Mutex::new(firstpass_core::PassPredictor::new(
5188 0.05, 1e-4,
5189 )))
5190 });
5191 let state = AppState {
5192 config: Arc::new(config),
5193 http: reqwest::Client::new(),
5194 providers: ProviderRegistry::from_map(map),
5195 gate_health: Arc::new(GateHealthRegistry::new()),
5196 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
5197 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
5198 traces,
5199 adaptive: None,
5200 bandit: None,
5201 predictor,
5202 tenant_rate_limiter: None,
5203 spill: None,
5204 };
5205 (state, rx)
5206 }
5207
5208 #[tokio::test]
5209 async fn predictor_off_leaves_predicted_pass_none() {
5210 let (state, rx) =
5211 predictor_state(false, Ok(model_resp("anthropic/claude-haiku-4-5", "ok")));
5212 let trace = run_enforce_get_trace(state, rx).await;
5213 assert!(
5214 trace.predicted_pass.is_none(),
5215 "predictor off => no field (byte-identical)"
5216 );
5217 let j = serde_json::to_string(&trace).unwrap();
5219 assert!(!j.contains("predicted_pass"), "None must be omitted: {j}");
5220 }
5221
5222 #[tokio::test]
5223 async fn predictor_on_records_shadow_prediction_and_serves_identically() {
5224 let (state_off, rx_off) = predictor_state(
5227 false,
5228 Ok(model_resp("anthropic/claude-haiku-4-5", "served answer")),
5229 );
5230 let off = run_enforce_get_trace(state_off, rx_off).await;
5231
5232 let (state_on, rx_on) = predictor_state(
5233 true,
5234 Ok(model_resp("anthropic/claude-haiku-4-5", "served answer")),
5235 );
5236 let on = run_enforce_get_trace(state_on, rx_on).await;
5237
5238 assert_eq!(
5239 on.final_.served_rung, off.final_.served_rung,
5240 "served rung identical"
5241 );
5242 assert_eq!(on.attempts.len(), off.attempts.len(), "same attempts");
5243 assert_eq!(
5244 on.final_.total_cost_usd, off.final_.total_cost_usd,
5245 "predictor never adds served cost"
5246 );
5247 let p = on
5248 .predicted_pass
5249 .expect("predictor on => predicted_pass recorded");
5250 assert!(p > 0.0 && p < 1.0, "shadow prediction in (0,1): {p}");
5251 }
5252
5253 fn rollout_state(
5255 percent: f64,
5256 key: &str,
5257 outcomes: Vec<(&str, Result<ModelResponse, ProviderError>)>,
5258 ) -> (AppState, mpsc::Receiver<Trace>) {
5259 let toml = format!(
5260 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
5261 ladder = [\"anthropic/claude-haiku-4-5\"]\ngates = [\"non-empty\"]\n\
5262 [route.rollout]\npercent = {percent}\nkey = \"{key}\"\n"
5263 );
5264 let config = ProxyConfig::from_lookup(|k| match k {
5265 "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
5266 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
5267 _ => None,
5268 })
5269 .unwrap();
5270 let mut outs = HashMap::new();
5271 for (model, out) in outcomes {
5272 outs.insert(model.to_owned(), out);
5273 }
5274 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
5275 map.insert(
5276 "anthropic".to_owned(),
5277 Arc::new(MockProvider::new("anthropic", outs)),
5278 );
5279 let (traces, rx) = mpsc::channel(64);
5280 let state = AppState {
5281 config: Arc::new(config),
5282 http: reqwest::Client::new(),
5283 providers: ProviderRegistry::from_map(map),
5284 gate_health: Arc::new(GateHealthRegistry::new()),
5285 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
5286 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
5287 traces,
5288 adaptive: None,
5289 bandit: None,
5290 predictor: None,
5291 tenant_rate_limiter: None,
5292 spill: None,
5293 };
5294 (state, rx)
5295 }
5296
5297 #[tokio::test]
5301 async fn rollout_at_zero_percent_never_enforces() {
5302 for i in 0..25 {
5303 let (state, mut rx) = rollout_state(
5304 0.0,
5305 "session",
5306 vec![(
5307 "anthropic/claude-haiku-4-5",
5308 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
5309 )],
5310 );
5311 let mut headers = HeaderMap::new();
5312 headers.insert("x-firstpass-session", format!("s{i}").parse().unwrap());
5313 let resp = messages(
5314 State(state),
5315 Extension(TenantId("default".to_owned())),
5316 headers,
5317 user_body(),
5318 )
5319 .await;
5320 let _ = resp;
5323 if let Ok(t) = rx.try_recv() {
5324 assert!(
5325 t.attempts.is_empty(),
5326 "0% rollout produced an enforced decision for session s{i}"
5327 );
5328 }
5329 }
5330 }
5331
5332 #[tokio::test]
5335 async fn rollout_at_hundred_percent_always_enforces() {
5336 for i in 0..25 {
5337 let (state, mut rx) = rollout_state(
5338 100.0,
5339 "session",
5340 vec![(
5341 "anthropic/claude-haiku-4-5",
5342 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
5343 )],
5344 );
5345 let mut headers = HeaderMap::new();
5346 headers.insert("x-firstpass-session", format!("s{i}").parse().unwrap());
5347 let resp = messages(
5348 State(state),
5349 Extension(TenantId("default".to_owned())),
5350 headers,
5351 user_body(),
5352 )
5353 .await;
5354 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5355 let t = rx.try_recv().expect("trace enqueued");
5356 assert!(
5357 !t.attempts.is_empty(),
5358 "100% rollout skipped enforcement for session s{i}"
5359 );
5360 }
5361 }
5362
5363 #[tokio::test]
5371 async fn dispatch_arm_matches_the_predicted_arm_per_session() {
5372 let mut checked_enforced = 0;
5373 let mut checked_control = 0;
5374 for i in 0..24 {
5375 let session = format!("session-{i}");
5376 let (state, mut rx) = rollout_state(
5377 50.0,
5378 "session",
5379 vec![(
5380 "anthropic/claude-haiku-4-5",
5381 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
5382 )],
5383 );
5384 let expected = firstpass_core::rollout::decide(
5387 &state.config.prompt_salt,
5388 &firstpass_core::Rollout {
5389 percent: 50.0,
5390 key: firstpass_core::RolloutKey::Session,
5391 },
5392 &session,
5393 )
5394 .enforced;
5395
5396 let mut headers = HeaderMap::new();
5397 headers.insert("x-firstpass-session", session.parse().unwrap());
5398 let _ = messages(
5399 State(state),
5400 Extension(TenantId("default".to_owned())),
5401 headers,
5402 user_body(),
5403 )
5404 .await;
5405 let observed = rx
5406 .try_recv()
5407 .map(|t| !t.attempts.is_empty())
5408 .unwrap_or(false);
5409
5410 assert_eq!(
5411 observed,
5412 expected,
5413 "{session}: dispatch put it in the {} arm, bucketing predicted the {} arm",
5414 if observed { "enforced" } else { "control" },
5415 if expected { "enforced" } else { "control" }
5416 );
5417 if expected {
5418 checked_enforced += 1;
5419 } else {
5420 checked_control += 1;
5421 }
5422 }
5423 assert!(
5425 checked_enforced > 0 && checked_control > 0,
5426 "test saw only one arm ({checked_enforced} enforced, {checked_control} control)"
5427 );
5428 }
5429}