1use std::sync::Arc;
5use std::time::Instant;
6
7use axum::body::Body;
8use axum::extract::{Request, State};
9use axum::http::HeaderMap;
10use axum::middleware::Next;
11use axum::response::{IntoResponse, Response};
12use axum::routing::{get, post};
13use axum::{Extension, Json, Router};
14use bytes::Bytes;
15use firstpass_core::features::{hour_bucket, token_bucket};
16use firstpass_core::hashchain::sha256_hex;
17use firstpass_core::{
18 Attempt, DeferredVerdict, Dialect, FEATURE_VERSION, Features, FinalOutcome, GENESIS_HASH, Mode,
19 ModelRef, PolicyRef, ProbeRegime, ProbeSignal, RequestInfo, RoutingMode, Score, ServedFrom,
20 TaskKind, Trace, Verdict,
21};
22use serde::Deserialize;
23use serde_json::Value;
24use std::future::Future;
25use std::time::Duration;
26use tokio::sync::mpsc::error::TrySendError;
27use uuid::Uuid;
28
29use crate::config::ProxyConfig;
30use crate::error::ProxyError;
31use crate::gate::{GateHealthRegistry, aggregate_with_policy, resolve_gates};
32use crate::provider::{Auth, ChatMessage, ModelRequest, ModelResponse, ProviderRegistry};
33use crate::router::{EnforceCtx, EngineOutcome, route_enforce};
34use crate::store;
35use crate::tenant_auth::{TenantId, auth_middleware};
36use crate::upstream::{forward_anthropic, forward_anthropic_streaming};
37use firstpass_core::Route;
38use firstpass_core::trace::ShadowSignal;
39
40#[derive(Clone)]
43pub struct AppState {
44 pub config: Arc<ProxyConfig>,
46 pub http: reqwest::Client,
48 pub providers: ProviderRegistry,
50 pub gate_health: Arc<GateHealthRegistry>,
52 pub shadow_ledger: Arc<crate::shadow::ShadowLedger>,
55 pub guardrails: Arc<crate::guard::GuardrailRegistry>,
58 pub traces: store::TraceSender,
60 pub adaptive: Option<Arc<std::sync::Mutex<firstpass_core::conformal::AdaptiveConformal>>>,
64 pub bandit: Option<Arc<std::sync::Mutex<crate::bandit::StartRungBandit>>>,
69 pub promoter: Option<Arc<crate::affinity::SessionPromoter>>,
74 pub predictor: Option<Arc<std::sync::Mutex<firstpass_core::PassPredictor>>>,
80 pub tenant_rate_limiter: Option<Arc<governor::DefaultKeyedRateLimiter<String>>>,
84 pub spill: Option<store::SpillHandle>,
88}
89
90#[must_use]
106fn routing_cfg_condense(state: &AppState) -> Option<&firstpass_core::config::CondenseConfig> {
107 state
108 .config
109 .routing
110 .as_ref()
111 .and_then(|r| r.escalation.condense.as_ref())
112}
113
114pub fn build_promoter(
115 config: &ProxyConfig,
116) -> firstpass_core::Result<Option<Arc<crate::affinity::SessionPromoter>>> {
117 match config
118 .routing
119 .as_ref()
120 .and_then(|r| r.escalation.session_promotion.as_ref())
121 {
122 Some(sp) => Ok(Some(Arc::new(crate::affinity::SessionPromoter::new(
123 sp.clone(),
124 )?))),
125 None => Ok(None),
126 }
127}
128
129#[must_use]
130pub fn build_tenant_rate_limiter(
131 config: &ProxyConfig,
132) -> Option<Arc<governor::DefaultKeyedRateLimiter<String>>> {
133 let per_sec = config.tenant_rate_per_sec?;
134 Some(Arc::new(governor::RateLimiter::keyed(
135 governor::Quota::per_second(per_sec),
136 )))
137}
138
139pub async fn tenant_rate_limit_middleware(
143 State(state): State<AppState>,
144 Extension(tenant): Extension<TenantId>,
145 req: Request,
146 next: Next,
147) -> Response {
148 if let Some(limiter) = &state.tenant_rate_limiter
149 && limiter.check_key(&tenant.0).is_err()
150 {
151 return ProxyError::RateLimited.into_response();
152 }
153 next.run(req).await
154}
155
156impl std::fmt::Debug for AppState {
157 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
158 f.debug_struct("AppState")
159 .field("config", &self.config)
160 .finish_non_exhaustive()
161 }
162}
163
164fn offer_trace(traces: &store::TraceSender, spill: Option<&store::SpillHandle>, trace: Trace) {
181 record_trace_metrics(&trace);
182 match traces.try_send(trace) {
183 Ok(()) => {}
184 Err(TrySendError::Full(t)) => {
185 if let Some(handle) = spill {
186 match store::append_to_spill(handle, &t) {
187 Ok(()) => {
188 metrics::counter!("firstpass_receipts_spilled_total").increment(1);
189 }
190 Err(e) => {
191 tracing::error!(%e, "durable mode: spill write failed; trace lost");
192 metrics::counter!("firstpass_traces_dropped_total").increment(1);
193 }
194 }
195 } else {
196 tracing::warn!("trace channel full; dropping trace (writer behind under load)");
197 metrics::counter!("firstpass_traces_dropped_total").increment(1);
198 }
199 }
200 Err(TrySendError::Closed(_)) => {
201 tracing::warn!("trace writer is gone; dropping trace");
202 }
203 }
204}
205
206fn record_trace_metrics(trace: &Trace) {
210 if trace.mode == Mode::Enforce {
211 metrics::histogram!("firstpass_enforce_latency_ms")
212 .record(trace.final_.total_latency_ms as f64);
213 if trace.final_.escalations > 0 {
214 metrics::counter!("firstpass_escalations_total")
215 .increment(u64::from(trace.final_.escalations));
216 }
217 }
218 let served_from = match trace.final_.served_from {
219 ServedFrom::Attempt => "attempt",
220 ServedFrom::BestAttempt => "best_attempt",
221 ServedFrom::Error => "error",
222 };
223 metrics::counter!("firstpass_served_total", "served_from" => served_from).increment(1);
224 if trace.final_.served_from == ServedFrom::Error {
225 metrics::counter!("firstpass_upstream_failures_total").increment(1);
226 }
227 metrics::gauge!("firstpass_cost_usd_total").increment(trace.final_.total_cost_usd);
231 metrics::gauge!("firstpass_gate_cost_usd_total").increment(trace.final_.gate_cost_usd);
232 metrics::gauge!("firstpass_baseline_usd_total")
233 .increment(trace.final_.counterfactual_baseline_usd);
234 metrics::gauge!("firstpass_savings_usd_total").increment(trace.final_.savings_usd);
235 if let Some(rung) = trace.final_.served_rung {
237 let model = trace
238 .attempts
239 .iter()
240 .find(|a| a.rung == rung)
241 .map(|a| a.model.clone())
242 .unwrap_or_else(|| "unknown".to_owned());
243 metrics::counter!(
244 "firstpass_served_rung_total",
245 "rung" => rung.to_string(),
246 "model" => model
247 )
248 .increment(1);
249 }
250}
251
252const MAX_BODY_BYTES: usize = 16 * 1024 * 1024;
256
257pub(crate) fn u01(seed: u128) -> f64 {
268 let lo = splitmix64_finalise(seed as u64);
269 let hi = splitmix64_finalise((seed >> 64) as u64);
270 ((lo ^ hi) >> 11) as f64 * (1.0_f64 / (1u64 << 53) as f64)
272}
273
274fn splitmix64_finalise(mut z: u64) -> u64 {
275 z = z.wrapping_add(0x9E37_79B9_7F4A_7C15);
276 z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
277 z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
278 z ^ (z >> 31)
279}
280
281#[must_use]
288pub(crate) fn epsilon_propensity(chosen: u32, greedy: u32, epsilon: f64, k: usize) -> f64 {
289 let greedy_term = if chosen == greedy { 1.0 - epsilon } else { 0.0 };
290 greedy_term + epsilon / k as f64
291}
292
293pub fn app(state: AppState) -> Result<Router, ProxyError> {
300 crate::metrics::install()?;
301 let max_concurrency = state.config.max_concurrency;
302
303 let business = Router::new()
312 .route("/v1/messages", post(messages))
313 .route("/v1/chat/completions", post(chat_completions))
314 .route("/v1/responses", post(responses))
315 .route("/v1/feedback", post(feedback))
316 .route("/v1/capabilities", get(capabilities))
317 .route("/v1/models", get(models))
318 .route("/v1/receipts", get(receipts))
321 .layer(axum::middleware::from_fn_with_state(
322 state.clone(),
323 tenant_rate_limit_middleware,
324 ))
325 .layer(axum::middleware::from_fn_with_state(
326 state.clone(),
327 auth_middleware,
328 ));
329
330 Ok(Router::new()
331 .merge(business)
332 .route("/healthz", get(healthz))
333 .route("/metrics", get(crate::metrics::handler))
334 .route("/panel", get(panel))
337 .layer(axum::extract::DefaultBodyLimit::max(MAX_BODY_BYTES))
339 .layer(tower::limit::GlobalConcurrencyLimitLayer::new(
342 max_concurrency,
343 ))
344 .with_state(state))
345}
346
347async fn healthz() -> impl IntoResponse {
349 Json(serde_json::json!({
355 "status": "ok",
356 "service": "firstpass",
357 "version": env!("CARGO_PKG_VERSION"),
358 }))
359}
360
361const PANEL_HTML: &str = include_str!("panel.html");
365
366async fn receipts(
372 State(state): State<AppState>,
373 Extension(TenantId(tenant)): Extension<TenantId>,
374 axum::extract::Query(q): axum::extract::Query<ReceiptsQuery>,
375) -> Response {
376 let limit = q.limit.unwrap_or(20).clamp(1, 200);
378 let db = state.config.db_path.clone();
379 let traces =
380 match tokio::task::spawn_blocking(move || crate::store::load_tenant_traces(&db, &tenant))
381 .await
382 {
383 Ok(Ok(t)) => t,
384 _ => Vec::new(),
387 };
388 let items: Vec<serde_json::Value> = traces
389 .iter()
390 .rev()
391 .take(limit)
392 .map(|t| {
393 let f = &t.final_;
394 serde_json::json!({
395 "trace_id": t.trace_id,
396 "served_rung": f.served_rung,
397 "served_model": f.served_rung
402 .and_then(|r| t.attempts.iter().find(|a| a.rung == r))
403 .map(|a| a.model.clone()),
404 "attempts": t.attempts.iter().map(|a| serde_json::json!({
405 "rung": a.rung,
406 "model": a.model,
407 "verdict": a.verdict,
408 "cost_usd": a.cost_usd,
409 "latency_ms": a.latency_ms,
410 })).collect::<Vec<_>>(),
411 "total_cost_usd": f.total_cost_usd,
412 "baseline_usd": f.counterfactual_baseline_usd,
413 "savings_usd": f.savings_usd,
414 })
415 })
416 .collect();
417 axum::Json(serde_json::json!({ "receipts": items })).into_response()
418}
419
420#[derive(Debug, serde::Deserialize)]
422struct ReceiptsQuery {
423 limit: Option<usize>,
425}
426
427async fn panel() -> impl IntoResponse {
433 (
434 [(axum::http::header::CONTENT_TYPE, "text/html; charset=utf-8")],
435 PANEL_HTML,
436 )
437}
438
439async fn capabilities(State(state): State<AppState>) -> impl IntoResponse {
440 let (ladder, gates) = state
443 .config
444 .routing
445 .as_ref()
446 .and_then(|c| c.routes.iter().find(|r| r.mode == Mode::Enforce))
447 .map(|r| (r.ladder.clone(), r.gates.clone()))
448 .unwrap_or_default();
449 let routing_modes: Vec<serde_json::Value> = RoutingMode::ALL
450 .iter()
451 .map(|m| {
452 let p = m.preset();
453 serde_json::json!({
454 "name": m.as_str(),
455 "description": p.description,
456 "tradeoff": p.tradeoff,
457 })
458 })
459 .collect();
460 Json(serde_json::json!({
461 "service": "firstpass",
462 "version": env!("CARGO_PKG_VERSION"),
463 "feature_version": FEATURE_VERSION,
464 "modes": ["observe", "enforce"],
465 "routing_modes": routing_modes,
466 "wire_apis": ["anthropic.messages", "openai.chat_completions", "openai.responses"],
467 "ladder": ladder,
468 "gates": gates,
469 "feedback_api": "POST /v1/feedback",
470 "offboarding": "unset ANTHROPIC_BASE_URL (or OPENAI_BASE_URL for OpenAI clients)",
471 }))
472}
473
474async fn models(State(state): State<AppState>) -> impl IntoResponse {
486 Json(model_list(state.config.routing.as_ref()))
487}
488
489fn model_list(routing: Option<&firstpass_core::config::Config>) -> serde_json::Value {
492 let prices = routing.map(|c| c.price_table()).unwrap_or_default();
493 let mut seen = std::collections::HashSet::new();
496 let data: Vec<serde_json::Value> = routing
497 .map(|c| c.routes.as_slice())
498 .unwrap_or_default()
499 .iter()
500 .flat_map(|r| r.ladder.iter())
501 .filter(|id| seen.insert(id.as_str()))
502 .map(|id| {
503 let price = prices.get(id);
504 serde_json::json!({
505 "id": id,
506 "object": "model",
507 "created": 0,
510 "owned_by": id.split_once('/').map_or("firstpass", |(p, _)| p),
511 "firstpass": {
512 "input_per_mtok": price.map(|p| p.input_per_mtok),
513 "output_per_mtok": price.map(|p| p.output_per_mtok),
514 },
515 })
516 })
517 .collect();
518 serde_json::json!({ "object": "list", "data": data })
519}
520
521#[derive(Debug, Deserialize)]
523struct FeedbackRequest {
524 trace_id: String,
526 gate_id: String,
528 verdict: String,
530 #[serde(default)]
532 score: Option<f64>,
533 reporter: String,
535}
536
537async fn feedback(
542 State(state): State<AppState>,
543 Extension(TenantId(tenant)): Extension<TenantId>,
544 body: Bytes,
545) -> Response {
546 let req: FeedbackRequest = match serde_json::from_slice(&body) {
547 Ok(r) => r,
548 Err(e) => {
549 return ProxyError::BadRequest(format!("invalid feedback body: {e}")).into_response();
550 }
551 };
552 let verdict = match req.verdict.as_str() {
553 "pass" => Verdict::Pass,
554 "fail" => Verdict::Fail,
555 "abstain" => Verdict::Abstain,
556 other => {
557 return ProxyError::BadRequest(format!("unknown verdict {other:?}")).into_response();
558 }
559 };
560 let score = match req.score {
561 Some(s) => match Score::new(s) {
562 Ok(sc) => Some(sc),
563 Err(_) => {
564 return ProxyError::BadRequest(format!("score {s} out of range [0,1]"))
565 .into_response();
566 }
567 },
568 None => None,
569 };
570
571 let db = state.config.db_path.clone();
572
573 let (db_check, tenant_check, tid_check) = (db.clone(), tenant.clone(), req.trace_id.clone());
578 match tokio::task::spawn_blocking(move || {
579 store::trace_exists(&db_check, &tenant_check, &tid_check)
580 })
581 .await
582 {
583 Ok(Ok(true)) => {}
584 Ok(Ok(false)) => {
585 return ProxyError::NotFound(format!("unknown trace_id {:?}", req.trace_id))
586 .into_response();
587 }
588 Ok(Err(e)) => {
589 tracing::error!(%e, "feedback: trace_exists check failed");
590 return ProxyError::Internal(e.to_string()).into_response();
591 }
592 Err(e) => {
593 tracing::error!(%e, "feedback: trace_exists task panicked");
594 return ProxyError::Internal(e.to_string()).into_response();
595 }
596 }
597
598 let feedback_signal = match verdict {
600 Verdict::Pass => Some(true),
601 Verdict::Fail => Some(false),
602 Verdict::Abstain => None,
603 };
604 let dv = DeferredVerdict {
605 gate_id: req.gate_id,
606 verdict,
607 score,
608 reported_at: jiff::Timestamp::now(),
609 reporter: req.reporter,
610 };
611 let trace_id = req.trace_id.clone();
612 match tokio::task::spawn_blocking(move || store::append_deferred(&db, &req.trace_id, &dv)).await
613 {
614 Ok(Ok(())) => {
615 if let (Some(a), Some(correct)) = (state.adaptive.as_ref(), feedback_signal)
617 && let Ok(mut g) = a.lock()
618 {
619 g.observe_served(correct);
620 metrics::gauge!("firstpass_serve_threshold").set(g.threshold());
621 metrics::gauge!("firstpass_realized_served_failure")
622 .set(g.realized_served_failure());
623 }
624
625 let attributed_route = tokio::task::spawn_blocking({
630 let (db, ten, tid) = (
631 state.config.db_path.clone(),
632 tenant.clone(),
633 trace_id.clone(),
634 );
635 move || store::trace_route(&db, &ten, &tid)
636 })
637 .await
638 .ok()
639 .and_then(Result::ok)
640 .flatten()
641 .flatten()
642 .unwrap_or(0) as usize;
643
644 if let (Some(cfg), Some(correct)) = (
649 state.config.routing.as_ref().and_then(|r| r.guardrail),
650 feedback_signal,
651 ) {
652 let cooldown = state
653 .config
654 .routing
655 .as_ref()
656 .map_or(3_600, |r| r.guardrail_cooldown_secs);
657 let reaction = state.guardrails.record(
658 &tenant,
659 attributed_route,
660 &cfg,
661 correct,
662 jiff::Timestamp::now().as_second(),
663 cooldown,
664 );
665 match &reaction {
666 crate::guard::Reaction::Demoted(v) => {
667 metrics::counter!("firstpass_guardrail_demotions_total").increment(1);
668 tracing::error!(
669 tenant = %tenant,
670 n = v.n,
671 rate = v.rate,
672 bound = v.bound,
673 alpha = cfg.alpha,
674 "GUARDRAIL: served-failure bound exceeded target — route demoted to observe; traffic now serves as it would without Firstpass"
675 );
676 }
677 crate::guard::Reaction::Alarmed(v) => {
678 metrics::counter!("firstpass_guardrail_breaches_total").increment(1);
679 tracing::error!(
680 tenant = %tenant,
681 n = v.n,
682 rate = v.rate,
683 bound = v.bound,
684 alpha = cfg.alpha,
685 "GUARDRAIL: served-failure bound exceeded target (alarm only — routing unchanged)"
686 );
687 }
688 crate::guard::Reaction::None => {}
689 }
690 }
691 (
692 axum::http::StatusCode::ACCEPTED,
693 Json(serde_json::json!({ "status": "recorded", "trace_id": trace_id })),
694 )
695 .into_response()
696 }
697 Ok(Err(e)) => {
698 tracing::error!(%e, "feedback: append_deferred failed");
699 ProxyError::Internal(e.to_string()).into_response()
700 }
701 Err(e) => {
702 tracing::error!(%e, "feedback: append_deferred task panicked");
703 ProxyError::Internal(e.to_string()).into_response()
704 }
705 }
706}
707
708const SESSION_HEADER: &str = "x-firstpass-session";
711
712const AGENT_HEADER: &str = "x-firstpass-agent";
714const SUBAGENT_HEADER: &str = "x-firstpass-subagent";
716#[expect(
722 clippy::too_many_arguments,
723 reason = "mirrors the enforce context it evaluates; a wrapper struct would only move the list"
724)]
725fn spawn_shadow(
726 state: &AppState,
727 route: &firstpass_core::Route,
728 route_ix: usize,
729 body: Bytes,
730 auth: Auth,
731 features: Features,
732 tenant: String,
733 session_id: String,
734 api: &str,
735) {
736 let Some(shadow) = route.shadow else {
737 return;
738 };
739 if !shadow.sampled(&state.config.prompt_salt, &session_id) {
742 return;
743 }
744 let (state, route, api) = (state.clone(), route.clone(), api.to_owned());
745 tokio::spawn(async move {
746 let signal = evaluate_shadow(
747 &state, &route, shadow, route_ix, &body, auth, &features, tenant, session_id, &api, 0.0,
748 )
749 .await;
750 tracing::debug!(
752 would_pass = signal.would_pass,
753 projected_usd = signal.projected_cost_usd,
754 skipped = ?signal.skipped,
755 "shadow evaluation complete"
756 );
757 });
758}
759
760#[expect(
767 clippy::too_many_arguments,
768 reason = "mirrors the enforce context; grouping them into a struct would only move the list"
769)]
770async fn evaluate_shadow(
771 state: &AppState,
772 route: &firstpass_core::Route,
773 shadow: firstpass_core::rollout::Shadow,
774 route_ix: usize,
775 body: &Bytes,
776 auth: Auth,
777 features: &Features,
778 tenant: String,
779 session_id: String,
780 api: &str,
781 actual_cost_usd: f64,
782) -> ShadowSignal {
783 let skipped = |why: &str| ShadowSignal {
784 would_serve_rung: None,
785 would_pass: false,
786 projected_cost_usd: 0.0,
787 actual_cost_usd,
788 skipped: Some(why.to_owned()),
789 };
790
791 let now = jiff::Timestamp::now();
792 if !state
793 .shadow_ledger
794 .may_spend(&tenant, route_ix, shadow.max_usd_per_day, now)
795 {
796 return skipped("budget_exhausted");
799 }
800
801 let Some(base_request) = parse_model_request(body) else {
802 return skipped("unparseable_request");
803 };
804 let gate_defs = state
805 .config
806 .routing
807 .as_ref()
808 .map_or(&[][..], |cfg| &cfg.gate_defs);
809 let gates = resolve_gates(
810 &route.gates,
811 gate_defs,
812 &state.providers,
813 &auth,
814 &state.config.prices,
815 );
816 let (budget, max_rungs) = state
817 .config
818 .routing
819 .as_ref()
820 .map_or((None, u32::MAX), |cfg| {
821 (
822 cfg.budget.per_request_usd,
823 cfg.escalation.max_rungs_per_request,
824 )
825 });
826
827 let mut base_request = base_request;
828 base_request.cache_prefix = state
832 .config
833 .routing
834 .as_ref()
835 .is_some_and(|r| r.escalation.prompt_cache);
836 let ctx = EnforceCtx {
837 condense: routing_cfg_condense(state),
838 ladder: &route.ladder,
839 gates: &gates,
840 health: &state.gate_health,
841 base_request: &base_request,
842 providers: &state.providers,
843 auth: &auth,
844 prices: &state.config.prices,
845 budget_per_request_usd: budget,
846 max_rungs,
847 speculation: 0,
850 serve_threshold: None,
851 elastic: None,
852 features: features.clone(),
853 start_rung: 0,
854 tenant_id: tenant.clone(),
855 session_id,
856 prompt_hash: prompt_hash(&state.config.prompt_salt, body),
857 api: api.to_owned(),
858 policy_id: "shadow".to_owned(),
859 };
860
861 let (outcome, trace) = route_enforce(ctx).await;
862 let spent = trace.final_.total_cost_usd;
863 state.shadow_ledger.debit(&tenant, route_ix, spent, now);
864
865 let would_pass = matches!(outcome, EngineOutcome::Served(_));
868 let would_serve_rung = if would_pass {
869 trace.final_.served_rung
870 } else {
871 None
872 };
873 ShadowSignal {
874 would_serve_rung,
875 would_pass,
876 projected_cost_usd: spent,
877 actual_cost_usd,
878 skipped: None,
879 }
880}
881
882const MODE_PROFILE_HEADER: &str = "x-firstpass-mode";
885
886fn resolve_mode(headers: &HeaderMap, route: &Route, config: &ProxyConfig) -> RoutingMode {
895 if let Some(val) = header_str(headers, MODE_PROFILE_HEADER) {
897 match val.trim().to_ascii_lowercase().as_str() {
898 "observe" => return RoutingMode::Observe,
899 "cost" => return RoutingMode::Cost,
900 "balanced" => return RoutingMode::Balanced,
901 "quality" => return RoutingMode::Quality,
902 "latency" => return RoutingMode::Latency,
903 "max" => return RoutingMode::Max,
904 other => {
905 tracing::warn!(
906 value = other,
907 "unknown x-firstpass-mode value; ignoring \
908 (valid: observe|cost|balanced|quality|latency|max)"
909 );
910 }
911 }
912 }
913 if let Some(m) = route.routing_mode {
915 return m;
916 }
917 config.default_routing_mode
919}
920
921async fn messages(
926 State(state): State<AppState>,
927 Extension(TenantId(tenant)): Extension<TenantId>,
928 headers: HeaderMap,
929 body: Bytes,
930) -> Response {
931 let session_header = header_str(&headers, SESSION_HEADER);
932
933 if let Some(routing) = state.config.routing.as_ref() {
936 let features = extract_features(&headers, &body);
937 if let Some(route) = routing
938 .route_for(&features)
939 .filter(|r| r.mode == Mode::Enforce && !r.ladder.is_empty())
940 {
941 let route_ix = routing
944 .routes
945 .iter()
946 .position(|r| std::ptr::eq(r, route))
947 .unwrap_or(0);
948 let route = route.clone();
951 let routing_mode = resolve_mode(&headers, &route, &state.config);
953 if routing_mode == RoutingMode::Observe {
955 return observe_passthrough(state, headers, body, session_header, tenant).await;
956 }
957 if state
961 .guardrails
962 .is_demoted(&tenant, route_ix, jiff::Timestamp::now().as_second())
963 {
964 return observe_passthrough(state, headers, body, session_header, tenant).await;
965 }
966 if let Some(rollout) = route.rollout.as_ref() {
971 let key_value = match rollout.key {
972 firstpass_core::RolloutKey::Session => session_header
973 .clone()
974 .unwrap_or_else(|| firstpass_core::rollout::request_identity(&body)),
975 firstpass_core::RolloutKey::Request => {
981 firstpass_core::rollout::request_identity(&body)
982 }
983 firstpass_core::RolloutKey::Tenant => tenant.to_string(),
984 };
985 let decision =
986 firstpass_core::rollout::decide(&state.config.prompt_salt, rollout, &key_value);
987 if !decision.enforced {
988 let shadow_ctx = route.shadow.map(|_| {
990 (
991 body.clone(),
992 Auth::from_headers(&headers),
993 features.clone(),
994 tenant.clone(),
995 session_header
996 .clone()
997 .unwrap_or_else(|| Uuid::now_v7().to_string()),
998 )
999 });
1000 let resp =
1001 observe_passthrough(state.clone(), headers, body, session_header, tenant)
1002 .await;
1003 if let Some((b, auth, feats, ten, sess)) = shadow_ctx {
1006 spawn_shadow(
1007 &state,
1008 &route,
1009 route_ix,
1010 b,
1011 auth,
1012 feats,
1013 ten,
1014 sess,
1015 "anthropic",
1016 );
1017 }
1018 return resp;
1019 }
1020 }
1021 if enforce_can_handle(
1022 &features,
1023 &body,
1024 routing.escalation.enforce_structured,
1025 &route.ladder,
1026 &state.providers,
1027 Dialect::Anthropic,
1028 ) {
1029 return handle_enforce(
1030 &state,
1031 &headers,
1032 &body,
1033 features,
1034 &route,
1035 route_ix,
1036 session_header,
1037 tenant,
1038 routing_mode,
1039 )
1040 .await;
1041 }
1042 tracing::info!(
1046 "enforce route matched but structured request can't be routed faithfully (flag/ladder); serving via observe passthrough"
1047 );
1048 }
1049 }
1050 observe_passthrough(state, headers, body, session_header, tenant).await
1051}
1052
1053fn enforce_can_handle(
1067 features: &Features,
1068 body: &[u8],
1069 enforce_structured: bool,
1070 ladder: &[String],
1071 providers: &crate::provider::ProviderRegistry,
1072 inbound: Dialect,
1073) -> bool {
1074 let structured = features.tool_count > 0
1075 || features.has_images
1076 || match inbound {
1077 Dialect::Anthropic => messages_have_tool_blocks(body),
1078 Dialect::Openai => openai_messages_have_tool_calls(body),
1079 Dialect::Gemini => false,
1080 };
1081 if !structured {
1082 return true;
1083 }
1084 if !enforce_structured {
1085 return false;
1086 }
1087 let all_verbatim = ladder.iter().all(|rung| {
1089 let provider_id = rung.split('/').next().unwrap_or_default();
1090 providers
1091 .get(provider_id)
1092 .is_some_and(|p| p.carries_structured_verbatim(inbound))
1093 });
1094 if all_verbatim {
1095 return true;
1096 }
1097 if inbound == Dialect::Openai && !openai_has_http_images(body) {
1101 let all_anthropic = ladder.iter().all(|rung| {
1102 let pid = rung.split('/').next().unwrap_or_default();
1103 providers
1104 .get(pid)
1105 .is_some_and(|p| p.carries_structured_verbatim(Dialect::Anthropic))
1106 });
1107 if all_anthropic {
1108 return true;
1109 }
1110 }
1111 false
1112}
1113
1114fn messages_have_tool_blocks(body: &[u8]) -> bool {
1117 serde_json::from_slice::<Value>(body)
1118 .ok()
1119 .and_then(|json| {
1120 json.get("messages")
1121 .and_then(Value::as_array)
1122 .map(|messages| messages.iter().any(message_has_tool_block))
1123 })
1124 .unwrap_or(false)
1125}
1126
1127fn message_has_tool_block(message: &Value) -> bool {
1129 message
1130 .get("content")
1131 .and_then(Value::as_array)
1132 .is_some_and(|blocks| {
1133 blocks.iter().any(|block| {
1134 matches!(
1135 block.get("type").and_then(Value::as_str),
1136 Some("tool_use" | "tool_result")
1137 )
1138 })
1139 })
1140}
1141
1142fn is_stream_request(body: &[u8]) -> bool {
1144 serde_json::from_slice::<Value>(body)
1145 .ok()
1146 .and_then(|json| json.get("stream").and_then(Value::as_bool))
1147 .unwrap_or(false)
1148}
1149
1150fn header_str(headers: &HeaderMap, name: &str) -> Option<String> {
1152 headers
1153 .get(name)
1154 .and_then(|v| v.to_str().ok())
1155 .map(str::to_owned)
1156}
1157
1158fn extract_features(headers: &HeaderMap, body: &[u8]) -> Features {
1161 let (_model, tool_count, has_images) = request_features(body);
1162 let mut f = Features::new(TaskKind::Other);
1163 f.agent = header_str(headers, AGENT_HEADER);
1164 f.subagent = header_str(headers, SUBAGENT_HEADER);
1165 f.tool_count = tool_count;
1166 f.has_images = has_images;
1167 f.prompt_token_bucket = token_bucket(body.len() as u64);
1170 f.hour_bucket = hour_bucket(jiff::Timestamp::now());
1171 f
1172}
1173
1174#[allow(clippy::too_many_arguments)]
1177async fn handle_enforce(
1178 state: &AppState,
1179 headers: &HeaderMap,
1180 body: &Bytes,
1181 features: Features,
1182 route: &Route,
1183 route_ix: usize,
1186 session_header: Option<String>,
1187 tenant: String,
1188 routing_mode: RoutingMode,
1189) -> Response {
1190 if is_stream_request(body) {
1198 let (tx, rx) = tokio::sync::oneshot::channel::<Result<Value, ProxyError>>();
1199 let (state_c, headers_c, body_c, route_c) =
1200 (state.clone(), headers.clone(), body.clone(), route.clone());
1201 tokio::spawn(async move {
1202 let out = enforce_pipeline(
1203 &state_c,
1204 &headers_c,
1205 &body_c,
1206 features,
1207 &route_c,
1208 route_ix,
1209 session_header,
1210 tenant,
1211 routing_mode,
1212 )
1213 .await;
1214 let _ = tx.send(out);
1215 });
1216 return sse_keepalive_response(rx, anthropic_sse_from_message);
1217 }
1218 match enforce_pipeline(
1219 state,
1220 headers,
1221 body,
1222 features,
1223 route,
1224 route_ix,
1225 session_header,
1226 tenant,
1227 routing_mode,
1228 )
1229 .await
1230 {
1231 Ok(message) => (axum::http::StatusCode::OK, Json(message)).into_response(),
1232 Err(e) => e.into_response(),
1233 }
1234}
1235
1236#[allow(clippy::too_many_arguments)] async fn enforce_pipeline_inner(
1241 state: &AppState,
1242 body: &Bytes,
1243 base_request: ModelRequest,
1244 auth: Auth,
1245 features: Features,
1246 route: &Route,
1247 session_header: Option<String>,
1248 tenant: String,
1249 api: &str,
1250 routing_mode: RoutingMode,
1251 route_ix: usize,
1252) -> Result<ModelResponse, ProxyError> {
1253 let gate_defs = state
1254 .config
1255 .routing
1256 .as_ref()
1257 .map_or(&[][..], |cfg| &cfg.gate_defs);
1258 let gates = resolve_gates(
1259 &route.gates,
1260 gate_defs,
1261 &state.providers,
1262 &auth,
1263 &state.config.prices,
1264 );
1265 let session_id = session_header.unwrap_or_else(|| Uuid::now_v7().to_string());
1266 let (budget, max_rungs, speculation, serve_threshold, elastic) =
1267 match state.config.routing.as_ref() {
1268 Some(cfg) => (
1269 cfg.budget.per_request_usd,
1270 cfg.escalation.max_rungs_per_request,
1271 cfg.escalation.speculation,
1272 cfg.escalation.serve_threshold,
1273 cfg.escalation.elastic.as_ref(),
1274 ),
1275 None => (None, 3, 0, None, None),
1276 };
1277 let serve_threshold = state
1280 .adaptive
1281 .as_ref()
1282 .and_then(|a| a.lock().ok().map(|g| g.threshold()))
1283 .or(serve_threshold);
1284
1285 let preset = routing_mode.preset();
1288 let max_rungs = if let Some(delta) = preset.max_rungs_delta {
1289 (max_rungs as i32 + delta).max(1) as u32
1290 } else {
1291 max_rungs
1292 };
1293 let speculation = preset.speculation.unwrap_or(speculation);
1294
1295 let bandit_ctx = crate::bandit::ContextBucket::from_features(&features);
1299
1300 let (greedy_rung, base_policy_id, ts_propensity) = {
1304 let (chosen, ts_p) = state
1305 .bandit
1306 .as_ref()
1307 .and_then(|b| b.lock().ok())
1308 .map(|mut b| {
1309 b.choose_start_with_propensity(&bandit_ctx, &route.ladder, &state.config.prices)
1310 })
1311 .unwrap_or((0, None));
1312 let policy = if ts_p.is_some() {
1313 "bandit@v2-ts".to_owned()
1314 } else if chosen > 0 {
1315 "bandit@v1".to_owned()
1316 } else {
1317 "static-ladder@v0".to_owned()
1318 };
1319 (chosen, policy, ts_p)
1320 };
1321
1322 let exploration_epsilon = state
1327 .config
1328 .routing
1329 .as_ref()
1330 .and_then(|cfg| cfg.escalation.exploration.as_ref())
1331 .map(|e| e.epsilon);
1332
1333 let (start_rung, policy_id, explore_flag, propensity) = if ts_propensity.is_some() {
1334 if exploration_epsilon.is_some() {
1337 tracing::warn!(
1338 "bandit.algorithm = thompson already logs propensities; \
1339 [escalation.exploration] epsilon is ignored"
1340 );
1341 }
1342 (greedy_rung, base_policy_id, false, ts_propensity)
1343 } else if let Some(epsilon) = exploration_epsilon {
1344 let k = route.ladder.len().max(1);
1345 let u = u01(Uuid::now_v7().as_u128());
1347 let (chosen, eps_branch) = if u < epsilon {
1348 let idx = ((u / epsilon) * k as f64) as u32;
1350 (idx.min(k as u32 - 1), true)
1351 } else {
1352 (greedy_rung, false)
1353 };
1354 let p = epsilon_propensity(chosen, greedy_rung, epsilon, k);
1355 (chosen, format!("{base_policy_id}+eps"), eps_branch, Some(p))
1356 } else {
1357 (greedy_rung, base_policy_id, false, None)
1358 };
1359
1360 let promotion = state
1365 .promoter
1366 .as_ref()
1367 .map(|p| p.decide(&tenant, &session_id, std::time::Instant::now()))
1368 .unwrap_or(crate::affinity::Decision::Cold);
1369 let start_rung = start_rung.max(promotion.start_rung());
1370 let promo_key = state
1373 .promoter
1374 .as_ref()
1375 .map(|_| (tenant.clone(), session_id.clone()));
1376
1377 let start_rung = if preset.start_at_top {
1380 route.ladder.len().saturating_sub(1) as u32
1381 } else {
1382 start_rung
1383 };
1384
1385 let speculation = match state
1391 .config
1392 .routing
1393 .as_ref()
1394 .and_then(|cfg| cfg.escalation.speculation_band)
1395 {
1396 Some([lo, hi]) if speculation > 0 => {
1397 let estimate = state
1398 .bandit
1399 .as_ref()
1400 .and_then(|b| b.lock().ok())
1401 .and_then(|b| b.pass_estimate(&bandit_ctx, start_rung));
1402 match estimate {
1403 Some(p) if p < lo || p > hi => {
1404 metrics::counter!("firstpass_speculation_skipped_total").increment(1);
1405 0
1406 }
1407 _ => speculation,
1408 }
1409 }
1410 _ => speculation,
1411 };
1412
1413 if state.bandit.is_some() {
1415 metrics::counter!(
1416 "firstpass_bandit_start_rung",
1417 "rung" => start_rung.to_string()
1418 )
1419 .increment(1);
1420 }
1421
1422 let mut base_request = base_request;
1423 base_request.cache_prefix = state
1427 .config
1428 .routing
1429 .as_ref()
1430 .is_some_and(|r| r.escalation.prompt_cache);
1431 let ctx = EnforceCtx {
1432 condense: routing_cfg_condense(state),
1433 ladder: &route.ladder,
1434 gates: &gates,
1435 health: &state.gate_health,
1436 base_request: &base_request,
1437 providers: &state.providers,
1438 auth: &auth,
1439 prices: &state.config.prices,
1440 budget_per_request_usd: budget,
1441 max_rungs,
1442 speculation,
1443 serve_threshold,
1444 elastic,
1445 features,
1446 start_rung,
1447 tenant_id: tenant,
1450 session_id,
1451 prompt_hash: prompt_hash(&state.config.prompt_salt, body),
1452 api: api.to_owned(),
1453 policy_id,
1454 };
1455
1456 let (outcome, mut trace) = route_enforce(ctx).await;
1457
1458 trace.route_ix = Some(u32::try_from(route_ix).unwrap_or(0));
1462 trace.policy.explore = explore_flag;
1465 trace.policy.propensity = propensity;
1466 if routing_mode != RoutingMode::Balanced {
1469 trace.policy.mode_profile = Some(routing_mode.as_str().to_owned());
1470 }
1471
1472 if let Some(bandit) = state.bandit.as_ref()
1476 && let Ok(mut b) = bandit.lock()
1477 {
1478 for attempt in &trace.attempts {
1479 b.observe(&bandit_ctx, attempt.rung, attempt.verdict);
1480 }
1481 }
1482
1483 if let Some(promoter) = state.promoter.as_ref()
1488 && let Some((tenant_key, session_key)) = promo_key.as_ref()
1489 && let Some(served) = trace.final_.served_rung
1490 {
1491 promoter.record(
1492 tenant_key,
1493 session_key,
1494 served,
1495 served > start_rung,
1496 std::time::Instant::now(),
1497 );
1498 }
1499
1500 if let Some(predictor) = state.predictor.as_ref()
1506 && let Ok(mut p) = predictor.lock()
1507 {
1508 let predicted = p.predict(&trace.request.features, start_rung);
1510 for attempt in &trace.attempts {
1511 match attempt.verdict {
1512 Verdict::Pass => p.update(&trace.request.features, attempt.rung, true),
1513 Verdict::Fail => p.update(&trace.request.features, attempt.rung, false),
1514 Verdict::Abstain => {} }
1516 }
1517 trace.predicted_pass = Some(predicted);
1518 metrics::histogram!("firstpass_predictor_pass_prob").record(predicted);
1519 }
1520
1521 if let Some(probe_cfg) = state
1528 .config
1529 .routing
1530 .as_ref()
1531 .and_then(|c| c.escalation.probe)
1532 && u01(Uuid::now_v7().as_u128()) < probe_cfg.sample_rate
1533 {
1534 let probe_rung = (start_rung as usize).min(route.ladder.len().saturating_sub(1));
1536 if let Some(probe_model_str) = route.ladder.get(probe_rung).cloned() {
1537 let probe_provider = ModelRef::parse(&probe_model_str)
1538 .ok()
1539 .and_then(|m| state.providers.get(&m.provider));
1540
1541 if let Some(probe_provider) = probe_provider {
1542 let mut join_set = tokio::task::JoinSet::new();
1545 for _ in 0..probe_cfg.k {
1546 let mut probe_req = base_request.clone();
1547 probe_req.model = probe_model_str.clone();
1548 let probe_auth = auth.clone();
1549 let prov = probe_provider.clone();
1550 join_set.spawn(async move { prov.complete(&probe_req, &probe_auth).await });
1551 }
1552
1553 let fail_closed_owned: std::collections::HashSet<String> = gates
1556 .iter()
1557 .filter(|g| g.abstain_fails_closed())
1558 .map(|g| g.id().to_owned())
1559 .collect();
1560
1561 let mut gate_pass_count = 0u32;
1562 let mut probe_cost_usd = 0.0f64;
1563
1564 while let Some(task_result) = join_set.join_next().await {
1565 let Ok(Ok(probe_resp)) = task_result else {
1566 continue; };
1568 probe_cost_usd += state
1571 .config
1572 .prices
1573 .cost_usd_with_cache(
1574 &probe_model_str,
1575 probe_resp.in_tokens,
1576 probe_resp.cache_write_tokens,
1577 probe_resp.cache_read_tokens,
1578 probe_resp.out_tokens,
1579 )
1580 .unwrap_or(0.0);
1581
1582 let mut probe_gate_req = base_request.clone();
1583 probe_gate_req.model = probe_model_str.clone();
1584
1585 let mut probe_gate_results = Vec::with_capacity(gates.len());
1588 for g in &gates {
1589 if !state.gate_health.enabled(&trace.tenant_id, g.id()) {
1591 continue;
1592 }
1593 let r = g.evaluate(&probe_gate_req, &probe_resp).await;
1594 probe_gate_results.push(r);
1596 }
1597
1598 let fail_closed_refs: std::collections::HashSet<&str> =
1599 fail_closed_owned.iter().map(|s| s.as_str()).collect();
1600 let verdict = aggregate_with_policy(&probe_gate_results, &fail_closed_refs);
1601 let passes = match serve_threshold {
1604 None => verdict == Verdict::Pass,
1605 Some(t) => crate::calibrate::gate_score(&probe_gate_results, verdict) >= t,
1606 };
1607 if passes {
1608 gate_pass_count += 1;
1609 }
1610 }
1611
1612 let regime = ProbeRegime::classify(gate_pass_count, probe_cfg.k);
1613 let regime_label = match regime {
1614 ProbeRegime::ConfidentPass => "confident_pass",
1615 ProbeRegime::ConfidentFail => "confident_fail",
1616 ProbeRegime::Ambiguous => "ambiguous",
1617 };
1618 metrics::counter!(
1619 "firstpass_probe_regime_total",
1620 "regime" => regime_label
1621 )
1622 .increment(1);
1623 metrics::gauge!("firstpass_probe_cost_usd_total").increment(probe_cost_usd);
1624 trace.probe = Some(ProbeSignal {
1625 k: probe_cfg.k,
1626 gate_pass_count,
1627 regime,
1628 probe_cost_usd,
1629 });
1630 }
1631 }
1632 }
1633 offer_trace(&state.traces, state.spill.as_ref(), trace);
1637
1638 match outcome {
1639 EngineOutcome::Served(resp) => Ok(resp),
1640 EngineOutcome::Failed(msg) => Err(ProxyError::Engine(msg)),
1641 }
1642}
1643
1644#[allow(clippy::too_many_arguments)]
1647async fn enforce_pipeline(
1648 state: &AppState,
1649 headers: &HeaderMap,
1650 body: &Bytes,
1651 features: Features,
1652 route: &Route,
1653 route_ix: usize,
1655 session_header: Option<String>,
1656 tenant: String,
1657 routing_mode: RoutingMode,
1658) -> Result<Value, ProxyError> {
1659 let Some(base_request) = parse_model_request(body) else {
1660 return Err(ProxyError::BadRequest(
1661 "request body is not a valid Anthropic Messages request".to_owned(),
1662 ));
1663 };
1664 let auth = Auth::from_headers(headers);
1665 let resp = enforce_pipeline_inner(
1666 state,
1667 body,
1668 base_request,
1669 auth,
1670 features,
1671 route,
1672 session_header,
1673 tenant,
1674 "anthropic.messages",
1675 routing_mode,
1676 route_ix,
1677 )
1678 .await?;
1679 Ok(anthropic_response_json(&resp))
1680}
1681
1682#[allow(clippy::too_many_arguments)]
1685async fn enforce_pipeline_openai(
1686 state: &AppState,
1687 headers: &HeaderMap,
1688 body: &Bytes,
1689 features: Features,
1690 route: &Route,
1691 route_ix: usize,
1693 session_header: Option<String>,
1694 tenant: String,
1695 routing_mode: RoutingMode,
1696) -> Result<Value, ProxyError> {
1697 enforce_pipeline_openai_as(
1698 state,
1699 headers,
1700 body,
1701 body,
1702 "openai.chat_completions",
1703 features,
1704 route,
1705 route_ix,
1706 session_header,
1707 tenant,
1708 routing_mode,
1709 )
1710 .await
1711}
1712
1713#[allow(clippy::too_many_arguments)]
1722async fn enforce_pipeline_openai_as(
1723 state: &AppState,
1724 headers: &HeaderMap,
1725 body: &Bytes,
1726 receipt_body: &Bytes,
1727 api: &str,
1728 features: Features,
1729 route: &Route,
1730 route_ix: usize,
1731 session_header: Option<String>,
1732 tenant: String,
1733 routing_mode: RoutingMode,
1734) -> Result<Value, ProxyError> {
1735 let providers = &state.providers;
1737 let all_openai = route.ladder.iter().all(|rung| {
1738 let pid = rung.split('/').next().unwrap_or_default();
1739 providers
1740 .get(pid)
1741 .is_some_and(|p| p.carries_structured_verbatim(Dialect::Openai))
1742 });
1743 let Some(base_request) = parse_openai_request(body, all_openai) else {
1744 return Err(ProxyError::BadRequest(
1745 "request body is not a valid OpenAI Chat Completions request".to_owned(),
1746 ));
1747 };
1748 let auth = Auth::from_headers(headers);
1749 let resp = enforce_pipeline_inner(
1750 state,
1751 receipt_body,
1752 base_request,
1753 auth,
1754 features,
1755 route,
1756 session_header,
1757 tenant,
1758 api,
1759 routing_mode,
1760 route_ix,
1761 )
1762 .await?;
1763 Ok(openai_response_json(&resp))
1764}
1765
1766const SSE_KEEPALIVE_EVERY: Duration = Duration::from_secs(5);
1768
1769fn sse_keepalive_response(
1778 rx: tokio::sync::oneshot::Receiver<Result<Value, ProxyError>>,
1779 format_message: fn(&Value) -> String,
1780) -> Response {
1781 let mut ticks = tokio::time::interval(SSE_KEEPALIVE_EVERY);
1782 ticks.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
1783 ticks.reset(); let stream = KeepaliveStream {
1785 rx: Some(rx),
1786 ticks,
1787 format_message,
1788 };
1789 (
1790 axum::http::StatusCode::OK,
1791 [(
1792 axum::http::header::CONTENT_TYPE,
1793 "text/event-stream; charset=utf-8",
1794 )],
1795 axum::body::Body::from_stream(stream),
1796 )
1797 .into_response()
1798}
1799
1800struct KeepaliveStream {
1803 rx: Option<tokio::sync::oneshot::Receiver<Result<Value, ProxyError>>>,
1805 ticks: tokio::time::Interval,
1806 format_message: fn(&Value) -> String,
1808}
1809
1810impl futures_core::Stream for KeepaliveStream {
1811 type Item = Result<Bytes, std::convert::Infallible>;
1812
1813 fn poll_next(
1814 mut self: std::pin::Pin<&mut Self>,
1815 cx: &mut std::task::Context<'_>,
1816 ) -> std::task::Poll<Option<Self::Item>> {
1817 use std::task::Poll;
1818 let Some(rx) = self.rx.as_mut() else {
1819 return Poll::Ready(None); };
1821 if let Poll::Ready(out) = std::pin::Pin::new(rx).poll(cx) {
1822 let fmt = self.format_message;
1823 let frame = match out {
1824 Ok(Ok(message)) => fmt(&message),
1825 Ok(Err(e)) => sse_error_event(&e),
1826 Err(_) => sse_error_event(&ProxyError::Internal(
1827 "enforce pipeline task dropped".to_owned(),
1828 )),
1829 };
1830 self.rx = None;
1831 return Poll::Ready(Some(Ok(Bytes::from(frame))));
1832 }
1833 if self.ticks.poll_tick(cx).is_ready() {
1834 return Poll::Ready(Some(Ok(Bytes::from_static(b": firstpass routing\n\n"))));
1835 }
1836 Poll::Pending
1837 }
1838}
1839
1840fn sse_error_event(e: &ProxyError) -> String {
1843 let mut out = String::new();
1844 sse_event(
1845 &mut out,
1846 "error",
1847 &serde_json::json!({
1848 "type": "error",
1849 "error": { "type": "api_error", "message": e.client_message() }
1850 }),
1851 );
1852 out
1853}
1854
1855fn parse_model_request(body: &[u8]) -> Option<ModelRequest> {
1864 let json: Value = serde_json::from_slice(body).ok()?;
1865 let raw = json.clone();
1866 let messages_json = json.get("messages")?.as_array()?;
1867 let messages = messages_json
1868 .iter()
1869 .map(|m| ChatMessage {
1870 role: m
1871 .get("role")
1872 .and_then(Value::as_str)
1873 .unwrap_or("user")
1874 .to_owned(),
1875 content: m
1876 .get("content")
1877 .cloned()
1878 .unwrap_or_else(|| Value::String(String::new())),
1879 })
1880 .collect();
1881 let system = json
1882 .get("system")
1883 .and_then(Value::as_str)
1884 .map(str::to_owned);
1885 let max_tokens = json
1886 .get("max_tokens")
1887 .and_then(Value::as_u64)
1888 .and_then(|n| u32::try_from(n).ok())
1889 .unwrap_or(1024);
1890 let tools = json.get("tools").cloned().unwrap_or(Value::Null);
1891 Some(ModelRequest {
1892 model: json
1893 .get("model")
1894 .and_then(Value::as_str)
1895 .unwrap_or_default()
1896 .to_owned(),
1897 system,
1898 messages,
1899 max_tokens,
1900 tools,
1901 raw,
1902 cache_prefix: false,
1903 })
1904}
1905
1906fn anthropic_response_json(resp: &ModelResponse) -> Value {
1916 let content = resp
1917 .raw
1918 .get("content")
1919 .filter(|c| c.is_array())
1920 .cloned()
1921 .unwrap_or_else(|| serde_json::json!([{ "type": "text", "text": resp.text }]));
1922 serde_json::json!({
1923 "id": format!("msg_{}", Uuid::now_v7()),
1924 "type": "message",
1925 "role": "assistant",
1926 "model": resp.model,
1927 "content": content,
1928 "usage": { "input_tokens": resp.in_tokens, "output_tokens": resp.out_tokens },
1929 })
1930}
1931
1932fn sse_event(out: &mut String, event: &str, data: &Value) {
1934 out.push_str("event: ");
1935 out.push_str(event);
1936 out.push_str("\ndata: ");
1937 out.push_str(&data.to_string());
1938 out.push_str("\n\n");
1939}
1940
1941fn anthropic_sse_from_message(message: &Value) -> String {
1948 let mut out = String::new();
1949
1950 let mut start_msg = message.clone();
1952 start_msg["content"] = Value::Array(Vec::new());
1953 sse_event(
1954 &mut out,
1955 "message_start",
1956 &serde_json::json!({ "type": "message_start", "message": start_msg }),
1957 );
1958
1959 let empty = Vec::new();
1960 let blocks = message
1961 .get("content")
1962 .and_then(Value::as_array)
1963 .unwrap_or(&empty);
1964 for (i, block) in blocks.iter().enumerate() {
1965 match block.get("type").and_then(Value::as_str) {
1966 Some("tool_use") => {
1967 let mut shell = block.clone();
1969 shell["input"] = serde_json::json!({});
1970 sse_event(
1971 &mut out,
1972 "content_block_start",
1973 &serde_json::json!({ "type": "content_block_start", "index": i, "content_block": shell }),
1974 );
1975 let input_json = block
1976 .get("input")
1977 .map_or_else(|| "{}".to_owned(), std::string::ToString::to_string);
1978 sse_event(
1979 &mut out,
1980 "content_block_delta",
1981 &serde_json::json!({ "type": "content_block_delta", "index": i,
1982 "delta": { "type": "input_json_delta", "partial_json": input_json } }),
1983 );
1984 }
1985 _ => {
1986 let text = block.get("text").and_then(Value::as_str).unwrap_or("");
1988 sse_event(
1989 &mut out,
1990 "content_block_start",
1991 &serde_json::json!({ "type": "content_block_start", "index": i,
1992 "content_block": { "type": "text", "text": "" } }),
1993 );
1994 sse_event(
1995 &mut out,
1996 "content_block_delta",
1997 &serde_json::json!({ "type": "content_block_delta", "index": i,
1998 "delta": { "type": "text_delta", "text": text } }),
1999 );
2000 }
2001 }
2002 sse_event(
2003 &mut out,
2004 "content_block_stop",
2005 &serde_json::json!({ "type": "content_block_stop", "index": i }),
2006 );
2007 }
2008
2009 let out_tokens = message
2010 .pointer("/usage/output_tokens")
2011 .cloned()
2012 .unwrap_or_else(|| Value::from(0));
2013 sse_event(
2014 &mut out,
2015 "message_delta",
2016 &serde_json::json!({ "type": "message_delta", "delta": { "stop_reason": "end_turn" },
2017 "usage": { "output_tokens": out_tokens } }),
2018 );
2019 sse_event(
2020 &mut out,
2021 "message_stop",
2022 &serde_json::json!({ "type": "message_stop" }),
2023 );
2024 out
2025}
2026
2027async fn observe_passthrough(
2029 state: AppState,
2030 headers: HeaderMap,
2031 body: Bytes,
2032 session_header: Option<String>,
2033 tenant: String,
2034) -> Response {
2035 if is_stream_request(&body) {
2037 return observe_stream(state, headers, body, session_header, tenant).await;
2038 }
2039 let start = Instant::now();
2040 let result = forward_anthropic(
2041 &state.http,
2042 &state.config.upstream_anthropic,
2043 &headers,
2044 body.clone(),
2045 )
2046 .await;
2047 let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
2048
2049 match result {
2050 Ok((status, resp_headers, resp_body)) => {
2051 spawn_trace(
2055 &state,
2056 body,
2057 Some(resp_body.clone()),
2058 latency_ms,
2059 session_header,
2060 tenant,
2061 );
2062 (status, resp_headers, resp_body).into_response()
2063 }
2064 Err(err) => {
2065 spawn_trace(&state, body, None, latency_ms, session_header, tenant);
2066 err.into_response()
2067 }
2068 }
2069}
2070
2071async fn observe_stream(
2080 state: AppState,
2081 headers: HeaderMap,
2082 body: Bytes,
2083 session_header: Option<String>,
2084 tenant: String,
2085) -> Response {
2086 let start = Instant::now();
2087 let result = forward_anthropic_streaming(
2088 &state.http,
2089 &state.config.upstream_anthropic,
2090 &headers,
2091 body.clone(),
2092 )
2093 .await;
2094 let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
2095
2096 match result {
2097 Ok((status, resp_headers, response)) => {
2098 spawn_stream_trace(&state, body, latency_ms, session_header, tenant);
2099 let stream_body = Body::from_stream(response.bytes_stream());
2100 (status, resp_headers, stream_body).into_response()
2101 }
2102 Err(err) => {
2103 spawn_trace(&state, body, None, latency_ms, session_header, tenant);
2104 err.into_response()
2105 }
2106 }
2107}
2108
2109fn spawn_stream_trace(
2111 state: &AppState,
2112 req_body: Bytes,
2113 latency_ms: u64,
2114 session_header: Option<String>,
2115 tenant: String,
2116) {
2117 let config = state.config.clone();
2118 let traces = state.traces.clone();
2119 let spill = state.spill.clone();
2120 tokio::spawn(async move {
2121 let mut trace =
2122 build_stream_trace(&config, &req_body, latency_ms, session_header.as_deref());
2123 trace.tenant_id = tenant;
2125 offer_trace(&traces, spill.as_ref(), trace);
2126 });
2127}
2128
2129fn spawn_trace(
2134 state: &AppState,
2135 req_body: Bytes,
2136 resp_body: Option<Bytes>,
2137 latency_ms: u64,
2138 session_header: Option<String>,
2139 tenant: String,
2140) {
2141 let config = state.config.clone();
2142 let traces = state.traces.clone();
2143 let spill = state.spill.clone();
2144 tokio::spawn(async move {
2145 let mut trace = match resp_body {
2146 Some(resp) => build_trace(
2147 &config,
2148 &req_body,
2149 &resp,
2150 latency_ms,
2151 session_header.as_deref(),
2152 ),
2153 None => build_error_trace(&config, &req_body, latency_ms, session_header.as_deref()),
2154 };
2155 trace.tenant_id = tenant;
2157 offer_trace(&traces, spill.as_ref(), trace);
2158 });
2159}
2160
2161fn session_id(session_header: Option<&str>, trace_id: Uuid) -> String {
2163 session_header
2164 .map(str::to_owned)
2165 .unwrap_or_else(|| trace_id.to_string())
2166}
2167
2168fn prompt_hash(salt: &str, body: &[u8]) -> String {
2171 let mut salted = Vec::with_capacity(salt.len() + body.len());
2172 salted.extend_from_slice(salt.as_bytes());
2173 salted.extend_from_slice(body);
2174 sha256_hex(&salted)
2175}
2176
2177fn request_features(body: &[u8]) -> (Option<String>, u32, bool) {
2181 let Ok(json) = serde_json::from_slice::<Value>(body) else {
2182 return (None, 0, false);
2183 };
2184 let model = json.get("model").and_then(Value::as_str).map(str::to_owned);
2185 let tool_count = json
2186 .get("tools")
2187 .and_then(Value::as_array)
2188 .map_or(0, |tools| u32::try_from(tools.len()).unwrap_or(u32::MAX));
2189 let has_images = json
2190 .get("messages")
2191 .and_then(Value::as_array)
2192 .is_some_and(|messages| messages.iter().any(message_has_image));
2193 (model, tool_count, has_images)
2194}
2195
2196fn message_has_image(message: &Value) -> bool {
2198 message
2199 .get("content")
2200 .and_then(Value::as_array)
2201 .is_some_and(|blocks| {
2202 blocks
2203 .iter()
2204 .any(|block| block.get("type").and_then(Value::as_str) == Some("image"))
2205 })
2206}
2207
2208#[derive(Debug, Default, Clone, PartialEq, Eq)]
2210struct ResponseUsage {
2211 model: Option<String>,
2212 in_tokens: u64,
2213 cache_write_tokens: u64,
2214 cache_read_tokens: u64,
2215 out_tokens: u64,
2216}
2217
2218fn response_usage(body: &[u8]) -> ResponseUsage {
2226 let Ok(json) = serde_json::from_slice::<Value>(body) else {
2227 return ResponseUsage::default();
2228 };
2229 let u = |ptr: &str| json.pointer(ptr).and_then(Value::as_u64).unwrap_or(0);
2230 ResponseUsage {
2231 model: json.get("model").and_then(Value::as_str).map(str::to_owned),
2232 in_tokens: u("/usage/input_tokens"),
2233 cache_write_tokens: u("/usage/cache_creation_input_tokens"),
2234 cache_read_tokens: u("/usage/cache_read_input_tokens"),
2235 out_tokens: u("/usage/output_tokens"),
2236 }
2237}
2238
2239fn build_trace(
2241 config: &ProxyConfig,
2242 req_body: &Bytes,
2243 resp_body: &Bytes,
2244 latency_ms: u64,
2245 session_header: Option<&str>,
2246) -> Trace {
2247 let (req_model, tool_count, has_images) = request_features(req_body);
2248 let usage = response_usage(resp_body);
2249 let (resp_model, in_tokens, out_tokens) = (usage.model, usage.in_tokens, usage.out_tokens);
2250 let (cache_write_tokens, cache_read_tokens) =
2251 (usage.cache_write_tokens, usage.cache_read_tokens);
2252 let model = resp_model
2253 .or(req_model)
2254 .unwrap_or_else(|| "unknown".to_owned());
2255
2256 let cost_usd = config
2257 .prices
2258 .cost_usd_with_cache(
2259 &format!("anthropic/{model}"),
2260 in_tokens,
2261 cache_write_tokens,
2262 cache_read_tokens,
2263 out_tokens,
2264 )
2265 .unwrap_or(0.0);
2266
2267 let attempt = Attempt {
2268 rung: 0,
2269 model,
2270 provider: "anthropic".to_owned(),
2271 in_tokens,
2272 cache_write_tokens,
2273 cache_read_tokens,
2274 out_tokens,
2275 cost_usd,
2276 latency_ms,
2277 gates: Vec::new(),
2278 verdict: Verdict::Pass,
2279 };
2280
2281 let mut trace = base_trace(config, req_body, latency_ms, session_header);
2282 trace.request.features.prompt_token_bucket = token_bucket(in_tokens);
2283 trace.request.features.tool_count = tool_count;
2284 trace.request.features.has_images = has_images;
2285 trace.attempts.push(attempt);
2286 trace.final_ = FinalOutcome {
2287 served_rung: Some(0),
2288 served_from: ServedFrom::Attempt,
2289 total_cost_usd: cost_usd,
2290 gate_cost_usd: 0.0,
2291 total_latency_ms: latency_ms,
2292 escalations: 0,
2293 counterfactual_baseline_usd: cost_usd,
2294 savings_usd: 0.0,
2295 };
2296 trace.recompute_savings();
2297 trace
2298}
2299
2300fn build_stream_trace(
2304 config: &ProxyConfig,
2305 req_body: &Bytes,
2306 latency_ms: u64,
2307 session_header: Option<&str>,
2308) -> Trace {
2309 let (req_model, tool_count, has_images) = request_features(req_body);
2310 let model = req_model.unwrap_or_else(|| "unknown".to_owned());
2311
2312 let attempt = Attempt {
2313 rung: 0,
2314 model,
2315 provider: "anthropic".to_owned(),
2316 in_tokens: 0,
2317 cache_write_tokens: 0,
2319 cache_read_tokens: 0,
2320 out_tokens: 0,
2321 cost_usd: 0.0,
2322 latency_ms,
2323 gates: Vec::new(),
2324 verdict: Verdict::Pass,
2325 };
2326
2327 let mut trace = base_trace(config, req_body, latency_ms, session_header);
2328 trace.request.features.tool_count = tool_count;
2329 trace.request.features.has_images = has_images;
2330 trace.attempts.push(attempt);
2331 trace.final_ = FinalOutcome {
2332 served_rung: Some(0),
2333 served_from: ServedFrom::Attempt,
2334 total_cost_usd: 0.0,
2335 gate_cost_usd: 0.0,
2336 total_latency_ms: latency_ms,
2337 escalations: 0,
2338 counterfactual_baseline_usd: 0.0,
2339 savings_usd: 0.0,
2340 };
2341 trace.recompute_savings();
2342 trace
2343}
2344
2345fn build_error_trace(
2349 config: &ProxyConfig,
2350 req_body: &Bytes,
2351 latency_ms: u64,
2352 session_header: Option<&str>,
2353) -> Trace {
2354 let (_, tool_count, has_images) = request_features(req_body);
2355 let mut trace = base_trace(config, req_body, latency_ms, session_header);
2356 trace.request.features.tool_count = tool_count;
2357 trace.request.features.has_images = has_images;
2358 trace.final_ = FinalOutcome {
2359 served_rung: None,
2360 served_from: ServedFrom::Error,
2361 total_cost_usd: 0.0,
2362 gate_cost_usd: 0.0,
2363 total_latency_ms: latency_ms,
2364 escalations: 0,
2365 counterfactual_baseline_usd: 0.0,
2366 savings_usd: 0.0,
2367 };
2368 trace.recompute_savings();
2369 trace
2370}
2371
2372fn base_trace(
2375 config: &ProxyConfig,
2376 req_body: &Bytes,
2377 latency_ms: u64,
2378 session_header: Option<&str>,
2379) -> Trace {
2380 let trace_id = Uuid::now_v7();
2381 let mut features = Features::new(TaskKind::Other);
2382 features.hour_bucket = hour_bucket(jiff::Timestamp::now());
2383
2384 Trace {
2385 trace_id,
2386 prev_hash: GENESIS_HASH.to_owned(),
2387 tenant_id: config.tenant_id.clone(),
2388 session_id: session_id(session_header, trace_id),
2389 ts: jiff::Timestamp::now(),
2390 mode: Mode::Observe,
2391 policy: PolicyRef {
2392 id: "observe-passthrough@v0".to_owned(),
2393 explore: false,
2394 propensity: None,
2395 mode_profile: None,
2396 },
2397 request: RequestInfo {
2398 api: "anthropic.messages".to_owned(),
2399 prompt_hash: prompt_hash(&config.prompt_salt, req_body),
2400 features,
2401 },
2402 attempts: Vec::new(),
2403 deferred: Vec::new(),
2404 final_: FinalOutcome {
2405 served_rung: None,
2406 served_from: ServedFrom::Error,
2407 total_cost_usd: 0.0,
2408 gate_cost_usd: 0.0,
2409 total_latency_ms: latency_ms,
2410 escalations: 0,
2411 counterfactual_baseline_usd: 0.0,
2412 savings_usd: 0.0,
2413 },
2414 probe: None,
2415 rollout: None,
2416 shadow: None,
2417 route_ix: None,
2418 predicted_pass: None,
2419 elastic: None,
2420 }
2421}
2422
2423fn openai_messages_have_tool_calls(body: &[u8]) -> bool {
2428 serde_json::from_slice::<Value>(body)
2429 .ok()
2430 .and_then(|json| {
2431 json.get("messages").and_then(Value::as_array).map(|msgs| {
2432 msgs.iter().any(|m| {
2433 m.get("tool_calls").is_some()
2434 || m.get("role").and_then(Value::as_str) == Some("tool")
2435 })
2436 })
2437 })
2438 .unwrap_or(false)
2439}
2440
2441fn openai_has_http_images(body: &[u8]) -> bool {
2445 serde_json::from_slice::<Value>(body)
2446 .ok()
2447 .and_then(|json| {
2448 json.get("messages").and_then(Value::as_array).map(|msgs| {
2449 msgs.iter().any(|m| {
2450 m.get("content")
2451 .and_then(Value::as_array)
2452 .is_some_and(|parts| {
2453 parts.iter().any(|p| {
2454 p.get("type").and_then(Value::as_str) == Some("image_url")
2455 && p.pointer("/image_url/url")
2456 .and_then(Value::as_str)
2457 .is_some_and(|u| {
2458 u.starts_with("http://") || u.starts_with("https://")
2459 })
2460 })
2461 })
2462 })
2463 })
2464 })
2465 .unwrap_or(false)
2466}
2467
2468fn openai_messages_have_images(body: &[u8]) -> bool {
2471 serde_json::from_slice::<Value>(body)
2472 .ok()
2473 .and_then(|json| {
2474 json.get("messages").and_then(Value::as_array).map(|msgs| {
2475 msgs.iter().any(|m| {
2476 m.get("content")
2477 .and_then(Value::as_array)
2478 .is_some_and(|parts| {
2479 parts
2480 .iter()
2481 .any(|p| p.get("type").and_then(Value::as_str) == Some("image_url"))
2482 })
2483 })
2484 })
2485 })
2486 .unwrap_or(false)
2487}
2488
2489fn extract_openai_features(headers: &HeaderMap, body: &[u8]) -> Features {
2492 let Ok(json) = serde_json::from_slice::<Value>(body) else {
2493 let mut f = Features::new(TaskKind::Other);
2494 f.hour_bucket = hour_bucket(jiff::Timestamp::now());
2495 return f;
2496 };
2497 let tool_count = json
2498 .get("tools")
2499 .and_then(Value::as_array)
2500 .map_or(0, |tools| u32::try_from(tools.len()).unwrap_or(u32::MAX));
2501 let has_images = openai_messages_have_images(body);
2502 let mut f = Features::new(TaskKind::Other);
2503 f.agent = header_str(headers, AGENT_HEADER);
2504 f.subagent = header_str(headers, SUBAGENT_HEADER);
2505 f.tool_count = tool_count;
2506 f.has_images = has_images;
2507 f.prompt_token_bucket = token_bucket(body.len() as u64);
2508 f.hour_bucket = hour_bucket(jiff::Timestamp::now());
2509 f
2510}
2511
2512fn parse_data_url(url: &str) -> Option<(&str, &str)> {
2516 let rest = url.strip_prefix("data:")?;
2517 let (meta, data) = rest.split_once(',')?;
2518 let media_type = meta.strip_suffix(";base64")?;
2519 Some((media_type, data))
2520}
2521
2522fn translate_openai_user_content(content: &Value) -> Option<Value> {
2525 match content {
2526 Value::String(_) => Some(content.clone()),
2528 Value::Array(parts) => {
2529 let mut blocks: Vec<Value> = Vec::with_capacity(parts.len());
2530 for part in parts {
2531 match part.get("type").and_then(Value::as_str) {
2532 Some("text") => {
2533 let text = part.get("text").and_then(Value::as_str).unwrap_or("");
2534 blocks.push(serde_json::json!({ "type": "text", "text": text }));
2535 }
2536 Some("image_url") => {
2537 let url = part.pointer("/image_url/url").and_then(Value::as_str)?;
2538 if url.starts_with("http://") || url.starts_with("https://") {
2539 return None; }
2541 let (media_type, data) = parse_data_url(url)?;
2543 blocks.push(serde_json::json!({
2544 "type": "image",
2545 "source": { "type": "base64", "media_type": media_type, "data": data }
2546 }));
2547 }
2548 _ => {} }
2550 }
2551 Some(Value::Array(blocks))
2552 }
2553 _ => Some(Value::String(String::new())),
2554 }
2555}
2556
2557fn translate_openai_tools(tools: &Value) -> Value {
2561 let Some(arr) = tools.as_array() else {
2562 return Value::Null;
2563 };
2564 let converted: Vec<Value> = arr
2565 .iter()
2566 .map(|tool| {
2567 let func = tool.get("function").unwrap_or(&Value::Null);
2568 let mut out = serde_json::json!({
2569 "name": func.get("name").cloned().unwrap_or(Value::String(String::new())),
2570 "input_schema": func.get("parameters").cloned()
2571 .unwrap_or_else(|| serde_json::json!({ "type": "object" })),
2572 });
2573 if let Some(desc) = func.get("description") {
2574 out["description"] = desc.clone();
2575 }
2576 out
2577 })
2578 .collect();
2579 Value::Array(converted)
2580}
2581
2582fn translate_openai_tool_choice(tc: &Value) -> Value {
2584 match tc {
2585 Value::String(s) => match s.as_str() {
2586 "auto" => serde_json::json!({ "type": "auto" }),
2587 "required" => serde_json::json!({ "type": "any" }),
2588 _ => serde_json::json!({ "type": "auto" }),
2590 },
2591 Value::Object(_) => {
2592 if tc.get("type").and_then(Value::as_str) == Some("function") {
2594 let name = tc.pointer("/function/name").cloned().unwrap_or(Value::Null);
2595 serde_json::json!({ "type": "tool", "name": name })
2596 } else {
2597 serde_json::json!({ "type": "auto" })
2598 }
2599 }
2600 _ => serde_json::json!({ "type": "auto" }),
2601 }
2602}
2603
2604pub fn parse_openai_request(body: &[u8], carry_raw: bool) -> Option<ModelRequest> {
2617 let json: Value = serde_json::from_slice(body).ok()?;
2618 let raw = if carry_raw { json.clone() } else { Value::Null };
2619
2620 let messages_json = json.get("messages")?.as_array()?;
2621
2622 let mut system: Option<String> = None;
2623 let mut messages: Vec<ChatMessage> = Vec::with_capacity(messages_json.len());
2624 let mut tools = Value::Null;
2625 let mut tool_choice_override: Option<Value> = None;
2626
2627 for msg in messages_json {
2628 let role = msg.get("role").and_then(Value::as_str).unwrap_or("user");
2629 match role {
2630 "system" => {
2631 if let Some(s) = msg.get("content").and_then(Value::as_str) {
2636 system = Some(s.to_owned());
2637 }
2638 }
2639 "user" => {
2640 let content_val = msg.get("content").unwrap_or(&Value::Null);
2641 let translated = translate_openai_user_content(content_val)?;
2642 messages.push(ChatMessage {
2643 role: "user".to_owned(),
2644 content: translated,
2645 });
2646 }
2647 "assistant" => {
2648 if let Some(tc_arr) = msg.get("tool_calls").and_then(Value::as_array) {
2649 let mut blocks: Vec<Value> = Vec::new();
2651 if let Some(text) = msg.get("content").and_then(Value::as_str)
2653 && !text.is_empty()
2654 {
2655 blocks.push(serde_json::json!({ "type": "text", "text": text }));
2656 }
2657 for tc in tc_arr {
2658 let id = tc.get("id").and_then(Value::as_str).unwrap_or("");
2659 let func = tc.get("function").unwrap_or(&Value::Null);
2660 let name = func.get("name").and_then(Value::as_str).unwrap_or("");
2661 let args_str = func
2662 .get("arguments")
2663 .and_then(Value::as_str)
2664 .unwrap_or("{}");
2665 let input: Value = serde_json::from_str(args_str)
2666 .unwrap_or_else(|_| serde_json::json!({}));
2667 blocks.push(serde_json::json!({
2668 "type": "tool_use",
2669 "id": id,
2670 "name": name,
2671 "input": input,
2672 }));
2673 }
2674 messages.push(ChatMessage {
2675 role: "assistant".to_owned(),
2676 content: Value::Array(blocks),
2677 });
2678 } else {
2679 let content = msg
2681 .get("content")
2682 .cloned()
2683 .unwrap_or_else(|| Value::String(String::new()));
2684 messages.push(ChatMessage {
2685 role: "assistant".to_owned(),
2686 content,
2687 });
2688 }
2689 }
2690 "tool" => {
2691 let tool_call_id = msg
2693 .get("tool_call_id")
2694 .and_then(Value::as_str)
2695 .unwrap_or("");
2696 let content = msg
2697 .get("content")
2698 .cloned()
2699 .unwrap_or_else(|| Value::String(String::new()));
2700 let result_block = serde_json::json!({
2701 "type": "tool_result",
2702 "tool_use_id": tool_call_id,
2703 "content": content,
2704 });
2705 messages.push(ChatMessage {
2706 role: "user".to_owned(),
2707 content: Value::Array(vec![result_block]),
2708 });
2709 }
2710 _ => {} }
2712 }
2713
2714 if !carry_raw {
2716 if let Some(t) = json.get("tools") {
2717 tools = translate_openai_tools(t);
2718 }
2719 if let Some(tc) = json.get("tool_choice") {
2720 tool_choice_override = Some(translate_openai_tool_choice(tc));
2721 }
2722 } else {
2723 tools = json.get("tools").cloned().unwrap_or(Value::Null);
2724 }
2725
2726 let max_tokens = json
2727 .get("max_tokens")
2728 .and_then(Value::as_u64)
2729 .and_then(|n| u32::try_from(n).ok())
2730 .unwrap_or(1024);
2731
2732 let model = json
2733 .get("model")
2734 .and_then(Value::as_str)
2735 .unwrap_or_default()
2736 .to_owned();
2737
2738 let _ = tool_choice_override; Some(ModelRequest {
2749 model,
2750 system,
2751 messages,
2752 max_tokens,
2753 tools,
2754 raw,
2755 cache_prefix: false,
2756 })
2757}
2758
2759fn extract_openai_content_and_tools(raw: &Value, text: &str) -> (Value, Option<Value>) {
2766 if let Some(blocks) = raw.get("content").and_then(Value::as_array) {
2768 let mut text_parts: Vec<&str> = Vec::new();
2769 let mut tool_calls: Vec<Value> = Vec::new();
2770 for block in blocks {
2771 match block.get("type").and_then(Value::as_str) {
2772 Some("text") => {
2773 if let Some(t) = block.get("text").and_then(Value::as_str) {
2774 text_parts.push(t);
2775 }
2776 }
2777 Some("tool_use") => {
2778 let id = block.get("id").and_then(Value::as_str).unwrap_or("");
2779 let name = block.get("name").and_then(Value::as_str).unwrap_or("");
2780 let input_str = block
2781 .get("input")
2782 .map_or_else(|| "{}".to_owned(), std::string::ToString::to_string);
2783 tool_calls.push(serde_json::json!({
2784 "id": id,
2785 "type": "function",
2786 "function": { "name": name, "arguments": input_str },
2787 }));
2788 }
2789 _ => {}
2790 }
2791 }
2792 let content_text = if tool_calls.is_empty() || !text_parts.is_empty() {
2793 Value::String(text_parts.join(""))
2794 } else {
2795 Value::Null };
2797 let tc = if tool_calls.is_empty() {
2798 None
2799 } else {
2800 Some(Value::Array(tool_calls))
2801 };
2802 return (content_text, tc);
2803 }
2804
2805 if let Some(msg) = raw.pointer("/choices/0/message") {
2807 let content = msg
2808 .get("content")
2809 .cloned()
2810 .unwrap_or(Value::String(text.to_owned()));
2811 let tc = msg.get("tool_calls").cloned();
2812 return (content, tc);
2813 }
2814
2815 (Value::String(text.to_owned()), None)
2817}
2818
2819fn openai_response_json(resp: &ModelResponse) -> Value {
2822 let (content_text, tool_calls) = extract_openai_content_and_tools(&resp.raw, &resp.text);
2823 let finish_reason = if tool_calls.is_some() {
2824 "tool_calls"
2825 } else {
2826 "stop"
2827 };
2828 let mut message = serde_json::json!({
2829 "role": "assistant",
2830 "content": content_text,
2831 });
2832 if let Some(tc) = tool_calls {
2833 message["tool_calls"] = tc;
2834 }
2835 serde_json::json!({
2836 "id": format!("chatcmpl-{}", Uuid::now_v7()),
2837 "object": "chat.completion",
2838 "created": jiff::Timestamp::now().as_second(),
2839 "model": resp.model,
2840 "choices": [{
2841 "index": 0,
2842 "message": message,
2843 "finish_reason": finish_reason,
2844 }],
2845 "usage": {
2846 "prompt_tokens": resp.in_tokens,
2847 "completion_tokens": resp.out_tokens,
2848 "total_tokens": resp.in_tokens + resp.out_tokens,
2849 }
2850 })
2851}
2852
2853fn openai_sse_from_message(message: &Value) -> String {
2857 let id = message
2858 .get("id")
2859 .and_then(Value::as_str)
2860 .unwrap_or("chatcmpl-unknown")
2861 .to_owned();
2862 let created = message.get("created").cloned().unwrap_or(Value::from(0));
2863 let model = message
2864 .get("model")
2865 .and_then(Value::as_str)
2866 .unwrap_or("unknown");
2867
2868 let choices = message.get("choices").and_then(Value::as_array);
2869 let msg = choices
2870 .and_then(|c| c.first())
2871 .and_then(|c| c.get("message"));
2872 let content = msg
2873 .and_then(|m| m.get("content"))
2874 .cloned()
2875 .unwrap_or(Value::Null);
2876 let tool_calls = msg.and_then(|m| m.get("tool_calls")).cloned();
2877 let finish_reason = choices
2878 .and_then(|c| c.first())
2879 .and_then(|c| c.get("finish_reason"))
2880 .cloned()
2881 .unwrap_or_else(|| Value::String("stop".to_owned()));
2882
2883 let mut out = String::new();
2884 let chunk = |delta: Value| {
2885 serde_json::json!({
2886 "id": id,
2887 "object": "chat.completion.chunk",
2888 "created": created,
2889 "model": model,
2890 "choices": [{ "index": 0, "delta": delta, "finish_reason": Value::Null }]
2891 })
2892 };
2893
2894 let role_chunk = chunk(serde_json::json!({ "role": "assistant", "content": "" }));
2896 out.push_str("data: ");
2897 out.push_str(&role_chunk.to_string());
2898 out.push_str("\n\n");
2899
2900 if let Value::String(text) = &content
2902 && !text.is_empty()
2903 {
2904 let content_chunk = chunk(serde_json::json!({ "content": text }));
2905 out.push_str("data: ");
2906 out.push_str(&content_chunk.to_string());
2907 out.push_str("\n\n");
2908 }
2909
2910 if let Some(tc) = tool_calls {
2912 let tc_chunk = chunk(serde_json::json!({ "tool_calls": tc }));
2913 out.push_str("data: ");
2914 out.push_str(&tc_chunk.to_string());
2915 out.push_str("\n\n");
2916 }
2917
2918 let finish_chunk = serde_json::json!({
2920 "id": id,
2921 "object": "chat.completion.chunk",
2922 "created": created,
2923 "model": model,
2924 "choices": [{ "index": 0, "delta": {}, "finish_reason": finish_reason }]
2925 });
2926 out.push_str("data: ");
2927 out.push_str(&finish_chunk.to_string());
2928 out.push_str("\n\n");
2929
2930 out.push_str("data: [DONE]\n\n");
2931 out
2932}
2933
2934#[allow(clippy::too_many_arguments)]
2939async fn handle_enforce_openai(
2940 state: &AppState,
2941 headers: &HeaderMap,
2942 body: &Bytes,
2943 features: Features,
2944 route: &Route,
2945 route_ix: usize,
2948 session_header: Option<String>,
2949 tenant: String,
2950 routing_mode: RoutingMode,
2951) -> Response {
2952 if is_stream_request(body) {
2953 let (tx, rx) = tokio::sync::oneshot::channel::<Result<Value, ProxyError>>();
2954 let (state_c, headers_c, body_c, route_c) =
2955 (state.clone(), headers.clone(), body.clone(), route.clone());
2956 tokio::spawn(async move {
2957 let out = enforce_pipeline_openai(
2958 &state_c,
2959 &headers_c,
2960 &body_c,
2961 features,
2962 &route_c,
2963 route_ix,
2964 session_header,
2965 tenant,
2966 routing_mode,
2967 )
2968 .await;
2969 let _ = tx.send(out);
2970 });
2971 return sse_keepalive_response(rx, openai_sse_from_message);
2972 }
2973 match enforce_pipeline_openai(
2974 state,
2975 headers,
2976 body,
2977 features,
2978 route,
2979 route_ix,
2980 session_header,
2981 tenant,
2982 routing_mode,
2983 )
2984 .await
2985 {
2986 Ok(message) => (axum::http::StatusCode::OK, Json(message)).into_response(),
2987 Err(e) => e.into_response(),
2988 }
2989}
2990
2991async fn observe_passthrough_openai(
2997 state: AppState,
2998 headers: HeaderMap,
2999 body: Bytes,
3000 session_header: Option<String>,
3001 tenant: String,
3002) -> Response {
3003 observe_passthrough_openai_path(
3004 state,
3005 headers,
3006 body,
3007 session_header,
3008 tenant,
3009 crate::upstream::OPENAI_CHAT_PATH,
3010 )
3011 .await
3012}
3013
3014async fn observe_passthrough_openai_path(
3020 state: AppState,
3021 headers: HeaderMap,
3022 body: Bytes,
3023 session_header: Option<String>,
3024 tenant: String,
3025 path: &'static str,
3026) -> Response {
3027 if is_stream_request(&body) {
3028 return observe_stream_openai_path(state, headers, body, session_header, tenant, path)
3029 .await;
3030 }
3031 let start = Instant::now();
3032 let result = crate::upstream::forward_openai_path(
3033 &state.http,
3034 &state.config.upstream_openai,
3035 path,
3036 &headers,
3037 body.clone(),
3038 )
3039 .await;
3040 let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
3041 match result {
3042 Ok((status, resp_headers, resp_body)) => {
3043 spawn_trace(
3044 &state,
3045 body,
3046 Some(resp_body.clone()),
3047 latency_ms,
3048 session_header,
3049 tenant,
3050 );
3051 (status, resp_headers, resp_body).into_response()
3052 }
3053 Err(err) => {
3054 spawn_trace(&state, body, None, latency_ms, session_header, tenant);
3055 err.into_response()
3056 }
3057 }
3058}
3059
3060async fn observe_stream_openai_path(
3064 state: AppState,
3065 headers: HeaderMap,
3066 body: Bytes,
3067 session_header: Option<String>,
3068 tenant: String,
3069 path: &'static str,
3070) -> Response {
3071 let start = Instant::now();
3072 let result = crate::upstream::forward_openai_streaming_path(
3073 &state.http,
3074 &state.config.upstream_openai,
3075 path,
3076 &headers,
3077 body.clone(),
3078 )
3079 .await;
3080 let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
3081 match result {
3082 Ok((status, resp_headers, response)) => {
3083 spawn_stream_trace(&state, body, latency_ms, session_header, tenant);
3084 let stream_body = Body::from_stream(response.bytes_stream());
3085 (status, resp_headers, stream_body).into_response()
3086 }
3087 Err(err) => {
3088 spawn_trace(&state, body, None, latency_ms, session_header, tenant);
3089 err.into_response()
3090 }
3091 }
3092}
3093
3094async fn chat_completions(
3099 State(state): State<AppState>,
3100 Extension(TenantId(tenant)): Extension<TenantId>,
3101 headers: HeaderMap,
3102 body: Bytes,
3103) -> Response {
3104 let session_header = header_str(&headers, SESSION_HEADER);
3105
3106 if let Some(routing) = state.config.routing.as_ref() {
3107 let features = extract_openai_features(&headers, &body);
3108 if let Some(route) = routing
3109 .route_for(&features)
3110 .filter(|r| r.mode == Mode::Enforce && !r.ladder.is_empty())
3111 {
3112 let route_ix = routing
3114 .routes
3115 .iter()
3116 .position(|r| std::ptr::eq(r, route))
3117 .unwrap_or(0);
3118 let route = route.clone();
3119 let routing_mode = resolve_mode(&headers, &route, &state.config);
3121 if routing_mode == RoutingMode::Observe {
3123 return observe_passthrough_openai(state, headers, body, session_header, tenant)
3124 .await;
3125 }
3126 if enforce_can_handle(
3127 &features,
3128 &body,
3129 routing.escalation.enforce_structured,
3130 &route.ladder,
3131 &state.providers,
3132 Dialect::Openai,
3133 ) {
3134 return handle_enforce_openai(
3135 &state,
3136 &headers,
3137 &body,
3138 features,
3139 &route,
3140 route_ix,
3141 session_header,
3142 tenant,
3143 routing_mode,
3144 )
3145 .await;
3146 }
3147 tracing::info!(
3148 "enforce route matched but OpenAI structured request can't be routed faithfully (flag/ladder); serving via observe passthrough"
3149 );
3150 }
3151 }
3152 observe_passthrough_openai(state, headers, body, session_header, tenant).await
3153}
3154
3155async fn responses(
3164 State(state): State<AppState>,
3165 Extension(TenantId(tenant)): Extension<TenantId>,
3166 headers: HeaderMap,
3167 body: Bytes,
3168) -> Response {
3169 let session_header = header_str(&headers, SESSION_HEADER);
3170
3171 if is_stream_request(&body) {
3175 return observe_passthrough_openai_path(
3176 state,
3177 headers,
3178 body,
3179 session_header,
3180 tenant,
3181 crate::upstream::OPENAI_RESPONSES_PATH,
3182 )
3183 .await;
3184 }
3185
3186 let Ok(parsed) = serde_json::from_slice::<Value>(&body) else {
3187 return observe_passthrough_openai_path(
3188 state,
3189 headers,
3190 body,
3191 session_header,
3192 tenant,
3193 crate::upstream::OPENAI_RESPONSES_PATH,
3194 )
3195 .await;
3196 };
3197 if crate::responses::has_untranslatable_content(&parsed) {
3201 return observe_passthrough_openai_path(
3202 state,
3203 headers,
3204 body,
3205 session_header,
3206 tenant,
3207 crate::upstream::OPENAI_RESPONSES_PATH,
3208 )
3209 .await;
3210 }
3211 let chat_body = Bytes::from(crate::responses::request_to_chat(&parsed).to_string());
3212
3213 if let Some(routing) = state.config.routing.as_ref() {
3214 let features = extract_openai_features(&headers, &chat_body);
3215 if let Some(route) = routing
3216 .route_for(&features)
3217 .filter(|r| r.mode == Mode::Enforce && !r.ladder.is_empty())
3218 {
3219 let route_ix = routing
3220 .routes
3221 .iter()
3222 .position(|r| std::ptr::eq(r, route))
3223 .unwrap_or(0);
3224 let route = route.clone();
3225 let routing_mode = resolve_mode(&headers, &route, &state.config);
3226 if routing_mode != RoutingMode::Observe
3227 && enforce_can_handle(
3228 &features,
3229 &chat_body,
3230 routing.escalation.enforce_structured,
3231 &route.ladder,
3232 &state.providers,
3233 Dialect::Openai,
3234 )
3235 {
3236 return match enforce_pipeline_openai_as(
3237 &state,
3238 &headers,
3239 &chat_body,
3240 &body,
3241 "openai.responses",
3242 features,
3243 &route,
3244 route_ix,
3245 session_header,
3246 tenant,
3247 routing_mode,
3248 )
3249 .await
3250 {
3251 Ok(chat) => Json(crate::responses::response_from_chat(&chat)).into_response(),
3252 Err(e) => e.into_response(),
3253 };
3254 }
3255 }
3256 }
3257 observe_passthrough_openai_path(
3258 state,
3259 headers,
3260 body,
3261 session_header,
3262 tenant,
3263 crate::upstream::OPENAI_RESPONSES_PATH,
3264 )
3265 .await
3266}
3267
3268#[cfg(test)]
3269mod tests {
3270 use bytes::Bytes;
3271
3272 use super::*;
3273
3274 fn test_config() -> ProxyConfig {
3275 ProxyConfig::from_lookup(|_| None).unwrap()
3276 }
3277
3278 #[test]
3279 fn build_trace_maps_request_and_response_fields() {
3280 let config = test_config();
3281 let req = Bytes::from_static(
3282 br#"{"model":"claude-haiku-4-5","tools":[{"name":"a"}],"messages":[]}"#,
3283 );
3284 let resp = Bytes::from_static(
3285 br#"{"model":"claude-haiku-4-5","usage":{"input_tokens":1200,"output_tokens":300}}"#,
3286 );
3287
3288 let trace = build_trace(&config, &req, &resp, 42, Some("sess-1"));
3289
3290 assert_eq!(trace.request.api, "anthropic.messages");
3291 assert_eq!(trace.session_id, "sess-1");
3292 assert_eq!(trace.attempts.len(), 1);
3293 let attempt = &trace.attempts[0];
3294 assert_eq!(attempt.model, "claude-haiku-4-5");
3295 assert_eq!(attempt.provider, "anthropic");
3296 assert_eq!(attempt.in_tokens, 1200);
3297 assert_eq!(attempt.out_tokens, 300);
3298 assert!(attempt.cost_usd > 0.0);
3299 assert_eq!(trace.request.features.tool_count, 1);
3300 assert!(!trace.request.features.has_images);
3301 assert_eq!(trace.final_.served_rung, Some(0));
3302 }
3303
3304 #[test]
3305 fn build_trace_falls_back_to_trace_id_session_when_header_absent() {
3306 let config = test_config();
3307 let req = Bytes::from_static(b"{}");
3308 let resp = Bytes::from_static(b"{}");
3309
3310 let trace = build_trace(&config, &req, &resp, 1, None);
3311
3312 assert_eq!(trace.session_id, trace.trace_id.to_string());
3313 }
3314
3315 #[test]
3316 fn build_error_trace_has_no_attempts_and_served_from_error() {
3317 let config = test_config();
3318 let req = Bytes::from_static(br#"{"model":"claude-haiku-4-5"}"#);
3319
3320 let trace = build_error_trace(&config, &req, 7, None);
3321
3322 assert!(trace.attempts.is_empty());
3323 assert_eq!(trace.final_.served_from, ServedFrom::Error);
3324 assert_eq!(trace.final_.served_rung, None);
3325 }
3326
3327 #[test]
3328 fn message_with_image_block_sets_has_images() {
3329 let req = br#"{"messages":[{"role":"user","content":[{"type":"image"}]}]}"#;
3330 let (_, _, has_images) = request_features(req);
3331 assert!(has_images);
3332 }
3333
3334 #[test]
3335 fn prompt_hash_never_contains_raw_prompt_text() {
3336 let hash = prompt_hash("salt", b"super secret prompt");
3337 assert!(!hash.contains("secret"));
3338 assert_eq!(hash.len(), 64);
3339 }
3340
3341 #[test]
3342 fn parse_model_request_preserves_content_verbatim_and_projects_text() {
3343 let body = br#"{"model":"m","system":"sys","max_tokens":50,
3344 "messages":[{"role":"user","content":[{"type":"text","text":"a"},{"type":"text","text":"b"}]},
3345 {"role":"assistant","content":"c"}]}"#;
3346 let req = parse_model_request(body).unwrap();
3347 assert_eq!(req.system.as_deref(), Some("sys"));
3348 assert_eq!(req.max_tokens, 50);
3349 assert_eq!(req.messages.len(), 2);
3350 assert_eq!(
3352 req.messages[0].content,
3353 serde_json::json!([{"type":"text","text":"a"},{"type":"text","text":"b"}])
3354 );
3355 assert_eq!(req.messages[1].content, Value::String("c".to_owned()));
3357 assert_eq!(req.messages[0].text_view(), "a\nb");
3359 assert_eq!(req.messages[1].text_view(), "c");
3360 }
3361
3362 #[test]
3363 fn tool_and_image_blocks_survive_the_request_round_trip() {
3364 let body = br#"{"model":"m","max_tokens":50,"messages":[
3366 {"role":"assistant","content":[{"type":"tool_use","id":"t1","name":"calc","input":{"x":1}}]},
3367 {"role":"user","content":[
3368 {"type":"tool_result","tool_use_id":"t1","content":"2"},
3369 {"type":"image","source":{"type":"base64","media_type":"image/png","data":"AA=="}}
3370 ]}]}"#;
3371 let req = parse_model_request(body).unwrap();
3372 let round_tripped = serde_json::to_value(&req.messages).unwrap();
3373 assert_eq!(
3374 round_tripped,
3375 serde_json::json!([
3376 {"role":"assistant","content":[{"type":"tool_use","id":"t1","name":"calc","input":{"x":1}}]},
3377 {"role":"user","content":[
3378 {"type":"tool_result","tool_use_id":"t1","content":"2"},
3379 {"type":"image","source":{"type":"base64","media_type":"image/png","data":"AA=="}}
3380 ]}
3381 ])
3382 );
3383 }
3384
3385 #[test]
3386 fn text_message_serializes_byte_identical_to_a_plain_string() {
3387 let m = ChatMessage::text("user", "hello");
3389 assert_eq!(
3390 serde_json::to_string(&m).unwrap(),
3391 r#"{"role":"user","content":"hello"}"#
3392 );
3393 }
3394
3395 #[test]
3396 fn parse_model_request_rejects_non_message_bodies() {
3397 assert!(parse_model_request(b"not json").is_none());
3398 assert!(parse_model_request(br#"{"no":"messages"}"#).is_none());
3399 }
3400
3401 use crate::provider::{MockProvider, ModelResponse, Provider, ProviderError, ProviderRegistry};
3404 use axum::extract::State;
3405 use std::collections::HashMap;
3406 use std::sync::Arc;
3407 use tokio::sync::mpsc;
3408
3409 fn model_resp(model: &str, text: &str) -> ModelResponse {
3410 ModelResponse {
3411 model: model.to_owned(),
3412 text: text.to_owned(),
3413 in_tokens: 1000,
3414 cache_write_tokens: 0,
3415 cache_read_tokens: 0,
3416 out_tokens: 400,
3417 raw: serde_json::Value::Null,
3418 }
3419 }
3420
3421 fn enforce_state(
3424 ladder: &[&str],
3425 gates: &[&str],
3426 outcomes: Vec<(&str, Result<ModelResponse, ProviderError>)>,
3427 ) -> (AppState, mpsc::Receiver<Trace>) {
3428 let toml = format!(
3429 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\nladder = [{}]\ngates = [{}]\n",
3430 ladder
3431 .iter()
3432 .map(|m| format!("\"{m}\""))
3433 .collect::<Vec<_>>()
3434 .join(", "),
3435 gates
3436 .iter()
3437 .map(|g| format!("\"{g}\""))
3438 .collect::<Vec<_>>()
3439 .join(", "),
3440 );
3441 let config = ProxyConfig::from_lookup(|k| match k {
3442 "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
3443 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
3444 _ => None,
3445 })
3446 .unwrap();
3447
3448 let mut outs = HashMap::new();
3449 for (model, out) in outcomes {
3450 outs.insert(model.to_owned(), out);
3451 }
3452 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
3453 map.insert(
3454 "anthropic".to_owned(),
3455 Arc::new(MockProvider::new("anthropic", outs)),
3456 );
3457 let providers = ProviderRegistry::from_map(map);
3458
3459 let (traces, rx) = mpsc::channel(64);
3460 let state = AppState {
3461 config: Arc::new(config),
3462 http: reqwest::Client::new(),
3463 providers,
3464 gate_health: Arc::new(GateHealthRegistry::new()),
3465 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
3466 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
3467 traces,
3468 adaptive: None,
3469 bandit: None,
3470 promoter: None,
3471 predictor: None,
3472 tenant_rate_limiter: None,
3473 spill: None,
3474 };
3475 (state, rx)
3476 }
3477
3478 fn user_body() -> Bytes {
3479 Bytes::from_static(
3480 br#"{"model":"ignored","max_tokens":64,"messages":[{"role":"user","content":"hi"}]}"#,
3481 )
3482 }
3483
3484 async fn body_json(resp: Response) -> Value {
3485 let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
3486 .await
3487 .unwrap();
3488 serde_json::from_slice(&bytes).unwrap()
3489 }
3490
3491 #[tokio::test]
3492 async fn mode_profile_stamped_on_trace_when_non_balanced() {
3493 let (state, mut rx) = enforce_state(
3494 &["anthropic/claude-haiku-4-5"],
3495 &["non-empty"],
3496 vec![(
3497 "anthropic/claude-haiku-4-5",
3498 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
3499 )],
3500 );
3501 let mut headers = HeaderMap::new();
3502 headers.insert("x-firstpass-mode", "quality".parse().unwrap());
3503 let resp = messages(
3504 State(state),
3505 Extension(TenantId("default".to_owned())),
3506 headers,
3507 user_body(),
3508 )
3509 .await;
3510 assert_eq!(resp.status(), axum::http::StatusCode::OK);
3511 let trace = rx.try_recv().expect("trace enqueued");
3512 assert_eq!(
3513 trace.policy.mode_profile.as_deref(),
3514 Some("quality"),
3515 "mode_profile must be stamped when quality mode is active"
3516 );
3517 }
3518
3519 #[tokio::test]
3520 async fn mode_profile_absent_from_trace_when_balanced() {
3521 let (state, mut rx) = enforce_state(
3522 &["anthropic/claude-haiku-4-5"],
3523 &["non-empty"],
3524 vec![(
3525 "anthropic/claude-haiku-4-5",
3526 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
3527 )],
3528 );
3529 let resp = messages(
3531 State(state),
3532 Extension(TenantId("default".to_owned())),
3533 HeaderMap::new(),
3534 user_body(),
3535 )
3536 .await;
3537 assert_eq!(resp.status(), axum::http::StatusCode::OK);
3538 let trace = rx.try_recv().expect("trace enqueued");
3539 assert!(
3540 trace.policy.mode_profile.is_none(),
3541 "mode_profile must be absent when Balanced (byte-identical invariant)"
3542 );
3543 }
3544
3545 #[tokio::test]
3546 async fn enforce_serves_first_pass_and_returns_anthropic_shape() {
3547 let (state, mut rx) = enforce_state(
3548 &["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"],
3549 &["non-empty"],
3550 vec![(
3551 "anthropic/claude-haiku-4-5",
3552 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
3553 )],
3554 );
3555 let resp = messages(
3556 State(state),
3557 Extension(TenantId("default".to_owned())),
3558 HeaderMap::new(),
3559 user_body(),
3560 )
3561 .await;
3562 assert_eq!(resp.status(), axum::http::StatusCode::OK);
3563 let json = body_json(resp).await;
3564 assert_eq!(json["type"], "message");
3565 assert_eq!(json["content"][0]["text"], "hello");
3566 assert_eq!(json["model"], "anthropic/claude-haiku-4-5");
3567
3568 let trace = rx.try_recv().expect("a trace was enqueued");
3569 assert_eq!(trace.mode, Mode::Enforce);
3570 assert_eq!(trace.final_.served_rung, Some(0));
3571 assert_eq!(trace.attempts.len(), 1);
3572 }
3573
3574 #[tokio::test]
3575 async fn enforce_escalates_then_serves_and_traces_two_attempts() {
3576 let (state, mut rx) = enforce_state(
3577 &["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"],
3578 &["non-empty"],
3579 vec![
3580 (
3581 "anthropic/claude-haiku-4-5",
3582 Ok(model_resp("anthropic/claude-haiku-4-5", " ")),
3583 ), (
3585 "anthropic/claude-sonnet-5",
3586 Ok(model_resp("anthropic/claude-sonnet-5", "answer")),
3587 ),
3588 ],
3589 );
3590 let resp = messages(
3591 State(state),
3592 Extension(TenantId("default".to_owned())),
3593 HeaderMap::new(),
3594 user_body(),
3595 )
3596 .await;
3597 let json = body_json(resp).await;
3598 assert_eq!(json["content"][0]["text"], "answer");
3599
3600 let trace = rx.try_recv().expect("trace enqueued");
3601 assert_eq!(trace.attempts.len(), 2);
3602 assert_eq!(trace.final_.escalations, 1);
3603 assert_eq!(trace.final_.served_rung, Some(1));
3604 }
3605
3606 #[tokio::test]
3607 async fn enforce_all_rungs_error_returns_502() {
3608 let (state, mut rx) = enforce_state(
3609 &["anthropic/claude-haiku-4-5"],
3610 &["non-empty"],
3611 vec![(
3612 "anthropic/claude-haiku-4-5",
3613 Err(ProviderError::Transport("down".into())),
3614 )],
3615 );
3616 let resp = messages(
3617 State(state),
3618 Extension(TenantId("default".to_owned())),
3619 HeaderMap::new(),
3620 user_body(),
3621 )
3622 .await;
3623 assert_eq!(resp.status(), axum::http::StatusCode::BAD_GATEWAY);
3624 assert!(rx.try_recv().is_ok());
3626 }
3627
3628 #[tokio::test]
3629 async fn no_routing_config_falls_through_to_observe_not_enforce() {
3630 let config = ProxyConfig::from_lookup(|k| match k {
3634 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
3635 _ => None,
3636 })
3637 .unwrap();
3638 let (traces, _rx) = mpsc::channel(64);
3639 let state = AppState {
3640 config: Arc::new(config),
3641 http: reqwest::Client::new(),
3642 providers: ProviderRegistry::new("http://127.0.0.1:1", "http://127.0.0.1:1"),
3643 gate_health: Arc::new(GateHealthRegistry::new()),
3644 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
3645 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
3646 traces,
3647 adaptive: None,
3648 bandit: None,
3649 promoter: None,
3650 predictor: None,
3651 tenant_rate_limiter: None,
3652 spill: None,
3653 };
3654 let resp = messages(
3655 State(state),
3656 Extension(TenantId("default".to_owned())),
3657 HeaderMap::new(),
3658 user_body(),
3659 )
3660 .await;
3661 assert_ne!(resp.status(), axum::http::StatusCode::OK);
3663 }
3664
3665 fn bare_enforce_route() -> Route {
3668 use firstpass_core::config::{Match, Mode};
3669 Route {
3670 match_: Match::default(),
3671 mode: Mode::Enforce,
3672 ladder: vec!["anthropic/claude-haiku-4-5".to_owned()],
3673 gates: vec![],
3674 deferred_gates: vec![],
3675 routing_mode: None,
3676 rollout: None,
3677 shadow: None,
3678 }
3679 }
3680
3681 #[test]
3682 fn balanced_preset_application_is_noop() {
3683 let base_max_rungs = 3u32;
3685 let base_speculation = 2u32;
3686 let preset = RoutingMode::Balanced.preset();
3687 let max_rungs = if let Some(d) = preset.max_rungs_delta {
3688 (base_max_rungs as i32 + d).max(1) as u32
3689 } else {
3690 base_max_rungs
3691 };
3692 let speculation = preset.speculation.unwrap_or(base_speculation);
3693 let start_at_top = preset.start_at_top;
3694 assert_eq!(max_rungs, 3, "Balanced must not change max_rungs");
3695 assert_eq!(speculation, 2, "Balanced must not change speculation");
3696 assert!(!start_at_top, "Balanced must not set start_at_top");
3697 }
3698
3699 #[test]
3700 fn resolve_mode_header_wins_over_route_and_global() {
3701 let mut headers = HeaderMap::new();
3702 headers.insert("x-firstpass-mode", "cost".parse().unwrap());
3703 let mut route = bare_enforce_route();
3704 route.routing_mode = Some(RoutingMode::Quality); let mut config = test_config();
3706 config.default_routing_mode = RoutingMode::Max; assert_eq!(
3708 resolve_mode(&headers, &route, &config),
3709 RoutingMode::Cost,
3710 "header must win"
3711 );
3712 }
3713
3714 #[test]
3715 fn resolve_mode_route_wins_over_global_when_no_header() {
3716 let mut route = bare_enforce_route();
3717 route.routing_mode = Some(RoutingMode::Latency);
3718 let mut config = test_config();
3719 config.default_routing_mode = RoutingMode::Max;
3720 assert_eq!(
3721 resolve_mode(&HeaderMap::new(), &route, &config),
3722 RoutingMode::Latency,
3723 "route must beat global default"
3724 );
3725 }
3726
3727 #[test]
3728 fn resolve_mode_global_when_header_and_route_absent() {
3729 let mut config = test_config();
3730 config.default_routing_mode = RoutingMode::Quality;
3731 assert_eq!(
3732 resolve_mode(&HeaderMap::new(), &bare_enforce_route(), &config),
3733 RoutingMode::Quality
3734 );
3735 }
3736
3737 #[test]
3738 fn resolve_mode_unknown_header_falls_through_to_route() {
3739 let mut headers = HeaderMap::new();
3740 headers.insert("x-firstpass-mode", "turbo-mode".parse().unwrap());
3742 let mut route = bare_enforce_route();
3743 route.routing_mode = Some(RoutingMode::Cost);
3744 let config = test_config();
3745 assert_eq!(
3746 resolve_mode(&headers, &route, &config),
3747 RoutingMode::Cost,
3748 "unknown header value must fall through to route"
3749 );
3750 }
3751
3752 #[test]
3753 fn resolve_mode_header_case_insensitive() {
3754 let mut headers = HeaderMap::new();
3755 headers.insert("x-firstpass-mode", "QUALITY".parse().unwrap());
3756 assert_eq!(
3757 resolve_mode(&headers, &bare_enforce_route(), &test_config()),
3758 RoutingMode::Quality
3759 );
3760 }
3761
3762 #[test]
3763 fn resolve_mode_no_mode_set_returns_balanced() {
3764 assert_eq!(
3766 resolve_mode(&HeaderMap::new(), &bare_enforce_route(), &test_config()),
3767 RoutingMode::Balanced
3768 );
3769 }
3770
3771 #[test]
3772 fn capabilities_json_includes_routing_modes() {
3773 let modes: Vec<&'static str> = RoutingMode::ALL.iter().map(|m| m.as_str()).collect();
3774 assert!(modes.contains(&"balanced"));
3775 assert!(modes.contains(&"cost"));
3776 assert!(modes.contains(&"quality"));
3777 assert!(modes.contains(&"latency"));
3778 assert!(modes.contains(&"max"));
3779 assert!(modes.contains(&"observe"));
3780 }
3781
3782 #[test]
3783 fn model_list_dedups_across_ladders_keeping_cheapest_first_order() {
3784 let config = firstpass_core::config::Config::parse(
3787 r#"
3788 [[route]]
3789 mode = "enforce"
3790 ladder = ["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"]
3791 [[route]]
3792 mode = "enforce"
3793 ladder = ["anthropic/claude-sonnet-5", "anthropic/claude-opus-4-8"]
3794 "#,
3795 )
3796 .expect("test config parses");
3797
3798 let out = model_list(Some(&config));
3799
3800 let ids: Vec<&str> = out["data"]
3801 .as_array()
3802 .expect("data is an array")
3803 .iter()
3804 .map(|m| m["id"].as_str().expect("id is a string"))
3805 .collect();
3806 assert_eq!(
3807 ids,
3808 [
3809 "anthropic/claude-haiku-4-5",
3810 "anthropic/claude-sonnet-5",
3811 "anthropic/claude-opus-4-8",
3812 ],
3813 "sonnet appears once, and ladder order is preserved"
3814 );
3815 assert_eq!(out["object"], "list");
3816 assert_eq!(out["data"][0]["owned_by"], "anthropic");
3817 let haiku_in = out["data"][0]["firstpass"]["input_per_mtok"]
3820 .as_f64()
3821 .expect("a first-party rung has a price");
3822 assert!(haiku_in > 0.0, "expected a real price, got {haiku_in}");
3823 let opus_in = out["data"][2]["firstpass"]["input_per_mtok"]
3825 .as_f64()
3826 .expect("a first-party rung has a price");
3827 assert!(haiku_in < opus_in, "{haiku_in} should undercut {opus_in}");
3828 }
3829
3830 #[tokio::test]
3831 async fn healthz_names_the_service_so_a_client_can_tell_us_from_a_stranger() {
3832 let body = healthz().await.into_response();
3836 let bytes = axum::body::to_bytes(body.into_body(), 64 * 1024)
3837 .await
3838 .expect("healthz body");
3839 let json: serde_json::Value = serde_json::from_slice(&bytes).expect("healthz is json");
3840 assert_eq!(json["status"], "ok");
3841 assert_eq!(json["service"], "firstpass");
3842 }
3843
3844 #[test]
3845 fn model_list_is_empty_without_routing_config() {
3846 let out = model_list(None);
3847 assert_eq!(out["object"], "list");
3848 assert_eq!(
3849 out["data"].as_array().expect("data is an array").len(),
3850 0,
3851 "an observe-only deployment advertises no models"
3852 );
3853 }
3854
3855 #[test]
3856 fn detects_stream_requests() {
3857 assert!(is_stream_request(br#"{"stream": true}"#));
3858 assert!(!is_stream_request(br#"{"stream": false}"#));
3859 assert!(!is_stream_request(br#"{"model":"m"}"#));
3860 assert!(!is_stream_request(b"not json"));
3861 }
3862
3863 #[test]
3864 fn detects_tool_blocks_in_messages() {
3865 let with =
3866 br#"{"messages":[{"role":"user","content":[{"type":"tool_result","content":"42"}]}]}"#;
3867 let without = br#"{"messages":[{"role":"user","content":"hi"}]}"#;
3868 assert!(messages_have_tool_blocks(with));
3869 assert!(!messages_have_tool_blocks(without));
3870 }
3871
3872 #[test]
3873 fn enforce_only_handles_plain_text() {
3874 let plain =
3875 Bytes::from_static(br#"{"model":"m","messages":[{"role":"user","content":"hi"}]}"#);
3876 let tools = Bytes::from_static(
3877 br#"{"model":"m","tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
3878 );
3879 let f_plain = extract_features(&HeaderMap::new(), &plain);
3880 let f_tools = extract_features(&HeaderMap::new(), &tools);
3881 let anthropic_ladder = vec!["anthropic/claude-haiku-4-5".to_owned()];
3882 let providers = test_registry();
3883 assert!(enforce_can_handle(
3885 &f_plain,
3886 &plain,
3887 false,
3888 &anthropic_ladder,
3889 &providers,
3890 Dialect::Anthropic,
3891 ));
3892 assert!(!enforce_can_handle(
3893 &f_tools,
3894 &tools,
3895 false,
3896 &anthropic_ladder,
3897 &providers,
3898 Dialect::Anthropic,
3899 ));
3900 }
3901
3902 #[test]
3903 fn structured_enforce_routes_tools_and_streaming() {
3904 let tools = Bytes::from_static(
3907 br#"{"model":"m","tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
3908 );
3909 let streaming_tools = Bytes::from_static(
3910 br#"{"model":"m","stream":true,"tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
3911 );
3912 let f = extract_features(&HeaderMap::new(), &tools);
3913 let anthropic_ladder = vec![
3914 "anthropic/claude-haiku-4-5".to_owned(),
3915 "anthropic/claude-sonnet-5".to_owned(),
3916 ];
3917 let providers = test_registry();
3918 assert!(enforce_can_handle(
3919 &f,
3920 &tools,
3921 true,
3922 &anthropic_ladder,
3923 &providers,
3924 Dialect::Anthropic,
3925 ));
3926 assert!(enforce_can_handle(
3927 &f,
3928 &streaming_tools,
3929 true,
3930 &anthropic_ladder,
3931 &providers,
3932 Dialect::Anthropic,
3933 ));
3934 }
3935
3936 fn test_registry() -> crate::provider::ProviderRegistry {
3938 crate::provider::ProviderRegistry::new("http://localhost", "http://localhost")
3939 }
3940
3941 #[test]
3942 fn fidelity_guard_blocks_structured_on_non_verbatim_ladder() {
3943 let tools = Bytes::from_static(
3947 br#"{"model":"m","tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
3948 );
3949 let plain =
3950 Bytes::from_static(br#"{"model":"m","messages":[{"role":"user","content":"hi"}]}"#);
3951 let f_tools = extract_features(&HeaderMap::new(), &tools);
3952 let f_plain = extract_features(&HeaderMap::new(), &plain);
3953 let providers = test_registry();
3954 let mixed_ladder = vec![
3955 "openai/gpt-4.1-mini".to_owned(),
3956 "anthropic/claude-sonnet-5".to_owned(),
3957 ];
3958 assert!(!enforce_can_handle(
3959 &f_tools,
3960 &tools,
3961 true,
3962 &mixed_ladder,
3963 &providers,
3964 Dialect::Anthropic,
3965 ));
3966 assert!(enforce_can_handle(
3967 &f_plain,
3968 &plain,
3969 true,
3970 &mixed_ladder,
3971 &providers,
3972 Dialect::Anthropic,
3973 ));
3974 }
3975
3976 #[test]
3977 fn enforce_sse_reemission_preserves_text_and_tool_use() {
3978 let resp = ModelResponse {
3981 model: "anthropic/claude-haiku-4-5".to_owned(),
3982 text: "let me check".to_owned(),
3983 in_tokens: 5,
3984 cache_write_tokens: 0,
3985 cache_read_tokens: 0,
3986 out_tokens: 7,
3987 raw: serde_json::json!({
3988 "content": [
3989 { "type": "text", "text": "let me check" },
3990 { "type": "tool_use", "id": "tu_1", "name": "get_weather", "input": { "city": "Paris" } }
3991 ]
3992 }),
3993 };
3994 let sse = anthropic_sse_from_message(&anthropic_response_json(&resp));
3995
3996 let frames: Vec<Value> = sse
3998 .lines()
3999 .filter_map(|l| l.strip_prefix("data: "))
4000 .map(|d| serde_json::from_str::<Value>(d).expect("each SSE data frame is valid JSON"))
4001 .collect();
4002
4003 assert_eq!(frames.first().unwrap()["type"], "message_start");
4005 assert_eq!(frames.last().unwrap()["type"], "message_stop");
4006 assert!(frames.iter().any(|f| f["delta"]["type"] == "text_delta"
4008 && f["delta"]["text"] == "let me check"));
4009 assert!(
4012 frames
4013 .iter()
4014 .any(|f| f["content_block"]["type"] == "tool_use"
4015 && f["content_block"]["name"] == "get_weather"
4016 && f["content_block"]["id"] == "tu_1")
4017 );
4018 assert!(
4019 frames
4020 .iter()
4021 .any(|f| f["delta"]["type"] == "input_json_delta"
4022 && f["delta"]["partial_json"] == r#"{"city":"Paris"}"#)
4023 );
4024 }
4025
4026 #[tokio::test]
4031 async fn enforce_falls_back_to_observe_for_tool_requests() {
4032 let toml = "[[price]]\nmodel = \"anthropic/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/m\"]\ngates = [\"non-empty\"]\n";
4033 let config = ProxyConfig::from_lookup(|k| match k {
4034 "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
4035 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
4036 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
4037 _ => None,
4038 })
4039 .unwrap();
4040 let mut outs = HashMap::new();
4041 outs.insert(
4042 "anthropic/m".to_owned(),
4043 Ok(model_resp("anthropic/m", "hello")),
4044 );
4045 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
4046 map.insert(
4047 "anthropic".to_owned(),
4048 Arc::new(MockProvider::new("anthropic", outs)),
4049 );
4050 let (traces, _rx) = mpsc::channel(64);
4051 let state = AppState {
4052 config: Arc::new(config),
4053 http: reqwest::Client::new(),
4054 providers: ProviderRegistry::from_map(map),
4055 gate_health: Arc::new(GateHealthRegistry::new()),
4056 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
4057 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
4058 traces,
4059 adaptive: None,
4060 bandit: None,
4061 promoter: None,
4062 predictor: None,
4063 tenant_rate_limiter: None,
4064 spill: None,
4065 };
4066
4067 let plain =
4069 Bytes::from_static(br#"{"model":"m","messages":[{"role":"user","content":"hi"}]}"#);
4070 let resp = messages(
4071 State(state.clone()),
4072 Extension(TenantId("default".to_owned())),
4073 HeaderMap::new(),
4074 plain,
4075 )
4076 .await;
4077 assert_eq!(
4078 resp.status(),
4079 axum::http::StatusCode::OK,
4080 "plain text should enforce"
4081 );
4082
4083 let tools = Bytes::from_static(
4086 br#"{"model":"m","tools":[{"name":"get_weather"}],"messages":[{"role":"user","content":"hi"}]}"#,
4087 );
4088 let resp = messages(
4089 State(state.clone()),
4090 Extension(TenantId("default".to_owned())),
4091 HeaderMap::new(),
4092 tools.clone(),
4093 )
4094 .await;
4095 assert_eq!(
4096 resp.status(),
4097 axum::http::StatusCode::OK,
4098 "tool request must route through enforce by default (ADR 0005 default-on)"
4099 );
4100
4101 let toml_off = "[[price]]\nmodel = \"anthropic/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/m\"]\ngates = [\"non-empty\"]\n[escalation]\nenforce_structured = false\n";
4104 let config_off = ProxyConfig::from_lookup(|k| match k {
4105 "FIRSTPASS_CONFIG_TOML" => Some(toml_off.to_owned()),
4106 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
4107 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
4108 _ => None,
4109 })
4110 .unwrap();
4111 let state_off = AppState {
4112 config: Arc::new(config_off),
4113 ..state.clone()
4114 };
4115 let resp = messages(
4116 State(state_off),
4117 Extension(TenantId("default".to_owned())),
4118 HeaderMap::new(),
4119 tools,
4120 )
4121 .await;
4122 assert_ne!(
4123 resp.status(),
4124 axum::http::StatusCode::OK,
4125 "with enforce_structured = false a tool request must fall back to observe"
4126 );
4127
4128 let toolres = Bytes::from_static(
4130 br#"{"model":"m","messages":[{"role":"user","content":[{"type":"tool_result","tool_use_id":"x","content":"42"}]}]}"#,
4131 );
4132 let resp = messages(
4133 State(state),
4134 Extension(TenantId("default".to_owned())),
4135 HeaderMap::new(),
4136 toolres,
4137 )
4138 .await;
4139 assert_eq!(
4140 resp.status(),
4141 axum::http::StatusCode::OK,
4142 "tool_result blocks route through enforce by default too (verbatim carry)"
4143 );
4144 }
4145
4146 #[tokio::test]
4150 async fn panel_is_public_markup_but_receipts_stay_behind_auth() {
4151 use tower::ServiceExt;
4152 let (state, _rx) = enforce_state(
4153 &["anthropic/claude-haiku-4-5"],
4154 &["non-empty"],
4155 vec![(
4156 "anthropic/claude-haiku-4-5",
4157 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
4158 )],
4159 );
4160 let router = app(state).expect("prometheus recorder installs");
4161
4162 let page = router
4163 .clone()
4164 .oneshot(
4165 axum::http::Request::builder()
4166 .uri("/panel")
4167 .body(Body::empty())
4168 .unwrap(),
4169 )
4170 .await
4171 .unwrap();
4172 assert_eq!(page.status(), axum::http::StatusCode::OK);
4173 let html = axum::body::to_bytes(page.into_body(), 1 << 20)
4174 .await
4175 .unwrap();
4176 let html = String::from_utf8_lossy(&html);
4177 assert!(
4178 html.contains("firstpass") && html.contains("/v1/receipts"),
4179 "the page must fetch its data rather than embed it"
4180 );
4181 assert!(
4184 !html.contains("http://") && !html.contains("https://"),
4185 "the panel must not reference any external origin"
4186 );
4187 assert!(html.contains("const esc ="), "the escaper must exist");
4192 for raw in [
4193 "${a.model}",
4194 "${a.rung}",
4195 "${a.verdict}",
4196 "${r.trace_id}",
4197 "${r.served_rung}",
4198 ] {
4199 assert!(
4200 !html.contains(raw),
4201 "receipt field interpolated unescaped into innerHTML: {raw}"
4202 );
4203 }
4204
4205 let data = router
4206 .oneshot(
4207 axum::http::Request::builder()
4208 .uri("/v1/receipts?limit=5")
4209 .body(Body::empty())
4210 .unwrap(),
4211 )
4212 .await
4213 .unwrap();
4214 assert_eq!(data.status(), axum::http::StatusCode::OK);
4218 let body = axum::body::to_bytes(data.into_body(), 1 << 20)
4219 .await
4220 .unwrap();
4221 let v: Value = serde_json::from_slice(&body).unwrap();
4222 assert!(v.get("receipts").and_then(Value::as_array).is_some());
4223 }
4224
4225 async fn feedback_state() -> (AppState, std::path::PathBuf, String) {
4229 let db = std::env::temp_dir().join(format!("firstpass-feedback-{}.db", Uuid::now_v7()));
4230 let (tx, handle) = crate::store::open(&db).unwrap();
4231
4232 let mut trace = build_error_trace(
4233 &ProxyConfig::from_lookup(|_| None).unwrap(),
4234 &Bytes::from_static(b"{}"),
4235 5,
4236 Some("sess-fb"),
4237 );
4238 trace.attempts.push(Attempt {
4239 rung: 0,
4240 model: "anthropic/claude-haiku-4-5".into(),
4241 provider: "anthropic".into(),
4242 in_tokens: 10,
4243 cache_write_tokens: 0,
4244 cache_read_tokens: 0,
4245 out_tokens: 5,
4246 cost_usd: 0.001,
4247 latency_ms: 5,
4248 gates: vec![],
4249 verdict: Verdict::Pass,
4250 });
4251 let trace_id = trace.trace_id.to_string();
4252 tx.try_send(trace).unwrap();
4253 drop(tx);
4254 handle.await.unwrap();
4255
4256 let db_str = db.to_string_lossy().into_owned();
4257 let config = ProxyConfig::from_lookup(move |k| match k {
4258 "FIRSTPASS_DB" => Some(db_str.clone()),
4259 _ => None,
4260 })
4261 .unwrap();
4262 let (traces, _rx) = mpsc::channel(64);
4263 let state = AppState {
4264 config: Arc::new(config),
4265 http: reqwest::Client::new(),
4266 providers: ProviderRegistry::new("http://127.0.0.1:1", "http://127.0.0.1:1"),
4267 gate_health: Arc::new(GateHealthRegistry::new()),
4268 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
4269 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
4270 traces,
4271 adaptive: None,
4272 bandit: None,
4273 promoter: None,
4274 predictor: None,
4275 tenant_rate_limiter: None,
4276 spill: None,
4277 };
4278 (state, db, trace_id)
4279 }
4280
4281 #[tokio::test]
4282 async fn feedback_nudges_the_adaptive_threshold() {
4283 use firstpass_core::conformal::AdaptiveConformal;
4284 let (mut state, _db, trace_id) = feedback_state().await;
4285 let aci = Arc::new(std::sync::Mutex::new(AdaptiveConformal::new(0.1, 0.2, 0.5)));
4286 state.adaptive = Some(aci.clone());
4287 let before = aci.lock().unwrap().threshold();
4288
4289 let fail = Bytes::from(
4291 serde_json::json!({ "trace_id": trace_id, "gate_id": "tests", "verdict": "fail", "reporter": "ci" })
4292 .to_string(),
4293 );
4294 assert_eq!(
4295 feedback(
4296 State(state.clone()),
4297 Extension(TenantId("default".to_owned())),
4298 fail
4299 )
4300 .await
4301 .status(),
4302 axum::http::StatusCode::ACCEPTED
4303 );
4304 let after_fail = aci.lock().unwrap().threshold();
4305 assert!(
4306 after_fail > before,
4307 "served fail should raise the live threshold: {before} -> {after_fail}"
4308 );
4309
4310 let pass = Bytes::from(
4312 serde_json::json!({ "trace_id": trace_id, "gate_id": "tests", "verdict": "pass", "reporter": "ci" })
4313 .to_string(),
4314 );
4315 let _ = feedback(
4316 State(state),
4317 Extension(TenantId("default".to_owned())),
4318 pass,
4319 )
4320 .await;
4321 assert!(aci.lock().unwrap().threshold() < after_fail);
4322 }
4323
4324 #[tokio::test]
4325 async fn feedback_records_a_deferred_verdict_without_breaking_the_chain() {
4326 let (state, db, trace_id) = feedback_state().await;
4327 let body = Bytes::from(
4328 serde_json::json!({
4329 "trace_id": trace_id,
4330 "gate_id": "tests",
4331 "verdict": "pass",
4332 "score": 1.0,
4333 "reporter": "ci",
4334 })
4335 .to_string(),
4336 );
4337 let resp = feedback(
4338 State(state),
4339 Extension(TenantId("default".to_owned())),
4340 body,
4341 )
4342 .await;
4343 assert_eq!(resp.status(), axum::http::StatusCode::ACCEPTED);
4344
4345 let view = crate::store::load_trace_view(&db, "default", &trace_id)
4347 .unwrap()
4348 .unwrap();
4349 assert_eq!(view.deferred.len(), 1);
4350 assert_eq!(view.deferred[0].gate_id, "tests");
4351 let traces = crate::store::load_all_traces(&db).unwrap();
4353 firstpass_core::verify_chain(&traces, GENESIS_HASH).unwrap();
4354
4355 let _ = std::fs::remove_file(&db);
4356 }
4357
4358 #[tokio::test]
4359 async fn feedback_for_unknown_trace_is_404() {
4360 let (state, db, _trace_id) = feedback_state().await;
4361 let body = Bytes::from(
4362 serde_json::json!({
4363 "trace_id": "does-not-exist",
4364 "gate_id": "tests",
4365 "verdict": "pass",
4366 "reporter": "ci",
4367 })
4368 .to_string(),
4369 );
4370 let resp = feedback(
4371 State(state),
4372 Extension(TenantId("default".to_owned())),
4373 body,
4374 )
4375 .await;
4376 assert_eq!(resp.status(), axum::http::StatusCode::NOT_FOUND);
4377 let _ = std::fs::remove_file(&db);
4378 }
4379
4380 #[tokio::test]
4383 async fn feedback_across_tenants_is_404_not_403() {
4384 let (state, db, trace_id) = feedback_state().await;
4385 let body = Bytes::from(
4386 serde_json::json!({
4387 "trace_id": trace_id,
4388 "gate_id": "tests",
4389 "verdict": "pass",
4390 "score": 1.0,
4391 "reporter": "attacker",
4392 })
4393 .to_string(),
4394 );
4395 let resp = feedback(
4397 State(state),
4398 Extension(TenantId("tenant-b".to_owned())),
4399 body,
4400 )
4401 .await;
4402 assert_eq!(
4403 resp.status(),
4404 axum::http::StatusCode::NOT_FOUND,
4405 "cross-tenant feedback must look exactly like a missing trace"
4406 );
4407 let _ = std::fs::remove_file(&db);
4408 }
4409
4410 #[tokio::test]
4411 async fn feedback_rejects_bad_verdict_and_score() {
4412 let (state, db, trace_id) = feedback_state().await;
4413 let bad_verdict = Bytes::from(
4414 serde_json::json!({ "trace_id": trace_id, "gate_id": "g", "verdict": "maybe", "reporter": "x" })
4415 .to_string(),
4416 );
4417 assert_eq!(
4418 feedback(
4419 State(state.clone()),
4420 Extension(TenantId("default".to_owned())),
4421 bad_verdict
4422 )
4423 .await
4424 .status(),
4425 axum::http::StatusCode::BAD_REQUEST
4426 );
4427 let bad_score = Bytes::from(
4428 serde_json::json!({ "trace_id": trace_id, "gate_id": "g", "verdict": "pass", "score": 9.0, "reporter": "x" })
4429 .to_string(),
4430 );
4431 assert_eq!(
4432 feedback(
4433 State(state),
4434 Extension(TenantId("default".to_owned())),
4435 bad_score
4436 )
4437 .await
4438 .status(),
4439 axum::http::StatusCode::BAD_REQUEST
4440 );
4441 let _ = std::fs::remove_file(&db);
4442 }
4443
4444 #[tokio::test]
4445 async fn metrics_endpoint_renders_after_a_real_request() {
4446 use tower::ServiceExt;
4447
4448 let (state, mut rx) = enforce_state(
4449 &["anthropic/claude-haiku-4-5"],
4450 &["non-empty"],
4451 vec![(
4452 "anthropic/claude-haiku-4-5",
4453 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
4454 )],
4455 );
4456 let router = app(state).expect("prometheus recorder installs");
4457
4458 let req = axum::http::Request::builder()
4459 .method("POST")
4460 .uri("/v1/messages")
4461 .header("content-type", "application/json")
4462 .body(Body::from(user_body()))
4463 .unwrap();
4464 let resp = router.clone().oneshot(req).await.unwrap();
4465 assert_eq!(resp.status(), axum::http::StatusCode::OK);
4466 rx.try_recv().expect("a trace was enqueued");
4467
4468 let metrics_req = axum::http::Request::builder()
4469 .method("GET")
4470 .uri("/metrics")
4471 .body(Body::empty())
4472 .unwrap();
4473 let metrics_resp = router.oneshot(metrics_req).await.unwrap();
4474 assert_eq!(metrics_resp.status(), axum::http::StatusCode::OK);
4475 let bytes = axum::body::to_bytes(metrics_resp.into_body(), 1 << 20)
4476 .await
4477 .unwrap();
4478 let body = String::from_utf8(bytes.to_vec()).unwrap();
4479 assert!(
4480 body.contains("firstpass_enforce_latency_ms"),
4481 "metrics body missing enforce latency histogram: {body}"
4482 );
4483 assert!(
4484 body.contains("firstpass_served_total"),
4485 "metrics body missing served counter: {body}"
4486 );
4487 }
4488
4489 fn auth_state(require_auth: bool, keys_json: Option<String>) -> AppState {
4493 auth_state_rated(require_auth, keys_json, None)
4494 }
4495
4496 fn auth_state_rated(
4499 require_auth: bool,
4500 keys_json: Option<String>,
4501 rate_per_sec: Option<u32>,
4502 ) -> AppState {
4503 let config = ProxyConfig::from_lookup(|k| match k {
4504 "FIRSTPASS_REQUIRE_AUTH" => require_auth.then(|| "true".to_owned()),
4505 "FIRSTPASS_TENANT_KEYS_JSON" => keys_json.clone(),
4506 "FIRSTPASS_TENANT_RATE_PER_SEC" => rate_per_sec.map(|n| n.to_string()),
4507 _ => None,
4508 })
4509 .unwrap();
4510 let (traces, _rx) = mpsc::channel(64);
4511 std::mem::forget(_rx);
4514 let providers: HashMap<String, Arc<dyn Provider>> = HashMap::new();
4515 let tenant_rate_limiter = build_tenant_rate_limiter(&config);
4516 AppState {
4517 config: Arc::new(config),
4518 http: reqwest::Client::new(),
4519 providers: ProviderRegistry::from_map(providers),
4520 gate_health: Arc::new(GateHealthRegistry::new()),
4521 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
4522 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
4523 traces,
4524 adaptive: None,
4525 bandit: None,
4526 promoter: None,
4527 predictor: None,
4528 tenant_rate_limiter,
4529 spill: None,
4530 }
4531 }
4532
4533 fn cap_request(auth_header: Option<&str>) -> axum::http::Request<Body> {
4534 let mut b = axum::http::Request::builder()
4535 .method("GET")
4536 .uri("/v1/capabilities");
4537 if let Some(h) = auth_header {
4538 b = b.header("authorization", h);
4539 }
4540 b.body(Body::empty()).unwrap()
4541 }
4542
4543 #[tokio::test]
4544 async fn auth_off_allows_unauthenticated_request() {
4545 use tower::ServiceExt;
4546 let router = app(auth_state(false, None)).expect("router");
4547 let resp = router.oneshot(cap_request(None)).await.unwrap();
4548 assert_eq!(resp.status(), axum::http::StatusCode::OK);
4550 }
4551
4552 #[tokio::test]
4553 async fn auth_on_missing_key_is_401_opaque() {
4554 use tower::ServiceExt;
4555 let hash = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
4556 let keys = format!("{{\"tenant-a\": {hash:?}}}");
4557 let router = app(auth_state(true, Some(keys))).expect("router");
4558
4559 let resp = router.oneshot(cap_request(None)).await.unwrap();
4560 assert_eq!(resp.status(), axum::http::StatusCode::UNAUTHORIZED);
4561 let json = body_json(resp).await;
4562 assert_eq!(json["error"]["type"], "unauthorized");
4563 let msg = json["error"]["message"].as_str().unwrap();
4565 assert!(!msg.contains("tenant"), "no tenant oracle in body: {msg}");
4566 }
4567
4568 #[tokio::test]
4569 async fn auth_on_invalid_key_is_401() {
4570 use tower::ServiceExt;
4571 let hash = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
4572 let keys = format!("{{\"tenant-a\": {hash:?}}}");
4573 let router = app(auth_state(true, Some(keys))).expect("router");
4574
4575 let resp = router
4576 .oneshot(cap_request(Some("Bearer wrong-key")))
4577 .await
4578 .unwrap();
4579 assert_eq!(resp.status(), axum::http::StatusCode::UNAUTHORIZED);
4580 }
4581
4582 #[tokio::test]
4583 async fn auth_on_valid_key_proceeds() {
4584 use tower::ServiceExt;
4585 let hash = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
4586 let keys = format!("{{\"tenant-a\": {hash:?}}}");
4587 let router = app(auth_state(true, Some(keys))).expect("router");
4588
4589 let resp = router
4591 .oneshot(cap_request(Some("Bearer tenant-a.key-a")))
4592 .await
4593 .unwrap();
4594 assert_eq!(resp.status(), axum::http::StatusCode::OK);
4596 }
4597
4598 fn two_tenant_state(rate_per_sec: Option<u32>) -> AppState {
4600 let hash_a = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
4601 let hash_b = crate::tenant_auth::TenantKeys::hash_key("key-b").unwrap();
4602 let keys = format!("{{\"tenant-a\": {hash_a:?}, \"tenant-b\": {hash_b:?}}}");
4603 auth_state_rated(true, Some(keys), rate_per_sec)
4604 }
4605
4606 #[tokio::test]
4607 async fn tenant_exceeding_rate_limit_gets_429_opaque() {
4608 use tower::ServiceExt;
4609 let router = app(two_tenant_state(Some(1))).expect("router");
4615
4616 let (r1, r2, r3, r4) = tokio::join!(
4617 router
4618 .clone()
4619 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
4620 router
4621 .clone()
4622 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
4623 router
4624 .clone()
4625 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
4626 router
4627 .clone()
4628 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
4629 );
4630 let responses = [r1.unwrap(), r2.unwrap(), r3.unwrap(), r4.unwrap()];
4631 let ok = responses
4632 .iter()
4633 .filter(|r| r.status() == axum::http::StatusCode::OK)
4634 .count();
4635 assert!(ok >= 1, "the burst's first request must pass");
4636 let limited: Vec<_> = responses
4637 .into_iter()
4638 .filter(|r| r.status() == axum::http::StatusCode::TOO_MANY_REQUESTS)
4639 .collect();
4640 assert!(
4641 !limited.is_empty(),
4642 "a 4-request burst against 1 req/sec must trip the limiter"
4643 );
4644
4645 let json = body_json(limited.into_iter().next().unwrap()).await;
4646 assert_eq!(json["error"]["type"], "rate_limited");
4647 let msg = json["error"]["message"].as_str().unwrap();
4649 assert!(!msg.contains('1'), "no limit value in body: {msg}");
4650 }
4651
4652 #[tokio::test]
4653 async fn rate_limit_buckets_are_independent_per_tenant() {
4654 use tower::ServiceExt;
4655 let router = app(two_tenant_state(Some(1))).expect("router");
4656
4657 let (a1, a2, a3) = tokio::join!(
4660 router
4661 .clone()
4662 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
4663 router
4664 .clone()
4665 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
4666 router
4667 .clone()
4668 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
4669 );
4670 let a_limited = [a1.unwrap(), a2.unwrap(), a3.unwrap()]
4671 .iter()
4672 .filter(|r| r.status() == axum::http::StatusCode::TOO_MANY_REQUESTS)
4673 .count();
4674 assert!(a_limited >= 1, "tenant A's burst must trip its limiter");
4675
4676 let b1 = router
4678 .clone()
4679 .oneshot(cap_request(Some("Bearer tenant-b.key-b")))
4680 .await
4681 .unwrap();
4682 assert_eq!(b1.status(), axum::http::StatusCode::OK);
4683 }
4684
4685 #[tokio::test]
4686 async fn rate_limit_unset_never_429s() {
4687 use tower::ServiceExt;
4688 let router = app(two_tenant_state(None)).expect("router");
4691 for _ in 0..20 {
4692 let resp = router
4693 .clone()
4694 .oneshot(cap_request(Some("Bearer tenant-a.key-a")))
4695 .await
4696 .unwrap();
4697 assert_eq!(resp.status(), axum::http::StatusCode::OK);
4698 }
4699 }
4700
4701 #[test]
4704 fn u01_is_deterministic_and_in_range() {
4705 let s1 = u01(0xDEAD_BEEF_CAFE_1234_u128);
4706 let s2 = u01(0xDEAD_BEEF_CAFE_1234_u128);
4707 assert_eq!(s1, s2, "u01 must be deterministic for the same seed");
4708 assert!((0.0..1.0).contains(&s1), "u01 must return [0, 1), got {s1}");
4709
4710 let s3 = u01(0x1234_5678_9ABC_DEF0_u128);
4712 assert_ne!(s1, s3, "different seeds should give different values");
4713
4714 for i in 0u64..256 {
4716 let v = u01(i as u128);
4717 assert!((0.0..1.0).contains(&v), "seed {i}: u01={v} out of [0,1)");
4718 }
4719 }
4720
4721 #[test]
4722 fn epsilon_propensity_formula() {
4723 let epsilon = 0.2_f64;
4724 let k = 3_usize;
4725 let greedy = 1_u32;
4726
4727 let p_greedy = epsilon_propensity(greedy, greedy, epsilon, k);
4729 let expected_greedy = (1.0 - epsilon) + epsilon / k as f64;
4730 assert!(
4731 (p_greedy - expected_greedy).abs() < 1e-12,
4732 "{p_greedy} != {expected_greedy}"
4733 );
4734
4735 let p_other = epsilon_propensity(0, greedy, epsilon, k);
4737 let expected_other = epsilon / k as f64;
4738 assert!(
4739 (p_other - expected_other).abs() < 1e-12,
4740 "{p_other} != {expected_other}"
4741 );
4742
4743 for chosen in 0..k as u32 {
4745 let p = epsilon_propensity(chosen, greedy, epsilon, k);
4746 assert!(
4747 p > 0.0 && p <= 1.0,
4748 "propensity {p} out of (0,1] for chosen={chosen}"
4749 );
4750 }
4751 }
4752
4753 #[test]
4754 fn epsilon_branch_and_greedy_branch_both_occur_over_many_seeds() {
4755 let epsilon = 0.3_f64;
4757 let mut saw_explore = false;
4758 let mut saw_greedy = false;
4759 for i in 0u64..200 {
4760 let u = u01(i as u128);
4761 if u < epsilon {
4762 saw_explore = true;
4763 } else {
4764 saw_greedy = true;
4765 }
4766 if saw_explore && saw_greedy {
4767 break;
4768 }
4769 }
4770 assert!(
4771 saw_explore,
4772 "epsilon branch must fire with epsilon=0.3 over 200 seeds"
4773 );
4774 assert!(
4775 saw_greedy,
4776 "greedy branch must occur with epsilon=0.3 over 200 seeds"
4777 );
4778 }
4779
4780 #[test]
4781 fn epsilon_propensity_sums_to_one_over_all_rungs() {
4782 let epsilon = 0.15_f64;
4784 let k = 4_usize;
4785 let greedy = 2_u32;
4786 let total: f64 = (0..k as u32)
4787 .map(|r| epsilon_propensity(r, greedy, epsilon, k))
4788 .sum();
4789 assert!(
4790 (total - 1.0).abs() < 1e-12,
4791 "propensities must sum to 1, got {total}"
4792 );
4793 }
4794
4795 #[tokio::test(start_paused = true)]
4798 async fn keepalive_stream_ticks_then_emits_final_frame() {
4799 let (tx, rx) = tokio::sync::oneshot::channel::<Result<Value, ProxyError>>();
4800 let mut ticks = tokio::time::interval(SSE_KEEPALIVE_EVERY);
4801 ticks.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
4802 ticks.reset();
4803 let mut stream = KeepaliveStream {
4804 rx: Some(rx),
4805 ticks,
4806 format_message: anthropic_sse_from_message,
4807 };
4808 async fn next(
4810 stream: &mut KeepaliveStream,
4811 ) -> Option<Option<Result<Bytes, std::convert::Infallible>>> {
4812 std::future::poll_fn(|cx| {
4813 std::task::Poll::Ready(
4814 match futures_core::Stream::poll_next(std::pin::Pin::new(&mut *stream), cx) {
4815 std::task::Poll::Ready(item) => Some(item),
4816 std::task::Poll::Pending => None,
4817 },
4818 )
4819 })
4820 .await
4821 }
4822
4823 assert!(
4825 next(&mut stream).await.is_none(),
4826 "no frame before an interval"
4827 );
4828
4829 tokio::time::advance(SSE_KEEPALIVE_EVERY + Duration::from_millis(1)).await;
4831 let frame = next(&mut stream)
4832 .await
4833 .expect("keepalive due")
4834 .unwrap()
4835 .unwrap();
4836 assert!(
4837 frame.starts_with(b": "),
4838 "keepalive must be an SSE comment (ignored by every conforming parser)"
4839 );
4840
4841 let message = serde_json::json!({
4843 "id": "msg_1", "type": "message", "role": "assistant", "model": "m",
4844 "content": [{ "type": "text", "text": "done" }],
4845 "usage": { "input_tokens": 1, "output_tokens": 1 }
4846 });
4847 tx.send(Ok(message)).unwrap();
4848 let frame = next(&mut stream)
4849 .await
4850 .expect("final frame")
4851 .unwrap()
4852 .unwrap();
4853 let text = String::from_utf8(frame.to_vec()).unwrap();
4854 assert!(text.contains("event: message_start"));
4855 assert!(text.contains("event: message_stop"));
4856 let eos = next(&mut stream)
4857 .await
4858 .expect("stream must end after the final frame");
4859 assert!(eos.is_none(), "end-of-stream after the final frame");
4860 }
4861
4862 #[tokio::test]
4865 async fn streaming_enforce_serves_full_sse_sequence() {
4866 let toml = "[[price]]\nmodel = \"anthropic/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/m\"]\ngates = [\"non-empty\"]\n";
4867 let config = ProxyConfig::from_lookup(|k| match k {
4868 "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
4869 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
4870 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
4871 _ => None,
4872 })
4873 .unwrap();
4874 let mut outs = HashMap::new();
4875 outs.insert(
4876 "anthropic/m".to_owned(),
4877 Ok(model_resp("anthropic/m", "gated answer")),
4878 );
4879 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
4880 map.insert(
4881 "anthropic".to_owned(),
4882 Arc::new(MockProvider::new("anthropic", outs)),
4883 );
4884 let (traces, _rx) = mpsc::channel(64);
4885 let state = AppState {
4886 config: Arc::new(config),
4887 http: reqwest::Client::new(),
4888 providers: ProviderRegistry::from_map(map),
4889 gate_health: Arc::new(GateHealthRegistry::new()),
4890 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
4891 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
4892 traces,
4893 adaptive: None,
4894 bandit: None,
4895 promoter: None,
4896 predictor: None,
4897 tenant_rate_limiter: None,
4898 spill: None,
4899 };
4900 let body = Bytes::from_static(
4901 br#"{"model":"m","stream":true,"messages":[{"role":"user","content":"hi"}]}"#,
4902 );
4903 let resp = messages(
4904 State(state),
4905 Extension(TenantId("default".to_owned())),
4906 HeaderMap::new(),
4907 body,
4908 )
4909 .await;
4910 assert_eq!(resp.status(), axum::http::StatusCode::OK);
4911 assert!(
4912 resp.headers()
4913 .get(axum::http::header::CONTENT_TYPE)
4914 .and_then(|v| v.to_str().ok())
4915 .is_some_and(|ct| ct.starts_with("text/event-stream")),
4916 "streaming client must get SSE"
4917 );
4918 let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
4919 .await
4920 .unwrap();
4921 let text = String::from_utf8(bytes.to_vec()).unwrap();
4922 assert!(text.contains("event: message_start"));
4923 assert!(text.contains("gated answer"));
4924 assert!(text.contains("event: message_stop"));
4925 }
4926
4927 #[test]
4932 fn parse_openai_request_plain_text() {
4933 let body = br#"{"model":"gpt-4o","max_tokens":256,"messages":[{"role":"user","content":"hello"}]}"#;
4935 let req = parse_openai_request(body, false).expect("must parse");
4936 assert_eq!(req.model, "gpt-4o");
4937 assert_eq!(req.max_tokens, 256);
4938 assert_eq!(req.messages.len(), 1);
4939 assert_eq!(req.messages[0].role, "user");
4940 assert_eq!(req.messages[0].content, Value::String("hello".to_owned()));
4941 assert!(req.system.is_none());
4942 assert_eq!(req.raw, Value::Null);
4944 }
4945
4946 #[test]
4947 fn parse_openai_request_system_message() {
4948 let body = br#"{"model":"gpt-4o","messages":[{"role":"system","content":"be concise"},{"role":"user","content":"hi"}]}"#;
4949 let req = parse_openai_request(body, false).expect("must parse");
4950 assert_eq!(req.system.as_deref(), Some("be concise"));
4951 assert_eq!(req.messages.len(), 1);
4952 assert_eq!(req.messages[0].role, "user");
4953 }
4954
4955 #[test]
4956 fn parse_openai_request_tool_calls_translate_to_tool_use() {
4957 let body = br#"{
4958 "model":"gpt-4o",
4959 "messages":[
4960 {"role":"user","content":"what's the weather?"},
4961 {"role":"assistant","content":null,"tool_calls":[
4962 {"id":"call_1","type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Paris\"}"}}
4963 ]},
4964 {"role":"tool","tool_call_id":"call_1","content":"15C, cloudy"}
4965 ]
4966 }"#;
4967 let req = parse_openai_request(body, false).expect("must parse");
4968 assert_eq!(req.messages.len(), 3);
4970 let asst = &req.messages[1];
4972 assert_eq!(asst.role, "assistant");
4973 let blocks = asst.content.as_array().expect("content array");
4974 assert_eq!(blocks[0]["type"], "tool_use");
4975 assert_eq!(blocks[0]["name"], "get_weather");
4976 assert_eq!(blocks[0]["id"], "call_1");
4977 assert_eq!(blocks[0]["input"]["city"], "Paris");
4978 let tool_msg = &req.messages[2];
4980 assert_eq!(tool_msg.role, "user");
4981 let result_blocks = tool_msg.content.as_array().expect("result blocks");
4982 assert_eq!(result_blocks[0]["type"], "tool_result");
4983 assert_eq!(result_blocks[0]["tool_use_id"], "call_1");
4984 }
4985
4986 #[test]
4987 fn parse_openai_request_tools_translate_to_anthropic_format() {
4988 let body = br#"{
4989 "model":"gpt-4o",
4990 "messages":[{"role":"user","content":"use a tool"}],
4991 "tools":[{"type":"function","function":{"name":"search","description":"web search","parameters":{"type":"object","properties":{"q":{"type":"string"}}}}}]
4992 }"#;
4993 let req = parse_openai_request(body, false).expect("must parse");
4994 let tools = req.tools.as_array().expect("tools array");
4995 assert_eq!(tools.len(), 1);
4996 assert_eq!(tools[0]["name"], "search");
4997 assert_eq!(tools[0]["description"], "web search");
4998 assert_eq!(tools[0]["input_schema"]["type"], "object");
4999 }
5000
5001 #[test]
5002 fn parse_openai_request_raw_carry_preserves_full_body() {
5003 let body =
5005 br#"{"model":"gpt-4o","max_tokens":100,"messages":[{"role":"user","content":"hi"}]}"#;
5006 let req = parse_openai_request(body, true).expect("must parse");
5007 assert!(req.raw.is_object(), "raw must be the full JSON object");
5008 assert_eq!(req.raw["model"], "gpt-4o");
5009 assert_eq!(req.raw["max_tokens"], 100);
5010 assert!(req.tools.is_null(), "no tools in this request");
5012 }
5013
5014 #[test]
5015 fn parse_openai_request_http_image_returns_none() {
5016 let body = br#"{"model":"gpt-4o","messages":[{"role":"user","content":[
5018 {"type":"text","text":"describe this"},
5019 {"type":"image_url","image_url":{"url":"https://example.com/cat.png"}}
5020 ]}]}"#;
5021 let result = parse_openai_request(body, false);
5022 assert!(result.is_none(), "http image URL must fail translation");
5023 }
5024
5025 #[test]
5026 fn parse_openai_request_data_url_image_translates_to_anthropic_base64() {
5027 let body = br#"{"model":"gpt-4o","messages":[{"role":"user","content":[
5028 {"type":"text","text":"describe"},
5029 {"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgo="}}
5030 ]}]}"#;
5031 let req = parse_openai_request(body, false).expect("data URL must parse");
5032 let blocks = req.messages[0].content.as_array().expect("blocks");
5033 let img = blocks
5034 .iter()
5035 .find(|b| b["type"] == "image")
5036 .expect("image block");
5037 assert_eq!(img["source"]["type"], "base64");
5038 assert_eq!(img["source"]["media_type"], "image/png");
5039 assert_eq!(img["source"]["data"], "iVBORw0KGgo=");
5040 }
5041
5042 #[test]
5045 fn openai_response_json_renders_text_response() {
5046 let resp = ModelResponse {
5047 model: "gpt-4o".to_owned(),
5048 text: "Hello!".to_owned(),
5049 in_tokens: 10,
5050 cache_write_tokens: 0,
5051 cache_read_tokens: 0,
5052 out_tokens: 5,
5053 raw: serde_json::json!({
5054 "content": [{ "type": "text", "text": "Hello!" }]
5055 }),
5056 };
5057 let json = openai_response_json(&resp);
5058 assert_eq!(json["object"], "chat.completion");
5059 assert_eq!(json["model"], "gpt-4o");
5060 assert_eq!(json["choices"][0]["message"]["role"], "assistant");
5061 assert_eq!(json["choices"][0]["message"]["content"], "Hello!");
5062 assert_eq!(json["choices"][0]["finish_reason"], "stop");
5063 assert_eq!(json["usage"]["prompt_tokens"], 10);
5064 assert_eq!(json["usage"]["completion_tokens"], 5);
5065 }
5066
5067 #[test]
5068 fn openai_response_json_renders_tool_call() {
5069 let resp = ModelResponse {
5070 model: "gpt-4o".to_owned(),
5071 text: String::new(),
5072 in_tokens: 20,
5073 cache_write_tokens: 0,
5074 cache_read_tokens: 0,
5075 out_tokens: 15,
5076 raw: serde_json::json!({
5077 "content": [{
5078 "type": "tool_use",
5079 "id": "call_abc",
5080 "name": "search",
5081 "input": {"q": "Rust async"}
5082 }]
5083 }),
5084 };
5085 let json = openai_response_json(&resp);
5086 assert_eq!(json["choices"][0]["finish_reason"], "tool_calls");
5087 let tc = &json["choices"][0]["message"]["tool_calls"][0];
5088 assert_eq!(tc["id"], "call_abc");
5089 assert_eq!(tc["type"], "function");
5090 assert_eq!(tc["function"]["name"], "search");
5091 assert_eq!(json["choices"][0]["message"]["content"], Value::Null);
5093 }
5094
5095 #[test]
5096 fn openai_sse_from_message_plain_text_has_role_then_content_then_stop() {
5097 let resp = ModelResponse {
5098 model: "gpt-4o".to_owned(),
5099 text: "Hi there!".to_owned(),
5100 in_tokens: 5,
5101 cache_write_tokens: 0,
5102 cache_read_tokens: 0,
5103 out_tokens: 3,
5104 raw: serde_json::json!({
5105 "content": [{ "type": "text", "text": "Hi there!" }]
5106 }),
5107 };
5108 let sse = openai_sse_from_message(&openai_response_json(&resp));
5109 for line in sse.lines() {
5111 assert!(
5112 line.is_empty() || line.starts_with("data: "),
5113 "bad SSE line: {line:?}"
5114 );
5115 }
5116 let frames: Vec<&str> = sse
5117 .lines()
5118 .filter_map(|l| l.strip_prefix("data: "))
5119 .collect();
5120 assert_eq!(*frames.last().unwrap(), "[DONE]");
5122 let role_frame: Value = serde_json::from_str(frames[0]).unwrap();
5124 assert_eq!(role_frame["choices"][0]["delta"]["role"], "assistant");
5125 assert!(frames.iter().any(|f| {
5127 if *f == "[DONE]" {
5128 return false;
5129 }
5130 serde_json::from_str::<Value>(f)
5131 .ok()
5132 .is_some_and(|v| v["choices"][0]["delta"]["content"] == "Hi there!")
5133 }));
5134 assert!(frames.iter().any(|f| {
5136 if *f == "[DONE]" {
5137 return false;
5138 }
5139 serde_json::from_str::<Value>(f)
5140 .ok()
5141 .is_some_and(|v| v["choices"][0]["finish_reason"] == "stop")
5142 }));
5143 }
5144
5145 #[test]
5148 fn detects_openai_tool_calls() {
5149 let with_tool_calls = Bytes::from_static(br#"{"messages":[
5150 {"role":"user","content":"hi"},
5151 {"role":"assistant","content":null,"tool_calls":[{"id":"c1","type":"function","function":{"name":"f","arguments":"{}"}}]}
5152 ]}"#);
5153 let without = Bytes::from_static(br#"{"messages":[{"role":"user","content":"hi"}]}"#);
5154 let with_tool_msg = Bytes::from_static(
5155 br#"{"messages":[
5156 {"role":"tool","tool_call_id":"c1","content":"result"}
5157 ]}"#,
5158 );
5159 assert!(openai_messages_have_tool_calls(&with_tool_calls));
5160 assert!(!openai_messages_have_tool_calls(&without));
5161 assert!(openai_messages_have_tool_calls(&with_tool_msg));
5162 }
5163
5164 #[test]
5165 fn detects_openai_http_images() {
5166 let http_img = Bytes::from_static(
5167 br#"{"messages":[{"role":"user","content":[
5168 {"type":"image_url","image_url":{"url":"https://example.com/img.png"}}
5169 ]}]}"#,
5170 );
5171 let data_img = Bytes::from_static(
5172 br#"{"messages":[{"role":"user","content":[
5173 {"type":"image_url","image_url":{"url":"data:image/png;base64,abc"}}
5174 ]}]}"#,
5175 );
5176 let no_img = Bytes::from_static(br#"{"messages":[{"role":"user","content":"hi"}]}"#);
5177 assert!(openai_has_http_images(&http_img));
5178 assert!(!openai_has_http_images(&data_img));
5179 assert!(!openai_has_http_images(&no_img));
5180 }
5181
5182 #[test]
5183 fn enforce_can_handle_openai_inbound_all_openai_ladder() {
5184 let tools_body = Bytes::from_static(br#"{"model":"gpt-4o","messages":[{"role":"assistant","content":null,"tool_calls":[{"id":"c","type":"function","function":{"name":"f","arguments":"{}"}}]}]}"#);
5186 let f = extract_openai_features(&HeaderMap::new(), &tools_body);
5187 let ladder = vec!["openai/gpt-4o-mini".to_owned(), "openai/gpt-4o".to_owned()];
5188 let providers = crate::provider::ProviderRegistry::new("http://x", "http://x");
5189 assert!(enforce_can_handle(
5190 &f,
5191 &tools_body,
5192 true,
5193 &ladder,
5194 &providers,
5195 Dialect::Openai
5196 ));
5197 }
5198
5199 #[test]
5200 fn enforce_can_handle_openai_inbound_all_anthropic_ladder_no_http_image() {
5201 let tools_body = Bytes::from_static(br#"{"model":"gpt-4o","messages":[{"role":"assistant","content":null,"tool_calls":[{"id":"c","type":"function","function":{"name":"f","arguments":"{}"}}]}]}"#);
5203 let f = extract_openai_features(&HeaderMap::new(), &tools_body);
5204 let ladder = vec!["anthropic/claude-haiku-4-5".to_owned()];
5205 let providers = crate::provider::ProviderRegistry::new("http://x", "http://x");
5206 assert!(enforce_can_handle(
5207 &f,
5208 &tools_body,
5209 true,
5210 &ladder,
5211 &providers,
5212 Dialect::Openai,
5213 ));
5214 }
5215
5216 #[test]
5217 fn enforce_can_handle_openai_inbound_http_image_falls_back() {
5218 let img_body = Bytes::from_static(
5220 br#"{"model":"gpt-4o","messages":[{"role":"user","content":[
5221 {"type":"image_url","image_url":{"url":"https://example.com/img.png"}}
5222 ]}]}"#,
5223 );
5224 let f = extract_openai_features(&HeaderMap::new(), &img_body);
5225 let ladder = vec!["anthropic/claude-haiku-4-5".to_owned()];
5226 let providers = crate::provider::ProviderRegistry::new("http://x", "http://x");
5227 assert!(!enforce_can_handle(
5228 &f,
5229 &img_body,
5230 true,
5231 &ladder,
5232 &providers,
5233 Dialect::Openai,
5234 ));
5235 }
5236
5237 fn openai_enforce_state(mock_resp: ModelResponse) -> AppState {
5241 let toml = "[[price]]\nmodel = \"mock/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"mock/m\"]\ngates = [\"non-empty\"]\n";
5242 let config = ProxyConfig::from_lookup(|k| match k {
5243 "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
5244 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
5245 _ => None,
5246 })
5247 .unwrap();
5248 let mut outs = HashMap::new();
5249 outs.insert("mock/m".to_owned(), Ok(mock_resp));
5250 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
5251 map.insert("mock".to_owned(), Arc::new(MockProvider::new("mock", outs)));
5252 let (traces, _rx) = mpsc::channel(64);
5253 std::mem::forget(_rx);
5254 let tenant_rate_limiter = build_tenant_rate_limiter(&config);
5255 AppState {
5256 config: Arc::new(config),
5257 http: reqwest::Client::new(),
5258 providers: ProviderRegistry::from_map(map),
5259 gate_health: Arc::new(GateHealthRegistry::new()),
5260 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
5261 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
5262 traces,
5263 adaptive: None,
5264 bandit: None,
5265 promoter: None,
5266 predictor: None,
5267 tenant_rate_limiter,
5268 spill: None,
5269 }
5270 }
5271
5272 #[tokio::test]
5273 async fn chat_completions_plain_text_enforce_returns_openai_shape() {
5274 let mock = model_resp("mock/m", "gated answer");
5275 let state = openai_enforce_state(mock);
5276 let body = Bytes::from_static(
5277 br#"{"model":"gpt-4o","messages":[{"role":"user","content":"hello"}]}"#,
5278 );
5279 let resp = chat_completions(
5280 State(state),
5281 Extension(TenantId("default".to_owned())),
5282 HeaderMap::new(),
5283 body,
5284 )
5285 .await;
5286 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5287 let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
5288 .await
5289 .unwrap();
5290 let json: Value = serde_json::from_slice(&bytes).expect("must be JSON");
5291 assert_eq!(json["object"], "chat.completion");
5292 assert_eq!(json["choices"][0]["message"]["role"], "assistant");
5293 assert_eq!(json["choices"][0]["message"]["content"], "gated answer");
5294 assert_eq!(json["choices"][0]["finish_reason"], "stop");
5295 assert!(
5296 json["id"]
5297 .as_str()
5298 .is_some_and(|id| id.starts_with("chatcmpl-"))
5299 );
5300 }
5301
5302 #[tokio::test]
5303 async fn chat_completions_stream_true_returns_sse_with_openai_chunks() {
5304 let mock = model_resp("mock/m", "gated answer");
5305 let state = openai_enforce_state(mock);
5306 let body = Bytes::from_static(
5307 br#"{"model":"gpt-4o","stream":true,"messages":[{"role":"user","content":"hello"}]}"#,
5308 );
5309 let resp = chat_completions(
5310 State(state),
5311 Extension(TenantId("default".to_owned())),
5312 HeaderMap::new(),
5313 body,
5314 )
5315 .await;
5316 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5317 assert!(
5318 resp.headers()
5319 .get(axum::http::header::CONTENT_TYPE)
5320 .and_then(|v| v.to_str().ok())
5321 .is_some_and(|ct| ct.starts_with("text/event-stream")),
5322 "stream:true must return SSE"
5323 );
5324 let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
5325 .await
5326 .unwrap();
5327 let text = String::from_utf8(bytes.to_vec()).unwrap();
5328 assert!(
5330 text.contains("chat.completion.chunk"),
5331 "must have OpenAI chunk frames"
5332 );
5333 assert!(text.contains("[DONE]"), "must end with [DONE]");
5334 assert!(
5335 text.contains("gated answer"),
5336 "content must be in the stream"
5337 );
5338 assert!(
5340 !text.contains("message_start"),
5341 "must not have Anthropic event types"
5342 );
5343 }
5344
5345 fn probe_state(
5349 sample_rate: f64,
5350 k: u32,
5351 outcomes: Vec<(&str, Result<ModelResponse, ProviderError>)>,
5352 ) -> (AppState, mpsc::Receiver<Trace>) {
5353 let toml = format!(
5354 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\nladder = [\"anthropic/claude-haiku-4-5\"]\ngates = [\"non-empty\"]\n\
5355 [escalation.probe]\nk = {k}\nsample_rate = {sample_rate}\n"
5356 );
5357 let config = ProxyConfig::from_lookup(|k_| match k_ {
5358 "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
5359 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
5360 _ => None,
5361 })
5362 .unwrap();
5363 let mut outs = HashMap::new();
5364 for (model, out) in outcomes {
5365 outs.insert(model.to_owned(), out);
5366 }
5367 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
5368 map.insert(
5369 "anthropic".to_owned(),
5370 Arc::new(MockProvider::new("anthropic", outs)),
5371 );
5372 let (traces, rx) = mpsc::channel(64);
5373 let state = AppState {
5374 config: Arc::new(config),
5375 http: reqwest::Client::new(),
5376 providers: ProviderRegistry::from_map(map),
5377 gate_health: Arc::new(GateHealthRegistry::new()),
5378 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
5379 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
5380 traces,
5381 adaptive: None,
5382 bandit: None,
5383 promoter: None,
5384 predictor: None,
5385 tenant_rate_limiter: None,
5386 spill: None,
5387 };
5388 (state, rx)
5389 }
5390
5391 async fn run_enforce_get_trace(state: AppState, mut rx: mpsc::Receiver<Trace>) -> Trace {
5393 let resp = messages(
5394 State(state),
5395 Extension(TenantId("default".to_owned())),
5396 HeaderMap::new(),
5397 user_body(),
5398 )
5399 .await;
5400 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5401 rx.try_recv().expect("trace must be enqueued")
5402 }
5403
5404 #[tokio::test]
5406 async fn probe_off_trace_has_no_probe_field() {
5407 let (state, rx) = enforce_state(
5408 &["anthropic/claude-haiku-4-5"],
5409 &["non-empty"],
5410 vec![(
5411 "anthropic/claude-haiku-4-5",
5412 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
5413 )],
5414 );
5415 assert!(
5416 state
5417 .config
5418 .routing
5419 .as_ref()
5420 .unwrap()
5421 .escalation
5422 .probe
5423 .is_none(),
5424 "probe must default to None"
5425 );
5426 let trace = run_enforce_get_trace(state, rx).await;
5427 assert!(
5428 trace.probe.is_none(),
5429 "probe=None config must not set trace.probe"
5430 );
5431 }
5432
5433 #[tokio::test]
5435 async fn probe_sample_rate_zero_never_fires() {
5436 let (state, rx) = probe_state(
5438 0.0,
5439 5,
5440 vec![(
5441 "anthropic/claude-haiku-4-5",
5442 Ok(model_resp("anthropic/claude-haiku-4-5", "hi")),
5443 )],
5444 );
5445 let trace = run_enforce_get_trace(state, rx).await;
5446 assert!(
5447 trace.probe.is_none(),
5448 "sample_rate=0.0 must never set trace.probe"
5449 );
5450 }
5451
5452 #[tokio::test]
5455 async fn probe_on_all_pass_sets_confident_pass() {
5456 let model = "anthropic/claude-haiku-4-5";
5458 let (state, mut rx) = probe_state(1.0, 3, vec![(model, Ok(model_resp(model, "hello")))]);
5459 let resp = messages(
5460 State(state),
5461 Extension(TenantId("default".to_owned())),
5462 HeaderMap::new(),
5463 user_body(),
5464 )
5465 .await;
5466 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5467 let json = body_json(resp).await;
5469 assert_eq!(
5470 json["content"][0]["text"], "hello",
5471 "served content unchanged"
5472 );
5473
5474 let trace = rx.try_recv().expect("trace enqueued");
5475 let sig = trace.probe.expect("probe must be set when sample_rate=1.0");
5476 assert_eq!(sig.k, 3);
5477 assert_eq!(sig.gate_pass_count, 3, "all 3 samples must pass non-empty");
5478 assert_eq!(
5479 sig.regime,
5480 firstpass_core::ProbeRegime::ConfidentPass,
5481 "all-pass → ConfidentPass"
5482 );
5483 assert!(
5484 sig.probe_cost_usd > 0.0,
5485 "k model calls must cost something"
5486 );
5487 }
5488
5489 #[tokio::test]
5492 async fn probe_on_all_fail_sets_confident_fail() {
5493 let model = "anthropic/claude-haiku-4-5";
5494 let (state, mut rx) = probe_state(1.0, 3, vec![(model, Ok(model_resp(model, "")))]);
5496 let resp = messages(
5498 State(state),
5499 Extension(TenantId("default".to_owned())),
5500 HeaderMap::new(),
5501 user_body(),
5502 )
5503 .await;
5504 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5505
5506 let trace = rx.try_recv().expect("trace enqueued");
5507 let sig = trace.probe.expect("probe must be set when sample_rate=1.0");
5508 assert_eq!(
5509 sig.gate_pass_count, 0,
5510 "empty response fails non-empty: all 0 pass"
5511 );
5512 assert_eq!(
5513 sig.regime,
5514 firstpass_core::ProbeRegime::ConfidentFail,
5515 "0 passes → ConfidentFail"
5516 );
5517 }
5518
5519 #[tokio::test]
5522 async fn probe_on_served_output_identical_to_probe_off() {
5523 let model = "anthropic/claude-haiku-4-5";
5524 let mk = |sample_rate: f64| {
5525 probe_state(
5526 sample_rate,
5527 2,
5528 vec![(model, Ok(model_resp(model, "gated answer")))],
5529 )
5530 };
5531
5532 let (state_off, mut rx_off) = mk(0.0);
5534 let resp_off = messages(
5535 State(state_off),
5536 Extension(TenantId("default".to_owned())),
5537 HeaderMap::new(),
5538 user_body(),
5539 )
5540 .await;
5541 let json_off = body_json(resp_off).await;
5542 let trace_off = rx_off.try_recv().unwrap();
5543
5544 let (state_on, mut rx_on) = mk(1.0);
5546 let resp_on = messages(
5547 State(state_on),
5548 Extension(TenantId("default".to_owned())),
5549 HeaderMap::new(),
5550 user_body(),
5551 )
5552 .await;
5553 let json_on = body_json(resp_on).await;
5554 let trace_on = rx_on.try_recv().unwrap();
5555
5556 assert_eq!(
5558 json_off["content"][0]["text"], json_on["content"][0]["text"],
5559 "served text must be identical regardless of probe"
5560 );
5561 assert!(
5563 (trace_off.final_.total_cost_usd - trace_on.final_.total_cost_usd).abs() < 1e-12,
5564 "total_cost_usd must not include probe cost: off={} on={}",
5565 trace_off.final_.total_cost_usd,
5566 trace_on.final_.total_cost_usd
5567 );
5568 assert!(trace_off.probe.is_none());
5570 assert!(trace_on.probe.is_some());
5571 assert!(
5573 trace_on.probe.as_ref().unwrap().probe_cost_usd > 0.0,
5574 "probe cost must be positive"
5575 );
5576 }
5577
5578 #[tokio::test]
5588 async fn probe_does_not_mutate_gate_health() {
5589 let model = "anthropic/claude-haiku-4-5";
5590 let (mut state, rx) = probe_state(1.0, 2, vec![(model, Ok(model_resp(model, "answer")))]);
5591
5592 let registry = GateHealthRegistry::new().with_budget("non-empty", 2, 0.4);
5595 registry.record("default", "non-empty", true);
5598 assert!(
5599 registry.enabled("default", "non-empty"),
5600 "gate must start enabled (window not full yet)"
5601 );
5602 state.gate_health = Arc::new(registry);
5603
5604 let trace = run_enforce_get_trace(state, rx).await;
5616 let sig = trace.probe.expect("probe must fire with sample_rate=1.0");
5617 assert_eq!(sig.k, 2);
5618 assert!(
5619 sig.gate_pass_count <= 2,
5620 "gate_pass_count must be in [0, k]"
5621 );
5622 }
5623
5624 fn predictor_state(
5626 enabled: bool,
5627 outcome: Result<ModelResponse, ProviderError>,
5628 ) -> (AppState, mpsc::Receiver<Trace>) {
5629 let toml = "[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/claude-haiku-4-5\"]\ngates = [\"non-empty\"]\n";
5630 let config = ProxyConfig::from_lookup(|k_| match k_ {
5631 "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
5632 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
5633 _ => None,
5634 })
5635 .unwrap();
5636 let mut outs = HashMap::new();
5637 outs.insert("anthropic/claude-haiku-4-5".to_owned(), outcome);
5638 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
5639 map.insert(
5640 "anthropic".to_owned(),
5641 Arc::new(MockProvider::new("anthropic", outs)),
5642 );
5643 let (traces, rx) = mpsc::channel(64);
5644 let predictor = enabled.then(|| {
5645 Arc::new(std::sync::Mutex::new(firstpass_core::PassPredictor::new(
5646 0.05, 1e-4,
5647 )))
5648 });
5649 let state = AppState {
5650 config: Arc::new(config),
5651 http: reqwest::Client::new(),
5652 providers: ProviderRegistry::from_map(map),
5653 gate_health: Arc::new(GateHealthRegistry::new()),
5654 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
5655 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
5656 traces,
5657 adaptive: None,
5658 bandit: None,
5659 promoter: None,
5660 predictor,
5661 tenant_rate_limiter: None,
5662 spill: None,
5663 };
5664 (state, rx)
5665 }
5666
5667 #[tokio::test]
5668 async fn predictor_off_leaves_predicted_pass_none() {
5669 let (state, rx) =
5670 predictor_state(false, Ok(model_resp("anthropic/claude-haiku-4-5", "ok")));
5671 let trace = run_enforce_get_trace(state, rx).await;
5672 assert!(
5673 trace.predicted_pass.is_none(),
5674 "predictor off => no field (byte-identical)"
5675 );
5676 let j = serde_json::to_string(&trace).unwrap();
5678 assert!(!j.contains("predicted_pass"), "None must be omitted: {j}");
5679 }
5680
5681 #[tokio::test]
5682 async fn predictor_on_records_shadow_prediction_and_serves_identically() {
5683 let (state_off, rx_off) = predictor_state(
5686 false,
5687 Ok(model_resp("anthropic/claude-haiku-4-5", "served answer")),
5688 );
5689 let off = run_enforce_get_trace(state_off, rx_off).await;
5690
5691 let (state_on, rx_on) = predictor_state(
5692 true,
5693 Ok(model_resp("anthropic/claude-haiku-4-5", "served answer")),
5694 );
5695 let on = run_enforce_get_trace(state_on, rx_on).await;
5696
5697 assert_eq!(
5698 on.final_.served_rung, off.final_.served_rung,
5699 "served rung identical"
5700 );
5701 assert_eq!(on.attempts.len(), off.attempts.len(), "same attempts");
5702 assert_eq!(
5703 on.final_.total_cost_usd, off.final_.total_cost_usd,
5704 "predictor never adds served cost"
5705 );
5706 let p = on
5707 .predicted_pass
5708 .expect("predictor on => predicted_pass recorded");
5709 assert!(p > 0.0 && p < 1.0, "shadow prediction in (0,1): {p}");
5710 }
5711
5712 fn rollout_state(
5714 percent: f64,
5715 key: &str,
5716 outcomes: Vec<(&str, Result<ModelResponse, ProviderError>)>,
5717 ) -> (AppState, mpsc::Receiver<Trace>) {
5718 let toml = format!(
5719 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
5720 ladder = [\"anthropic/claude-haiku-4-5\"]\ngates = [\"non-empty\"]\n\
5721 [route.rollout]\npercent = {percent}\nkey = \"{key}\"\n"
5722 );
5723 let config = ProxyConfig::from_lookup(|k| match k {
5724 "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
5725 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
5726 _ => None,
5727 })
5728 .unwrap();
5729 let mut outs = HashMap::new();
5730 for (model, out) in outcomes {
5731 outs.insert(model.to_owned(), out);
5732 }
5733 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
5734 map.insert(
5735 "anthropic".to_owned(),
5736 Arc::new(MockProvider::new("anthropic", outs)),
5737 );
5738 let (traces, rx) = mpsc::channel(64);
5739 let state = AppState {
5740 config: Arc::new(config),
5741 http: reqwest::Client::new(),
5742 providers: ProviderRegistry::from_map(map),
5743 gate_health: Arc::new(GateHealthRegistry::new()),
5744 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
5745 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
5746 traces,
5747 adaptive: None,
5748 bandit: None,
5749 promoter: None,
5750 predictor: None,
5751 tenant_rate_limiter: None,
5752 spill: None,
5753 };
5754 (state, rx)
5755 }
5756
5757 #[tokio::test]
5761 async fn rollout_at_zero_percent_never_enforces() {
5762 for i in 0..25 {
5763 let (state, mut rx) = rollout_state(
5764 0.0,
5765 "session",
5766 vec![(
5767 "anthropic/claude-haiku-4-5",
5768 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
5769 )],
5770 );
5771 let mut headers = HeaderMap::new();
5772 headers.insert("x-firstpass-session", format!("s{i}").parse().unwrap());
5773 let resp = messages(
5774 State(state),
5775 Extension(TenantId("default".to_owned())),
5776 headers,
5777 user_body(),
5778 )
5779 .await;
5780 let _ = resp;
5783 if let Ok(t) = rx.try_recv() {
5784 assert!(
5785 t.attempts.is_empty(),
5786 "0% rollout produced an enforced decision for session s{i}"
5787 );
5788 }
5789 }
5790 }
5791
5792 #[tokio::test]
5795 async fn rollout_at_hundred_percent_always_enforces() {
5796 for i in 0..25 {
5797 let (state, mut rx) = rollout_state(
5798 100.0,
5799 "session",
5800 vec![(
5801 "anthropic/claude-haiku-4-5",
5802 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
5803 )],
5804 );
5805 let mut headers = HeaderMap::new();
5806 headers.insert("x-firstpass-session", format!("s{i}").parse().unwrap());
5807 let resp = messages(
5808 State(state),
5809 Extension(TenantId("default".to_owned())),
5810 headers,
5811 user_body(),
5812 )
5813 .await;
5814 assert_eq!(resp.status(), axum::http::StatusCode::OK);
5815 let t = rx.try_recv().expect("trace enqueued");
5816 assert!(
5817 !t.attempts.is_empty(),
5818 "100% rollout skipped enforcement for session s{i}"
5819 );
5820 }
5821 }
5822
5823 #[tokio::test]
5831 async fn dispatch_arm_matches_the_predicted_arm_per_session() {
5832 let mut checked_enforced = 0;
5833 let mut checked_control = 0;
5834 for i in 0..24 {
5835 let session = format!("session-{i}");
5836 let (state, mut rx) = rollout_state(
5837 50.0,
5838 "session",
5839 vec![(
5840 "anthropic/claude-haiku-4-5",
5841 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
5842 )],
5843 );
5844 let expected = firstpass_core::rollout::decide(
5847 &state.config.prompt_salt,
5848 &firstpass_core::Rollout {
5849 percent: 50.0,
5850 key: firstpass_core::RolloutKey::Session,
5851 },
5852 &session,
5853 )
5854 .enforced;
5855
5856 let mut headers = HeaderMap::new();
5857 headers.insert("x-firstpass-session", session.parse().unwrap());
5858 let _ = messages(
5859 State(state),
5860 Extension(TenantId("default".to_owned())),
5861 headers,
5862 user_body(),
5863 )
5864 .await;
5865 let observed = rx
5866 .try_recv()
5867 .map(|t| !t.attempts.is_empty())
5868 .unwrap_or(false);
5869
5870 assert_eq!(
5871 observed,
5872 expected,
5873 "{session}: dispatch put it in the {} arm, bucketing predicted the {} arm",
5874 if observed { "enforced" } else { "control" },
5875 if expected { "enforced" } else { "control" }
5876 );
5877 if expected {
5878 checked_enforced += 1;
5879 } else {
5880 checked_control += 1;
5881 }
5882 }
5883 assert!(
5885 checked_enforced > 0 && checked_control > 0,
5886 "test saw only one arm ({checked_enforced} enforced, {checked_control} control)"
5887 );
5888 }
5889}