1use std::sync::Arc;
5use std::time::Instant;
6
7use axum::body::Body;
8use axum::extract::{Request, State};
9use axum::http::HeaderMap;
10use axum::middleware::Next;
11use axum::response::{IntoResponse, Response};
12use axum::routing::{get, post};
13use axum::{Extension, Json, Router};
14use bytes::Bytes;
15use firstpass_core::features::{hour_bucket, token_bucket};
16use firstpass_core::hashchain::sha256_hex;
17use firstpass_core::{
18 Attempt, DeferredVerdict, Dialect, FEATURE_VERSION, Features, FinalOutcome, GENESIS_HASH, Mode,
19 ModelRef, PolicyRef, ProbeRegime, ProbeSignal, RequestInfo, RoutingMode, Score, ServedFrom,
20 TaskKind, Trace, Verdict,
21 features::{DifficultyHint, TrajectorySignals},
22};
23use futures_core::Stream;
24use serde::Deserialize;
25use serde_json::Value;
26use std::future::Future;
27use std::task::{Context, Poll};
28use std::time::Duration;
29use tokio::sync::mpsc::error::TrySendError;
30use uuid::Uuid;
31
32use crate::config::ProxyConfig;
33use crate::error::ProxyError;
34use crate::gate::{GateHealthRegistry, aggregate_with_policy, resolve_gates};
35use crate::provider::{Auth, ChatMessage, ModelRequest, ModelResponse, ProviderRegistry};
36use crate::router::{EnforceCtx, EngineOutcome, route_enforce};
37use crate::store;
38use crate::tenant_auth::{TenantId, auth_middleware};
39use crate::upstream::{forward_anthropic, forward_anthropic_streaming};
40use firstpass_core::Route;
41use firstpass_core::trace::{RiskCertificate, ShadowSignal, ThresholdSource};
42
43#[derive(Clone)]
46pub struct AppState {
47 pub config: Arc<ProxyConfig>,
49 pub http: reqwest::Client,
51 pub providers: ProviderRegistry,
53 pub gate_health: Arc<GateHealthRegistry>,
55 pub shadow_ledger: Arc<crate::shadow::ShadowLedger>,
58 pub paired_ledger: Arc<crate::shadow::ShadowLedger>,
61 pub experiments: Arc<crate::experiment::ExperimentMonitor>,
64 pub guardrails: Arc<crate::guard::GuardrailRegistry>,
67 pub traces: store::TraceSender,
69 pub adaptive: Option<Arc<std::sync::Mutex<firstpass_core::conformal::AdaptiveConformal>>>,
73 pub eprocess: Option<Arc<std::sync::Mutex<firstpass_core::eprocess::EProcessRiskControl>>>,
78 pub bandit: Option<Arc<std::sync::Mutex<crate::bandit::StartRungBandit>>>,
83 pub promoter: Option<Arc<crate::affinity::SessionPromoter>>,
88 pub verified_cache: Option<Arc<dyn crate::verified_cache::CacheStore>>,
92 pub predictor: Option<Arc<std::sync::Mutex<firstpass_core::PassPredictor>>>,
98 pub prior: Option<Arc<crate::prior::PriorClient>>,
105 pub tenant_rate_limiter: Option<Arc<governor::DefaultKeyedRateLimiter<String>>>,
109 pub spill: Option<store::SpillHandle>,
113}
114
115#[must_use]
131fn routing_cfg_condense(state: &AppState) -> Option<&firstpass_core::config::CondenseConfig> {
132 state
133 .config
134 .routing
135 .as_ref()
136 .and_then(|r| r.escalation.condense.as_ref())
137}
138
139#[must_use]
144fn mode_now() -> jiff::Timestamp {
146 jiff::Timestamp::now()
147}
148
149fn cache_hit_trace(
157 ctx: &crate::router::EnforceCtx<'_>,
158 entry: &crate::verified_cache::Entry,
159 ts: jiff::Timestamp,
160) -> Trace {
161 Trace {
162 trace_id: uuid::Uuid::now_v7(),
163 prev_hash: firstpass_core::GENESIS_HASH.to_owned(),
164 tenant_id: ctx.tenant_id.clone(),
165 session_id: ctx.session_id.clone(),
166 ts,
167 mode: Mode::Enforce,
168 policy: PolicyRef {
169 id: format!("{}+cache", ctx.policy_id),
170 explore: false,
171 propensity: None,
172 mode_profile: None,
173 },
174 request: RequestInfo {
175 api: ctx.api.clone(),
176 prompt_hash: ctx.prompt_hash.clone(),
177 features: ctx.features.clone(),
178 },
179 attempts: Vec::new(),
180 final_: FinalOutcome {
181 served_rung: None,
182 served_from: ServedFrom::Cache,
183 total_cost_usd: 0.0,
187 gate_cost_usd: 0.0,
188 total_latency_ms: 0,
189 escalations: 0,
190 counterfactual_baseline_usd: entry.original_cost_usd,
191 savings_usd: entry.original_cost_usd,
192 cache_source: Some(entry.source),
193 },
194 deferred: Vec::new(),
195 predicted_pass: None,
196 decision_prior: None,
197 probe: None,
198 elastic: None,
199 risk: None,
200 rollout: None,
201 shadow: None,
202 route_ix: None,
203 experiment: None,
204 upstream_provider: String::new(),
205 }
206}
207
208pub async fn build_verified_cache(
209 config: &ProxyConfig,
210) -> Result<Option<Arc<dyn crate::verified_cache::CacheStore>>, String> {
211 let Some(c) = config
212 .routing
213 .as_ref()
214 .and_then(|r| r.escalation.verified_cache.as_ref())
215 else {
216 return Ok(None);
217 };
218 if let Some(url) = c.redis_url.as_deref() {
219 #[cfg(feature = "redis-cache")]
220 {
221 let store = crate::verified_cache::RedisStore::connect(url, c.ttl_secs).await?;
222 tracing::info!("verified cache: shared via redis");
223 return Ok(Some(Arc::new(store)));
224 }
225 #[cfg(not(feature = "redis-cache"))]
226 {
227 let _ = url;
228 return Err(
229 "[escalation.verified_cache] redis_url is set, but this binary was built without \
230 the `redis-cache` feature. Rebuild with `--features redis-cache`, or remove \
231 redis_url to use the in-process cache."
232 .to_owned(),
233 );
234 }
235 }
236 Ok(Some(Arc::new(crate::verified_cache::VerifiedCache::new(
237 std::time::Duration::from_secs(c.ttl_secs),
238 c.max_entries,
239 ))))
240}
241
242pub async fn build_promoter_async(
243 config: &ProxyConfig,
244) -> Result<Option<Arc<crate::affinity::SessionPromoter>>, String> {
245 let Some(sp) = config
246 .routing
247 .as_ref()
248 .and_then(|r| r.escalation.session_promotion.as_ref())
249 else {
250 return Ok(None);
251 };
252 let promoter = crate::affinity::SessionPromoter::new(sp.clone()).map_err(|e| e.to_string())?;
253 if let Some(url) = sp.redis_url.as_deref() {
254 #[cfg(feature = "redis-cache")]
255 {
256 let ttl = sp.window_duration().map_err(|e| e.to_string())?.as_secs();
257 let store = crate::affinity::RedisPromotionStore::connect(url, ttl).await?;
258 tracing::info!("session promotion: shared via redis");
259 return Ok(Some(Arc::new(promoter.with_shared(Arc::new(store)))));
260 }
261 #[cfg(not(feature = "redis-cache"))]
262 {
263 let _ = url;
264 return Err(
265 "[escalation.session_promotion] redis_url is set, but this binary was \
266 built without the `redis-cache` feature. Rebuild with \
267 `--features redis-cache`, or remove redis_url to keep promotion \
268 in-process."
269 .to_owned(),
270 );
271 }
272 }
273 Ok(Some(Arc::new(promoter)))
274}
275
276#[must_use]
295pub fn build_paired_ledger(config: &ProxyConfig, shared_log: bool) -> crate::shadow::ShadowLedger {
296 let ledger = crate::shadow::ShadowLedger::new();
297 let any_paired = config.routing.as_ref().is_some_and(|r| {
298 r.routes
299 .iter()
300 .any(|route| route.challenger.as_ref().is_some_and(|c| c.is_paired()))
301 });
302 if !any_paired {
303 return ledger;
304 }
305 match crate::store::load_all_experiment_pairs(&config.db_path) {
306 Ok(rows) => {
307 let now = jiff::Timestamp::now();
308 let mut seen = std::collections::HashSet::new();
315 for row in rows
316 .iter()
317 .filter(|r| !shared_log || r.in_log != crate::store::InLogStatus::InLog)
318 .filter(|r| r.trace_id.is_nil() || seen.insert(r.trace_id))
319 {
320 ledger.seed(
321 &row.tenant,
322 row.route_ix,
323 row.challenger_cost_usd,
324 row.ts,
325 now,
326 );
327 }
328 }
329 Err(e) => tracing::warn!(%e, "paired ledger: could not read today's spend from the store"),
330 }
331 ledger
332}
333
334pub async fn build_experiment_log(
350 config: &ProxyConfig,
351) -> Result<Option<Arc<dyn crate::experiment::ExperimentLogBackend>>, String> {
352 use std::collections::HashSet;
353
354 let urls: HashSet<&str> = config
355 .routing
356 .iter()
357 .flat_map(|r| &r.routes)
358 .filter_map(|r| r.challenger.as_ref()?.redis_url.as_deref())
359 .collect();
360
361 let mut url_iter = urls.into_iter();
363 let Some(url) = url_iter.next() else {
364 return Ok(None); };
366 if let Some(other) = url_iter.next() {
367 let mut found = vec![url, other];
368 found.extend(url_iter);
369 found.sort_unstable();
370 return Err(format!(
371 "[route.challenger] multiple distinct redis_url values across routes are not \
372 supported — all experiment logs must share one Redis instance (keys are namespaced \
373 by tenant + route + fingerprint). URLs found: {found:?}"
374 ));
375 }
376
377 #[cfg(feature = "redis-cache")]
378 {
379 let log = crate::experiment::RedisExperimentLog::connect(url).await?;
381 tracing::info!("experiment log: shared via redis");
382 Ok(Some(
383 Arc::new(log) as Arc<dyn crate::experiment::ExperimentLogBackend>
384 ))
385 }
386
387 #[cfg(not(feature = "redis-cache"))]
388 {
389 let _ = url;
390 Err(
391 "[route.challenger] redis_url is set, but this binary was built without the \
392 `redis-cache` feature. Rebuild with `--features redis-cache`, or remove redis_url \
393 from all [route.challenger] sections to keep experiment state in-process."
394 .to_owned(),
395 )
396 }
397}
398
399#[must_use]
400pub fn build_tenant_rate_limiter(
401 config: &ProxyConfig,
402) -> Option<Arc<governor::DefaultKeyedRateLimiter<String>>> {
403 let per_sec = config.tenant_rate_per_sec?;
404 Some(Arc::new(governor::RateLimiter::keyed(
405 governor::Quota::per_second(per_sec),
406 )))
407}
408
409pub async fn tenant_rate_limit_middleware(
413 State(state): State<AppState>,
414 Extension(tenant): Extension<TenantId>,
415 req: Request,
416 next: Next,
417) -> Response {
418 if let Some(limiter) = &state.tenant_rate_limiter
419 && limiter.check_key(&tenant.0).is_err()
420 {
421 return ProxyError::RateLimited.into_response();
422 }
423 next.run(req).await
424}
425
426impl std::fmt::Debug for AppState {
427 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
428 f.debug_struct("AppState")
429 .field("config", &self.config)
430 .finish_non_exhaustive()
431 }
432}
433
434fn offer_trace(traces: &store::TraceSender, spill: Option<&store::SpillHandle>, trace: Trace) {
451 record_trace_metrics(&trace);
452 match traces.try_send(trace) {
453 Ok(()) => {}
454 Err(TrySendError::Full(t)) => {
455 if let Some(handle) = spill {
456 match store::append_to_spill(handle, &t) {
457 Ok(()) => {
458 metrics::counter!("firstpass_receipts_spilled_total").increment(1);
459 }
460 Err(e) => {
461 tracing::error!(%e, "durable mode: spill write failed; trace lost");
462 metrics::counter!("firstpass_traces_dropped_total").increment(1);
463 }
464 }
465 } else {
466 tracing::warn!("trace channel full; dropping trace (writer behind under load)");
467 metrics::counter!("firstpass_traces_dropped_total").increment(1);
468 }
469 }
470 Err(TrySendError::Closed(_)) => {
471 tracing::warn!("trace writer is gone; dropping trace");
472 }
473 }
474}
475
476fn risk_certificate(
482 eprocess: Option<(firstpass_core::eprocess::Certification, f64, f64)>,
483 aci: Option<(f64, f64)>,
484 fixed: Option<f64>,
485) -> Option<RiskCertificate> {
486 let cert = |source, threshold, alpha| RiskCertificate {
487 source,
488 threshold,
489 alpha,
490 delta: None,
491 e_value: None,
492 certified_at_round: None,
493 };
494 if let Some((c, alpha, delta)) = eprocess {
495 return Some(RiskCertificate {
496 delta: Some(delta),
497 e_value: Some(c.e_value),
498 certified_at_round: Some(c.certified_at_round),
499 ..cert(ThresholdSource::Eprocess, c.threshold, Some(alpha))
500 });
501 }
502 aci.map(|(t, alpha)| cert(ThresholdSource::Aci, t, Some(alpha)))
503 .or_else(|| fixed.map(|t| cert(ThresholdSource::Fixed, t, None)))
504}
505
506fn record_trace_metrics(trace: &Trace) {
507 if trace.mode == Mode::Enforce {
508 metrics::histogram!("firstpass_enforce_latency_ms")
509 .record(trace.final_.total_latency_ms as f64);
510 if trace.final_.escalations > 0 {
511 metrics::counter!("firstpass_escalations_total")
512 .increment(u64::from(trace.final_.escalations));
513 }
514 }
515 let served_from = match trace.final_.served_from {
516 ServedFrom::Attempt => "attempt",
517 ServedFrom::BestAttempt => "best_attempt",
518 ServedFrom::Error => "error",
519 ServedFrom::Cache => "cache",
523 };
524 metrics::counter!("firstpass_served_total", "served_from" => served_from).increment(1);
525 if trace.final_.served_from == ServedFrom::Error {
526 let provider = trace
534 .attempts
535 .last()
536 .map(|a| a.provider.clone())
537 .or_else(|| {
538 if trace.upstream_provider.is_empty() {
539 None
540 } else {
541 Some(trace.upstream_provider.clone())
542 }
543 })
544 .unwrap_or_else(|| "unknown".to_owned());
545 metrics::counter!("firstpass_upstream_failures_total", "provider" => provider).increment(1);
546 }
547
548 for a in &trace.attempts {
554 metrics::histogram!(
555 "firstpass_attempt_latency_ms",
556 "provider" => a.provider.clone(),
557 "rung" => a.rung.to_string()
558 )
559 .record(a.latency_ms as f64);
560 metrics::counter!(
561 "firstpass_attempt_total",
562 "provider" => a.provider.clone(),
563 "rung" => a.rung.to_string()
564 )
565 .increment(1);
566 metrics::gauge!(
567 "firstpass_attempt_cost_usd_total",
568 "provider" => a.provider.clone(),
569 "rung" => a.rung.to_string()
570 )
571 .increment(a.cost_usd);
572
573 for g in &a.gates {
577 let verdict = match g.verdict {
578 Verdict::Pass => "pass",
579 Verdict::Fail => "fail",
580 Verdict::Abstain => "abstain",
581 };
582 metrics::counter!(
583 "firstpass_gate_verdict_total",
584 "gate_id" => g.gate_id.clone(),
585 "verdict" => verdict
586 )
587 .increment(1);
588 metrics::histogram!("firstpass_gate_latency_ms", "gate_id" => g.gate_id.clone())
589 .record(g.ms as f64);
590 metrics::gauge!("firstpass_gate_cost_by_id_usd_total", "gate_id" => g.gate_id.clone())
591 .increment(g.cost_usd);
592 }
593 }
594 metrics::gauge!("firstpass_cost_usd_total").increment(trace.final_.total_cost_usd);
598 metrics::gauge!("firstpass_gate_cost_usd_total").increment(trace.final_.gate_cost_usd);
599 metrics::gauge!("firstpass_baseline_usd_total")
600 .increment(trace.final_.counterfactual_baseline_usd);
601 metrics::gauge!("firstpass_savings_usd_total").increment(trace.final_.savings_usd);
602 if let Some(rung) = trace.final_.served_rung {
604 let model = trace
605 .attempts
606 .iter()
607 .find(|a| a.rung == rung)
608 .map(|a| a.model.clone())
609 .unwrap_or_else(|| "unknown".to_owned());
610 metrics::counter!(
611 "firstpass_served_rung_total",
612 "rung" => rung.to_string(),
613 "model" => model
614 )
615 .increment(1);
616 }
617}
618
619const MAX_BODY_BYTES: usize = 16 * 1024 * 1024;
623
624pub(crate) fn u01(seed: u128) -> f64 {
635 let lo = splitmix64_finalise(seed as u64);
636 let hi = splitmix64_finalise((seed >> 64) as u64);
637 ((lo ^ hi) >> 11) as f64 * (1.0_f64 / (1u64 << 53) as f64)
639}
640
641fn splitmix64_finalise(mut z: u64) -> u64 {
642 z = z.wrapping_add(0x9E37_79B9_7F4A_7C15);
643 z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
644 z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
645 z ^ (z >> 31)
646}
647
648#[must_use]
655pub(crate) fn epsilon_propensity(chosen: u32, greedy: u32, epsilon: f64, k: usize) -> f64 {
656 let greedy_term = if chosen == greedy { 1.0 - epsilon } else { 0.0 };
657 greedy_term + epsilon / k as f64
658}
659
660pub fn app(state: AppState) -> Result<Router, ProxyError> {
667 crate::metrics::install()?;
668 let max_concurrency = state.config.max_concurrency;
669
670 let business = Router::new()
679 .route("/v1/messages", post(messages))
680 .route("/v1/messages/count_tokens", post(count_tokens_passthrough))
683 .route("/v1/chat/completions", post(chat_completions))
684 .route("/v1/responses", post(responses))
685 .route(
688 "/v1beta/models/{model_action}",
689 post(gemini_generate_content),
690 )
691 .route("/v1/feedback", post(feedback))
692 .route("/v1/capabilities", get(capabilities))
693 .route("/v1/models", get(models))
694 .route("/v1/receipts", get(receipts))
697 .route("/v1/experiments", get(experiments))
699 .layer(axum::middleware::from_fn_with_state(
700 state.clone(),
701 tenant_rate_limit_middleware,
702 ))
703 .layer(axum::middleware::from_fn_with_state(
704 state.clone(),
705 auth_middleware,
706 ));
707
708 Ok(Router::new()
709 .merge(business)
710 .route("/healthz", get(healthz))
711 .route("/metrics", get(crate::metrics::handler))
712 .route("/panel", get(panel))
715 .layer(axum::extract::DefaultBodyLimit::max(MAX_BODY_BYTES))
717 .layer(tower::limit::GlobalConcurrencyLimitLayer::new(
720 max_concurrency,
721 ))
722 .with_state(state))
723}
724
725async fn healthz() -> impl IntoResponse {
727 Json(serde_json::json!({
733 "status": "ok",
734 "service": "firstpass",
735 "version": env!("CARGO_PKG_VERSION"),
736 }))
737}
738
739const PANEL_HTML: &str = include_str!("panel.html");
743
744async fn receipts(
750 State(state): State<AppState>,
751 Extension(TenantId(tenant)): Extension<TenantId>,
752 axum::extract::Query(q): axum::extract::Query<ReceiptsQuery>,
753) -> Response {
754 let limit = q.limit.unwrap_or(20).clamp(1, 200);
756 let db = state.config.db_path.clone();
757 let (traces, summary) = match tokio::task::spawn_blocking(move || {
760 Ok::<_, crate::store::StoreError>((
761 crate::store::load_recent_tenant_traces(&db, &tenant, limit)?,
762 crate::store::tenant_receipts_summary(&db, &tenant)?,
763 ))
764 })
765 .await
766 {
767 Ok(Ok(t)) => t,
768 _ => (Vec::new(), crate::store::ReceiptsSummary::default()),
771 };
772 let summary = summary_json(&summary);
773 let items: Vec<serde_json::Value> = traces
774 .iter()
775 .map(|t| {
776 let f = &t.final_;
777 serde_json::json!({
778 "trace_id": t.trace_id,
779 "ts": t.ts,
780 "session_id": t.session_id,
781 "served_rung": f.served_rung,
782 "served_from": f.served_from,
783 "served_model": f.served_rung
788 .and_then(|r| t.attempts.iter().find(|a| a.rung == r))
789 .map(|a| a.model.clone()),
790 "attempts": t.attempts.iter().map(|a| serde_json::json!({
791 "rung": a.rung,
792 "model": a.model,
793 "verdict": a.verdict,
794 "cost_usd": a.cost_usd,
795 "latency_ms": a.latency_ms,
796 "gates": a.gates.iter().map(|g| serde_json::json!({
797 "gate_id": g.gate_id,
798 "verdict": g.verdict,
799 "score": g.score,
800 "reason": g.reason,
801 })).collect::<Vec<_>>(),
802 })).collect::<Vec<_>>(),
803 "escalations": f.escalations,
804 "total_latency_ms": f.total_latency_ms,
805 "total_cost_usd": f.total_cost_usd,
806 "baseline_usd": f.counterfactual_baseline_usd,
807 "savings_usd": f.savings_usd,
808 "prev_hash": t.prev_hash,
812 "hash": t.hash().ok(),
813 "experiment": t.experiment.as_ref().map(|e| serde_json::json!({
815 "arm": e.arm,
816 "percent": e.percent,
817 "bucket": e.bucket,
818 })),
819 "deferred": t.deferred.iter().map(|d| serde_json::json!({
820 "gate_id": d.gate_id,
821 "verdict": d.verdict,
822 "reporter": d.reporter,
823 })).collect::<Vec<_>>(),
824 })
825 })
826 .collect();
827 axum::Json(serde_json::json!({ "receipts": items, "summary": summary })).into_response()
828}
829
830fn summary_json(s: &crate::store::ReceiptsSummary) -> serde_json::Value {
833 let [pass, fail, abstain] = s.gates;
834 serde_json::json!({
835 "requests": s.requests,
836 "spent_usd": s.spent_usd,
837 "baseline_usd": s.baseline_usd,
838 "saved_usd": s.baseline_usd - s.spent_usd,
841 "first_pass": s.first_pass,
842 "escalated": s.escalated,
843 "gates": { "pass": pass, "fail": fail, "abstain": abstain },
844 "truncated": s.truncated,
845 })
846}
847
848async fn experiments(
857 State(state): State<AppState>,
858 Extension(TenantId(tenant)): Extension<TenantId>,
859) -> Response {
860 let Some(routing) = state.config.routing.as_ref() else {
861 return axum::Json(serde_json::json!({ "experiments": [], "configured": [] }))
862 .into_response();
863 };
864 let snaps: Vec<_> = state
865 .experiments
866 .reports()
867 .into_iter()
868 .filter(|s| s.tenant == tenant)
869 .collect();
870 let views = crate::experiment::views(snaps, routing);
871 let configured: Vec<serde_json::Value> = routing
872 .routes
873 .iter()
874 .enumerate()
875 .filter_map(|(ix, r)| r.challenger.as_ref().map(|c| (ix, r, c)))
876 .map(|(ix, r, c)| {
877 let status = state.experiments.status(&tenant, ix);
878 let paired = c.is_paired().then(|| {
881 serde_json::json!({
882 "sample_rate": c.sample_rate,
883 "max_usd_per_day": c.max_usd_per_day,
884 "spent_today_usd": state.paired_ledger.spent_today(
885 &tenant,
886 ix,
887 jiff::Timestamp::now(),
888 ),
889 })
890 });
891 serde_json::json!({
892 "route_ix": ix,
893 "incumbent_ladder": r.ladder,
894 "challenger_ladder": c.ladder,
895 "mode": if c.is_paired() { "paired" } else { "live" },
896 "paired": paired,
897 "percent": c.percent,
898 "margin": c.margin,
899 "alpha": c.alpha,
900 "unavailable": state.experiments.challenger_blocked(&tenant, ix)
901 && status != firstpass_core::ExperimentStatus::Halted,
902 })
903 })
904 .collect();
905 axum::Json(serde_json::json!({ "experiments": views, "configured": configured }))
906 .into_response()
907}
908
909#[derive(Debug, serde::Deserialize)]
911struct ReceiptsQuery {
912 limit: Option<usize>,
914}
915
916async fn panel() -> impl IntoResponse {
922 (
923 [
924 (
925 axum::http::header::CONTENT_TYPE,
926 "text/html; charset=utf-8".to_owned(),
927 ),
928 (
929 axum::http::header::CONTENT_SECURITY_POLICY,
930 PANEL_CSP.clone(),
931 ),
932 (
933 axum::http::header::X_CONTENT_TYPE_OPTIONS,
934 "nosniff".to_owned(),
935 ),
936 (
937 axum::http::header::REFERRER_POLICY,
938 "no-referrer".to_owned(),
939 ),
940 ],
941 PANEL_HTML,
942 )
943}
944
945static PANEL_CSP: std::sync::LazyLock<String> = std::sync::LazyLock::new(|| {
952 use sha2::Digest as _;
953 let script = PANEL_HTML
954 .split_once("<script>")
955 .and_then(|(_, rest)| rest.split_once("</script>"))
956 .map_or("", |(body, _)| body);
957 let digest = sha2::Sha256::digest(script.as_bytes());
958 format!(
959 "default-src 'none'; script-src 'sha256-{}'; style-src 'unsafe-inline'; \
960 connect-src 'self'; img-src 'self' data:; base-uri 'none'; form-action 'none'; \
961 frame-ancestors 'none'",
962 base64_std(&digest)
963 )
964});
965
966fn base64_std(bytes: &[u8]) -> String {
969 const ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
970 let mut out = String::with_capacity(bytes.len().div_ceil(3) * 4);
971 for chunk in bytes.chunks(3) {
972 let b = [
973 chunk[0],
974 chunk.get(1).copied().unwrap_or(0),
975 chunk.get(2).copied().unwrap_or(0),
976 ];
977 let n = (u32::from(b[0]) << 16) | (u32::from(b[1]) << 8) | u32::from(b[2]);
978 for (i, shift) in [18u32, 12, 6, 0].into_iter().enumerate() {
979 if i <= chunk.len() {
980 out.push(char::from(ALPHABET[((n >> shift) & 63) as usize]));
981 } else {
982 out.push('=');
983 }
984 }
985 }
986 out
987}
988
989async fn capabilities(State(state): State<AppState>) -> impl IntoResponse {
990 let (ladder, gates) = state
993 .config
994 .routing
995 .as_ref()
996 .and_then(|c| c.routes.iter().find(|r| r.mode == Mode::Enforce))
997 .map(|r| (r.ladder.clone(), r.gates.clone()))
998 .unwrap_or_default();
999 let routing_modes: Vec<serde_json::Value> = RoutingMode::ALL
1000 .iter()
1001 .map(|m| {
1002 let p = m.preset();
1003 serde_json::json!({
1004 "name": m.as_str(),
1005 "description": p.description,
1006 "tradeoff": p.tradeoff,
1007 })
1008 })
1009 .collect();
1010 Json(serde_json::json!({
1011 "service": "firstpass",
1012 "version": env!("CARGO_PKG_VERSION"),
1013 "feature_version": FEATURE_VERSION,
1014 "modes": ["observe", "enforce"],
1015 "routing_modes": routing_modes,
1016 "wire_apis": ["anthropic.messages", "openai.chat_completions", "openai.responses", "gemini.generate_content"],
1017 "ladder": ladder,
1018 "gates": gates,
1019 "feedback_api": "POST /v1/feedback",
1020 "offboarding": "unset ANTHROPIC_BASE_URL (or OPENAI_BASE_URL for OpenAI clients, GOOGLE_GEMINI_BASE_URL for Gemini CLI)",
1021 }))
1022}
1023
1024async fn models(State(state): State<AppState>) -> impl IntoResponse {
1036 Json(model_list(state.config.routing.as_ref()))
1037}
1038
1039fn model_list(routing: Option<&firstpass_core::config::Config>) -> serde_json::Value {
1042 let prices = routing.map(|c| c.price_table()).unwrap_or_default();
1043 let mut seen = std::collections::HashSet::new();
1046 let data: Vec<serde_json::Value> = routing
1047 .map(|c| c.routes.as_slice())
1048 .unwrap_or_default()
1049 .iter()
1050 .flat_map(|r| r.ladder.iter())
1051 .filter(|id| seen.insert(id.as_str()))
1052 .map(|id| {
1053 let price = prices.get(id);
1054 serde_json::json!({
1055 "id": id,
1056 "object": "model",
1057 "created": 0,
1060 "owned_by": id.split_once('/').map_or("firstpass", |(p, _)| p),
1061 "firstpass": {
1062 "input_per_mtok": price.map(|p| p.input_per_mtok),
1063 "output_per_mtok": price.map(|p| p.output_per_mtok),
1064 },
1065 })
1066 })
1067 .collect();
1068 serde_json::json!({ "object": "list", "data": data })
1069}
1070
1071#[derive(Debug, Deserialize)]
1073struct FeedbackRequest {
1074 trace_id: String,
1076 gate_id: String,
1078 verdict: String,
1080 #[serde(default)]
1082 score: Option<f64>,
1083 reporter: String,
1085}
1086
1087async fn feedback(
1092 State(state): State<AppState>,
1093 Extension(TenantId(tenant)): Extension<TenantId>,
1094 body: Bytes,
1095) -> Response {
1096 let req: FeedbackRequest = match serde_json::from_slice(&body) {
1097 Ok(r) => r,
1098 Err(e) => {
1099 return ProxyError::BadRequest(format!("invalid feedback body: {e}")).into_response();
1100 }
1101 };
1102 let verdict = match req.verdict.as_str() {
1103 "pass" => Verdict::Pass,
1104 "fail" => Verdict::Fail,
1105 "abstain" => Verdict::Abstain,
1106 other => {
1107 return ProxyError::BadRequest(format!("unknown verdict {other:?}")).into_response();
1108 }
1109 };
1110 let score = match req.score {
1111 Some(s) => match Score::new(s) {
1112 Ok(sc) => Some(sc),
1113 Err(_) => {
1114 return ProxyError::BadRequest(format!("score {s} out of range [0,1]"))
1115 .into_response();
1116 }
1117 },
1118 None => None,
1119 };
1120
1121 let db = state.config.db_path.clone();
1122
1123 let (db_check, tenant_check, tid_check) = (db.clone(), tenant.clone(), req.trace_id.clone());
1128 match tokio::task::spawn_blocking(move || {
1129 store::trace_exists(&db_check, &tenant_check, &tid_check)
1130 })
1131 .await
1132 {
1133 Ok(Ok(true)) => {}
1134 Ok(Ok(false)) => {
1135 return ProxyError::NotFound(format!("unknown trace_id {:?}", req.trace_id))
1136 .into_response();
1137 }
1138 Ok(Err(e)) => {
1139 tracing::error!(%e, "feedback: trace_exists check failed");
1140 return ProxyError::Internal(e.to_string()).into_response();
1141 }
1142 Err(e) => {
1143 tracing::error!(%e, "feedback: trace_exists task panicked");
1144 return ProxyError::Internal(e.to_string()).into_response();
1145 }
1146 }
1147
1148 if verdict == Verdict::Fail
1153 && let Some(cache) = state.verified_cache.as_ref()
1154 {
1155 let reported = req.trace_id.parse::<uuid::Uuid>().ok();
1160 let origin = {
1161 let (d, t, i) = (db.clone(), tenant.clone(), req.trace_id.clone());
1162 tokio::task::spawn_blocking(move || store::trace_cache_source(&d, &t, &i))
1163 .await
1164 .ok()
1165 .and_then(Result::ok)
1166 .flatten()
1167 .flatten()
1168 };
1169 let mut dropped = 0usize;
1170 for id in reported.into_iter().chain(origin) {
1171 dropped += cache.retract(id).await;
1172 }
1173 if dropped > 0 {
1174 tracing::info!(
1175 trace = %req.trace_id,
1176 ?origin,
1177 dropped,
1178 "verified cache: retracted a disproven answer"
1179 );
1180 metrics::counter!("firstpass_cache_retractions_total").increment(dropped as u64);
1181 }
1182 }
1183
1184 let feedback_signal = match verdict {
1186 Verdict::Pass => Some(true),
1187 Verdict::Fail => Some(false),
1188 Verdict::Abstain => None,
1189 };
1190 let dv = DeferredVerdict {
1191 gate_id: req.gate_id,
1192 verdict,
1193 score,
1194 reported_at: jiff::Timestamp::now(),
1195 reporter: req.reporter,
1196 };
1197 let trace_id = req.trace_id.clone();
1198 match tokio::task::spawn_blocking(move || store::append_deferred(&db, &req.trace_id, &dv)).await
1199 {
1200 Ok(Ok(())) => {
1201 if let (Some(a), Some(correct)) = (state.adaptive.as_ref(), feedback_signal)
1203 && let Ok(mut g) = a.lock()
1204 {
1205 g.observe_served(correct);
1206 metrics::gauge!("firstpass_serve_threshold").set(g.threshold());
1207 metrics::gauge!("firstpass_realized_served_failure")
1208 .set(g.realized_served_failure());
1209 }
1210
1211 if let (Some(e), Some(correct)) = (state.eprocess.as_ref(), feedback_signal) {
1234 let db_s = state.config.db_path.clone();
1235 let (tenant_s, tid_s) = (tenant.clone(), trace_id.clone());
1236 let served = tokio::task::spawn_blocking(move || {
1237 store::load_trace_view(&db_s, &tenant_s, &tid_s)
1238 .ok()
1239 .flatten()
1240 .and_then(|t| {
1241 let rung = t.final_.served_rung?;
1242 let a = t.attempts.iter().find(|a| a.rung == rung)?;
1243 Some(crate::calibrate::gate_score(&a.gates, a.verdict))
1244 })
1245 })
1246 .await
1247 .ok()
1248 .flatten();
1249 if let (Some(sc), Ok(mut g)) = (served, e.lock()) {
1252 g.observe_served(sc, correct);
1253 metrics::counter!("firstpass_eprocess_rounds_total").increment(1);
1254 }
1255 }
1256
1257 let attributed_route = tokio::task::spawn_blocking({
1262 let (db, ten, tid) = (
1263 state.config.db_path.clone(),
1264 tenant.clone(),
1265 trace_id.clone(),
1266 );
1267 move || store::trace_route(&db, &ten, &tid)
1268 })
1269 .await
1270 .ok()
1271 .and_then(Result::ok)
1272 .flatten()
1273 .flatten()
1274 .unwrap_or(0) as usize;
1275
1276 if let (Some(cfg), Some(correct)) = (
1281 state.config.routing.as_ref().and_then(|r| r.guardrail),
1282 feedback_signal,
1283 ) {
1284 let cooldown = state
1285 .config
1286 .routing
1287 .as_ref()
1288 .map_or(3_600, |r| r.guardrail_cooldown_secs);
1289 let reaction = state.guardrails.record(
1290 &tenant,
1291 attributed_route,
1292 &cfg,
1293 correct,
1294 jiff::Timestamp::now().as_second(),
1295 cooldown,
1296 );
1297 match &reaction {
1298 crate::guard::Reaction::Demoted(v) => {
1299 metrics::counter!("firstpass_guardrail_demotions_total").increment(1);
1300 tracing::error!(
1301 tenant = %tenant,
1302 n = v.n,
1303 rate = v.rate,
1304 bound = v.bound,
1305 alpha = cfg.alpha,
1306 "GUARDRAIL: served-failure bound exceeded target — route demoted to observe; traffic now serves as it would without Firstpass"
1307 );
1308 }
1309 crate::guard::Reaction::Alarmed(v) => {
1310 metrics::counter!("firstpass_guardrail_breaches_total").increment(1);
1311 tracing::error!(
1312 tenant = %tenant,
1313 n = v.n,
1314 rate = v.rate,
1315 bound = v.bound,
1316 alpha = cfg.alpha,
1317 "GUARDRAIL: served-failure bound exceeded target (alarm only — routing unchanged)"
1318 );
1319 }
1320 crate::guard::Reaction::None => {}
1321 }
1322 }
1323 if state.experiments.is_active() && verdict != Verdict::Abstain {
1330 let (db_x, ten_x, tid_x) = (
1331 state.config.db_path.clone(),
1332 tenant.clone(),
1333 trace_id.clone(),
1334 );
1335 match tokio::task::spawn_blocking(move || {
1336 store::trace_experiment(&db_x, &ten_x, &tid_x)
1337 })
1338 .await
1339 {
1340 Ok(Ok(Some((session, route_ix, rec, cost_usd)))) => {
1341 state
1342 .experiments
1343 .record_outcome(
1344 &tenant,
1345 route_ix,
1346 &rec.fingerprint,
1347 rec.arm,
1348 &session,
1349 verdict,
1350 cost_usd,
1351 )
1352 .await;
1353 }
1354 Ok(Ok(None)) => {}
1355 Ok(Err(e)) => {
1356 tracing::warn!(%e, "feedback: experiment attribution lookup failed");
1357 }
1358 Err(e) => {
1359 tracing::warn!(%e, "feedback: experiment attribution task failed");
1360 }
1361 }
1362 }
1363 (
1364 axum::http::StatusCode::ACCEPTED,
1365 Json(serde_json::json!({ "status": "recorded", "trace_id": trace_id })),
1366 )
1367 .into_response()
1368 }
1369 Ok(Err(e)) => {
1370 tracing::error!(%e, "feedback: append_deferred failed");
1371 ProxyError::Internal(e.to_string()).into_response()
1372 }
1373 Err(e) => {
1374 tracing::error!(%e, "feedback: append_deferred task panicked");
1375 ProxyError::Internal(e.to_string()).into_response()
1376 }
1377 }
1378}
1379
1380const SESSION_HEADER: &str = "x-firstpass-session";
1383
1384const AGENT_HEADER: &str = "x-firstpass-agent";
1386const SUBAGENT_HEADER: &str = "x-firstpass-subagent";
1388#[expect(
1394 clippy::too_many_arguments,
1395 reason = "mirrors the enforce context it evaluates; a wrapper struct would only move the list"
1396)]
1397fn spawn_shadow(
1398 state: &AppState,
1399 route: &firstpass_core::Route,
1400 route_ix: usize,
1401 body: Bytes,
1402 auth: Auth,
1403 features: Features,
1404 tenant: String,
1405 session_id: String,
1406 api: &str,
1407) {
1408 let Some(shadow) = route.shadow else {
1409 return;
1410 };
1411 if !shadow.sampled(&state.config.prompt_salt, &session_id) {
1414 return;
1415 }
1416 let (state, route, api) = (state.clone(), route.clone(), api.to_owned());
1417 tokio::spawn(async move {
1418 let signal = evaluate_shadow(
1419 &state, &route, shadow, route_ix, &body, auth, &features, tenant, session_id, &api, 0.0,
1420 )
1421 .await;
1422 tracing::debug!(
1424 would_pass = signal.would_pass,
1425 projected_usd = signal.projected_cost_usd,
1426 skipped = ?signal.skipped,
1427 "shadow evaluation complete"
1428 );
1429 });
1430}
1431
1432#[expect(
1439 clippy::too_many_arguments,
1440 reason = "mirrors the enforce context; grouping them into a struct would only move the list"
1441)]
1442async fn evaluate_shadow(
1443 state: &AppState,
1444 route: &firstpass_core::Route,
1445 shadow: firstpass_core::rollout::Shadow,
1446 route_ix: usize,
1447 body: &Bytes,
1448 auth: Auth,
1449 features: &Features,
1450 tenant: String,
1451 session_id: String,
1452 api: &str,
1453 actual_cost_usd: f64,
1454) -> ShadowSignal {
1455 let skipped = |why: &str| ShadowSignal {
1456 would_serve_rung: None,
1457 would_pass: false,
1458 projected_cost_usd: 0.0,
1459 actual_cost_usd,
1460 skipped: Some(why.to_owned()),
1461 };
1462
1463 let now = jiff::Timestamp::now();
1464 if !state
1465 .shadow_ledger
1466 .may_spend(&tenant, route_ix, shadow.max_usd_per_day, now)
1467 {
1468 return skipped("budget_exhausted");
1471 }
1472
1473 let Some(base_request) = parse_model_request(body) else {
1474 return skipped("unparseable_request");
1475 };
1476 let gate_defs = state
1477 .config
1478 .routing
1479 .as_ref()
1480 .map_or(&[][..], |cfg| &cfg.gate_defs);
1481 let gates = resolve_gates(
1482 &route.gates,
1483 gate_defs,
1484 &state.providers,
1485 &auth,
1486 &state.config.prices,
1487 &state.http,
1488 );
1489 let (budget, max_rungs) = state
1490 .config
1491 .routing
1492 .as_ref()
1493 .map_or((None, u32::MAX), |cfg| {
1494 (
1495 cfg.budget.per_request_usd,
1496 cfg.escalation.max_rungs_per_request,
1497 )
1498 });
1499
1500 let mut base_request = base_request;
1501 base_request.cache_prefix = state
1505 .config
1506 .routing
1507 .as_ref()
1508 .is_some_and(|r| r.escalation.prompt_cache);
1509 let ctx = EnforceCtx {
1510 condense: routing_cfg_condense(state),
1511 ladder: &route.ladder,
1512 gates: &gates,
1513 health: &state.gate_health,
1514 base_request: &base_request,
1515 providers: &state.providers,
1516 auth: &auth,
1517 prices: &state.config.prices,
1518 budget_per_request_usd: budget,
1519 max_rungs,
1520 speculation: 0,
1523 serve_threshold: None,
1524 elastic: None,
1525 features: features.clone(),
1526 start_rung: 0,
1527 tenant_id: tenant.clone(),
1528 session_id,
1529 prompt_hash: prompt_hash(&state.config.prompt_salt, body),
1530 api: api.to_owned(),
1531 policy_id: "shadow".to_owned(),
1532 };
1533
1534 let (outcome, trace) = route_enforce(ctx).await;
1535 let spent = trace.final_.total_cost_usd;
1536 state.shadow_ledger.debit(&tenant, route_ix, spent, now);
1537
1538 let would_pass = matches!(outcome, EngineOutcome::Served(_))
1542 && trace.final_.served_from == firstpass_core::ServedFrom::Attempt;
1543 let would_serve_rung = if would_pass {
1544 trace.final_.served_rung
1545 } else {
1546 None
1547 };
1548 ShadowSignal {
1549 would_serve_rung,
1550 would_pass,
1551 projected_cost_usd: spent,
1552 actual_cost_usd,
1553 skipped: None,
1554 }
1555}
1556
1557fn pair_outcome(served_from: firstpass_core::ServedFrom) -> Option<f64> {
1562 match served_from {
1563 firstpass_core::ServedFrom::Attempt => Some(1.0),
1564 firstpass_core::ServedFrom::BestAttempt => Some(0.0),
1565 firstpass_core::ServedFrom::Cache | firstpass_core::ServedFrom::Error => None,
1566 }
1567}
1568
1569#[allow(clippy::too_many_arguments)]
1579fn spawn_paired_challenger(
1580 state: &AppState,
1581 route: &firstpass_core::Route,
1582 challenger: &firstpass_core::Challenger,
1583 fingerprint: String,
1584 route_ix: usize,
1585 body: bytes::Bytes,
1586 auth: Auth,
1587 features: firstpass_core::Features,
1588 tenant: String,
1589 session_id: String,
1590 api: &str,
1591 served_from: firstpass_core::ServedFrom,
1592 incumbent_cost_usd: f64,
1593 incumbent_trace_id: uuid::Uuid,
1594) {
1595 let Some(x_inc) = pair_outcome(served_from) else {
1596 return;
1597 };
1598
1599 if state.experiments.challenger_blocked(&tenant, route_ix) {
1604 return;
1605 }
1606
1607 let max_usd_per_day = match challenger.max_usd_per_day {
1608 Some(v) => v,
1609 None => {
1610 tracing::warn!(
1611 tenant,
1612 route = route_ix,
1613 "paired challenger is missing max_usd_per_day; skipping evaluation"
1614 );
1615 return;
1616 }
1617 };
1618
1619 let state = state.clone();
1620 let route = route.clone();
1621 let challenger = challenger.clone();
1622 let api = api.to_owned();
1623
1624 tokio::spawn(async move {
1625 if !state
1630 .experiments
1631 .ensure_synced(&tenant, route_ix, &fingerprint)
1632 .await
1633 {
1634 tracing::debug!(
1635 tenant,
1636 route = route_ix,
1637 "paired challenger: ensure_synced failed (log error), skipping evaluation"
1638 );
1639 return;
1640 }
1641
1642 if state.experiments.challenger_blocked(&tenant, route_ix) {
1644 return;
1645 }
1646
1647 let now = jiff::Timestamp::now();
1652 if !state
1653 .paired_ledger
1654 .may_spend(&tenant, route_ix, max_usd_per_day, now)
1655 {
1656 tracing::debug!(
1657 tenant,
1658 route = route_ix,
1659 "paired challenger: daily budget exhausted, skipping"
1660 );
1661 return;
1662 }
1663
1664 let Some(base_request) = parse_model_request(&body) else {
1665 return;
1666 };
1667 let gate_defs = state
1668 .config
1669 .routing
1670 .as_ref()
1671 .map_or(&[][..], |cfg| &cfg.gate_defs);
1672 let gates = resolve_gates(
1673 &route.gates,
1674 gate_defs,
1675 &state.providers,
1676 &auth,
1677 &state.config.prices,
1678 &state.http,
1679 );
1680 let (budget, max_rungs) = state
1681 .config
1682 .routing
1683 .as_ref()
1684 .map_or((None, u32::MAX), |cfg| {
1685 (
1686 cfg.budget.per_request_usd,
1687 cfg.escalation.max_rungs_per_request,
1688 )
1689 });
1690 let mut base_request = base_request;
1691 base_request.cache_prefix = state
1692 .config
1693 .routing
1694 .as_ref()
1695 .is_some_and(|r| r.escalation.prompt_cache);
1696
1697 let ctx = EnforceCtx {
1698 condense: routing_cfg_condense(&state),
1699 ladder: &challenger.ladder,
1700 gates: &gates,
1701 health: &state.gate_health,
1702 base_request: &base_request,
1703 providers: &state.providers,
1704 auth: &auth,
1705 prices: &state.config.prices,
1706 budget_per_request_usd: budget,
1707 max_rungs,
1708 speculation: 0,
1709 serve_threshold: None,
1710 elastic: None,
1711 features,
1712 start_rung: 0,
1713 tenant_id: tenant.clone(),
1714 session_id: session_id.clone(),
1715 prompt_hash: prompt_hash(&state.config.prompt_salt, &body),
1716 api: api.clone(),
1717 policy_id: "paired-challenger@v0".to_owned(),
1718 };
1719
1720 let (_, ch_trace) = route_enforce(ctx).await;
1721
1722 let pair_ts = jiff::Timestamp::now();
1728
1729 let challenger_cost = ch_trace.final_.total_cost_usd;
1730 let ch_errored = ch_trace.final_.served_from == firstpass_core::ServedFrom::Error;
1731 let ch_trunc = crate::experiment::truncated_attempts(&ch_trace);
1732
1733 state
1738 .paired_ledger
1739 .debit(&tenant, route_ix, challenger_cost, pair_ts);
1740 state.experiments.observe_health(
1741 &tenant,
1742 route_ix,
1743 &fingerprint,
1744 firstpass_core::Arm::Challenger,
1745 ch_errored,
1746 ch_trunc,
1747 );
1748
1749 let x_ch = match ch_trace.final_.served_from {
1754 firstpass_core::ServedFrom::Attempt => 1.0,
1755 firstpass_core::ServedFrom::BestAttempt | firstpass_core::ServedFrom::Error => 0.0,
1756 firstpass_core::ServedFrom::Cache => return,
1759 };
1760
1761 let appended = state
1781 .experiments
1782 .record_pair(
1783 &tenant,
1784 route_ix,
1785 &fingerprint,
1786 x_inc,
1787 x_ch,
1788 incumbent_cost_usd,
1789 challenger_cost,
1790 ch_errored,
1791 ch_trunc,
1792 pair_ts.as_second(),
1793 ch_trace.trace_id,
1794 incumbent_trace_id,
1795 )
1796 .await;
1797
1798 let pair_row = crate::store::ExperimentPairRow {
1799 tenant: tenant.clone(),
1800 route_ix,
1801 fingerprint: fingerprint.clone(),
1802 trace_id: ch_trace.trace_id,
1803 x_inc,
1804 x_ch,
1805 challenger_cost_usd: challenger_cost,
1806 ch_error: ch_errored,
1807 ch_truncated: ch_trunc,
1808 ts: pair_ts,
1809 in_log: if appended {
1813 crate::store::InLogStatus::InLog
1814 } else {
1815 crate::store::InLogStatus::NotInLog
1816 },
1817 };
1818
1819 if appended {
1820 if let Err(e) = crate::store::append_experiment_pair(&state.config.db_path, &pair_row) {
1823 tracing::warn!(%e, "paired challenger: failed to persist InLog row to store");
1824 }
1825 } else {
1827 match crate::store::append_experiment_pair(&state.config.db_path, &pair_row) {
1830 Ok(()) => {
1831 state.experiments.apply_pair_locally(
1835 &tenant,
1836 route_ix,
1837 &fingerprint,
1838 x_inc,
1839 x_ch,
1840 challenger_cost,
1841 ch_trace.trace_id,
1842 );
1843 }
1844 Err(e) => {
1845 tracing::warn!(
1849 %e,
1850 "paired challenger: SQLite write failed; evidence withheld for this pair"
1851 );
1852 }
1853 }
1854 }
1855
1856 tracing::debug!(
1857 tenant,
1858 route = route_ix,
1859 x_inc,
1860 x_ch,
1861 challenger_cost_usd = challenger_cost,
1862 "paired challenger evaluation complete"
1863 );
1864 });
1865}
1866
1867const MODE_PROFILE_HEADER: &str = "x-firstpass-mode";
1870
1871fn resolve_mode(headers: &HeaderMap, route: &Route, config: &ProxyConfig) -> RoutingMode {
1880 if let Some(val) = header_str(headers, MODE_PROFILE_HEADER) {
1882 match val.trim().to_ascii_lowercase().as_str() {
1883 "observe" => return RoutingMode::Observe,
1884 "cost" => return RoutingMode::Cost,
1885 "balanced" => return RoutingMode::Balanced,
1886 "quality" => return RoutingMode::Quality,
1887 "latency" => return RoutingMode::Latency,
1888 "max" => return RoutingMode::Max,
1889 other => {
1890 tracing::warn!(
1891 value = other,
1892 "unknown x-firstpass-mode value; ignoring \
1893 (valid: observe|cost|balanced|quality|latency|max)"
1894 );
1895 }
1896 }
1897 }
1898 if let Some(m) = route.routing_mode {
1900 return m;
1901 }
1902 config.default_routing_mode
1904}
1905
1906async fn count_tokens_passthrough(
1918 State(state): State<AppState>,
1919 Extension(TenantId(_tenant)): Extension<TenantId>,
1920 headers: HeaderMap,
1921 body: Bytes,
1922) -> Response {
1923 match crate::upstream::forward_anthropic_path(
1924 &state.http,
1925 &state.config.upstream_anthropic,
1926 "/v1/messages/count_tokens",
1927 &headers,
1928 body,
1929 )
1930 .await
1931 {
1932 Ok((status, resp_headers, resp_body)) => (status, resp_headers, resp_body).into_response(),
1933 Err(err) => err.into_response(),
1934 }
1935}
1936
1937async fn messages(
1942 State(state): State<AppState>,
1943 Extension(TenantId(tenant)): Extension<TenantId>,
1944 headers: HeaderMap,
1945 body: Bytes,
1946) -> Response {
1947 let mut session_header = header_session(&headers, &state.config.prompt_salt);
1949
1950 if let Some(routing) = state.config.routing.as_ref() {
1953 let features = extract_features(&headers, &body);
1954 if let Some(route) = routing
1955 .route_for(&features)
1956 .filter(|r| r.mode == Mode::Enforce && !r.ladder.is_empty())
1957 {
1958 session_header = session_header
1961 .take()
1962 .or_else(|| body_session_bytes(&body, &state.config.prompt_salt));
1963 let route_ix = routing
1966 .routes
1967 .iter()
1968 .position(|r| std::ptr::eq(r, route))
1969 .unwrap_or(0);
1970 let route = route.clone();
1973 let routing_mode = resolve_mode(&headers, &route, &state.config);
1975 if routing_mode == RoutingMode::Observe {
1977 return observe_passthrough(state, headers, body, session_header, tenant).await;
1978 }
1979 if state
1983 .guardrails
1984 .is_demoted(&tenant, route_ix, jiff::Timestamp::now().as_second())
1985 {
1986 return observe_passthrough(state, headers, body, session_header, tenant).await;
1987 }
1988 if let Some(rollout) = route.rollout.as_ref() {
1993 let key_value = match rollout.key {
1994 firstpass_core::RolloutKey::Session => session_header
1995 .clone()
1996 .unwrap_or_else(|| firstpass_core::rollout::request_identity(&body)),
1997 firstpass_core::RolloutKey::Request => {
2003 firstpass_core::rollout::request_identity(&body)
2004 }
2005 firstpass_core::RolloutKey::Tenant => tenant.to_string(),
2006 };
2007 let decision =
2008 firstpass_core::rollout::decide(&state.config.prompt_salt, rollout, &key_value);
2009 if !decision.enforced {
2010 let shadow_ctx = route.shadow.map(|_| {
2012 (
2013 body.clone(),
2014 Auth::from_headers(&headers, firstpass_core::Dialect::Anthropic),
2015 features.clone(),
2016 tenant.clone(),
2017 session_header
2018 .clone()
2019 .unwrap_or_else(|| Uuid::now_v7().to_string()),
2020 )
2021 });
2022 let resp =
2023 observe_passthrough(state.clone(), headers, body, session_header, tenant)
2024 .await;
2025 if let Some((b, auth, feats, ten, sess)) = shadow_ctx {
2028 spawn_shadow(
2029 &state,
2030 &route,
2031 route_ix,
2032 b,
2033 auth,
2034 feats,
2035 ten,
2036 sess,
2037 "anthropic",
2038 );
2039 }
2040 return resp;
2041 }
2042 }
2043 if enforce_can_handle_route(
2044 &features,
2045 &body,
2046 routing.escalation.enforce_structured,
2047 &route,
2048 &state.providers,
2049 Dialect::Anthropic,
2050 ) {
2051 return handle_enforce(
2052 &state,
2053 &headers,
2054 &body,
2055 features,
2056 &route,
2057 route_ix,
2058 session_header,
2059 tenant,
2060 routing_mode,
2061 )
2062 .await;
2063 }
2064 tracing::info!(
2068 "enforce route matched but structured request can't be routed faithfully (flag/ladder); serving via observe passthrough"
2069 );
2070 }
2071 }
2072 observe_passthrough(state, headers, body, session_header, tenant).await
2073}
2074
2075fn enforce_can_handle_route(
2092 features: &Features,
2093 body: &[u8],
2094 enforce_structured: bool,
2095 route: &Route,
2096 providers: &crate::provider::ProviderRegistry,
2097 inbound: Dialect,
2098) -> bool {
2099 enforce_can_handle(
2100 features,
2101 body,
2102 enforce_structured,
2103 &route.ladder,
2104 providers,
2105 inbound,
2106 ) && route.challenger.as_ref().is_none_or(|c| {
2107 enforce_can_handle(
2108 features,
2109 body,
2110 enforce_structured,
2111 &c.ladder,
2112 providers,
2113 inbound,
2114 )
2115 })
2116}
2117
2118fn enforce_can_handle(
2119 features: &Features,
2120 body: &[u8],
2121 enforce_structured: bool,
2122 ladder: &[String],
2123 providers: &crate::provider::ProviderRegistry,
2124 inbound: Dialect,
2125) -> bool {
2126 let structured = features.tool_count > 0
2127 || features.has_images
2128 || match inbound {
2129 Dialect::Anthropic => messages_have_tool_blocks(body),
2130 Dialect::Openai => openai_messages_have_tool_calls(body),
2131 Dialect::Gemini => false,
2132 };
2133 if !structured {
2134 return true;
2135 }
2136 if !enforce_structured {
2137 return false;
2138 }
2139 let all_verbatim = ladder.iter().all(|rung| {
2141 let provider_id = rung.split('/').next().unwrap_or_default();
2142 providers
2143 .get(provider_id)
2144 .is_some_and(|p| p.carries_structured_verbatim(inbound))
2145 });
2146 if all_verbatim {
2147 return true;
2148 }
2149 if inbound == Dialect::Openai && !openai_has_http_images(body) {
2153 let all_anthropic = ladder.iter().all(|rung| {
2154 let pid = rung.split('/').next().unwrap_or_default();
2155 providers
2156 .get(pid)
2157 .is_some_and(|p| p.carries_structured_verbatim(Dialect::Anthropic))
2158 });
2159 if all_anthropic {
2160 return true;
2161 }
2162 }
2163 false
2164}
2165
2166fn messages_have_tool_blocks(body: &[u8]) -> bool {
2169 serde_json::from_slice::<Value>(body)
2170 .ok()
2171 .and_then(|json| {
2172 json.get("messages")
2173 .and_then(Value::as_array)
2174 .map(|messages| messages.iter().any(message_has_tool_block))
2175 })
2176 .unwrap_or(false)
2177}
2178
2179fn message_has_tool_block(message: &Value) -> bool {
2181 message
2182 .get("content")
2183 .and_then(Value::as_array)
2184 .is_some_and(|blocks| {
2185 blocks.iter().any(|block| {
2186 matches!(
2187 block.get("type").and_then(Value::as_str),
2188 Some("tool_use" | "tool_result")
2189 )
2190 })
2191 })
2192}
2193
2194fn is_stream_request(body: &[u8]) -> bool {
2196 serde_json::from_slice::<Value>(body)
2197 .ok()
2198 .and_then(|json| json.get("stream").and_then(Value::as_bool))
2199 .unwrap_or(false)
2200}
2201
2202fn header_str(headers: &HeaderMap, name: &str) -> Option<String> {
2204 headers
2205 .get(name)
2206 .and_then(|v| v.to_str().ok())
2207 .map(str::to_owned)
2208}
2209
2210const AGENT_SESSION_HEADERS: [&str; 3] =
2212 ["x-claude-code-session-id", "session_id", "conversation_id"];
2213
2214const SESSION_MAX_BYTES: usize = 256;
2216
2217fn cap_session(s: &str) -> &str {
2218 let mut end = s.len().min(SESSION_MAX_BYTES);
2219 while !s.is_char_boundary(end) {
2220 end -= 1;
2221 }
2222 &s[..end]
2223}
2224
2225fn hashed_session(kind: &str, salt: &str, raw: &str) -> Option<String> {
2228 let h = prompt_hash(salt, cap_session(raw).as_bytes());
2229 Some(format!("{kind}:{}", h.get(..16)?))
2230}
2231
2232fn header_session(headers: &HeaderMap, salt: &str) -> Option<String> {
2236 if let Some(v) = header_str(headers, SESSION_HEADER) {
2237 let v = v.trim();
2238 if !v.is_empty() {
2239 return Some(cap_session(v).to_owned());
2240 }
2241 }
2242 for name in AGENT_SESSION_HEADERS {
2243 if let Some(v) = headers.get(name).and_then(|v| v.to_str().ok()) {
2244 let v = v.trim();
2245 if !v.is_empty() {
2246 return hashed_session("agent", salt, v);
2247 }
2248 }
2249 }
2250 None
2251}
2252
2253fn body_session_bytes(body: &[u8], salt: &str) -> Option<String> {
2256 body_session(&serde_json::from_slice::<Value>(body).ok()?, salt)
2257}
2258
2259fn body_session(json: &Value, salt: &str) -> Option<String> {
2271 if let Some(u) = json.pointer("/metadata/user_id").and_then(Value::as_str)
2272 && u.contains("session_")
2273 {
2274 return hashed_session("meta", salt, u);
2275 }
2276 let msgs = json.get("messages").and_then(Value::as_array);
2277 let input = json.get("input");
2278 let n_items = msgs
2279 .map(Vec::len)
2280 .or_else(|| input.and_then(Value::as_array).map(Vec::len))
2281 .unwrap_or(0);
2282 let has_tools = json
2283 .get("tools")
2284 .and_then(Value::as_array)
2285 .is_some_and(|t| !t.is_empty());
2286 if n_items <= 1 && !has_tools {
2287 return json
2290 .get("previous_response_id")
2291 .and_then(Value::as_str)
2292 .filter(|p| !p.trim().is_empty())
2293 .and_then(|p| hashed_session("prev", salt, p.trim()));
2294 }
2295 if let Some(k) = json.get("prompt_cache_key").and_then(Value::as_str) {
2296 let k = k.trim();
2297 if !k.is_empty() {
2298 return hashed_session("pck", salt, k);
2299 }
2300 }
2301 let mut tools: Vec<&str> = json
2304 .get("tools")
2305 .and_then(Value::as_array)
2306 .into_iter()
2307 .flatten()
2308 .filter_map(|t| {
2309 t.get("name")
2310 .or_else(|| t.pointer("/function/name"))
2311 .or_else(|| t.get("type"))
2312 .and_then(Value::as_str)
2313 })
2314 .collect();
2315 tools.sort_unstable();
2316 let mut prefix = vec![
2319 json.get("system").cloned().unwrap_or(Value::Null),
2320 json.get("instructions").cloned().unwrap_or(Value::Null),
2321 Value::from(tools),
2322 ];
2323 if let Some(msgs) = msgs {
2324 for m in msgs {
2325 prefix.push(m.clone());
2326 if m.get("role").and_then(Value::as_str) == Some("user") {
2327 break;
2328 }
2329 }
2330 } else if let Some(input) = input {
2331 let (role, text) = match input {
2334 Value::String(s) => ("user", s.clone()),
2335 other => {
2336 let first = other.as_array().and_then(|a| a.first())?;
2337 let role = first.get("role").and_then(Value::as_str).unwrap_or("user");
2338 let text = match first.get("content") {
2339 Some(Value::String(s)) => s.clone(),
2340 Some(Value::Array(parts)) => parts
2341 .iter()
2342 .filter_map(|p| p.get("text").and_then(Value::as_str))
2343 .collect::<Vec<_>>()
2344 .join("\n"),
2345 _ => first.to_string(),
2346 };
2347 (role, text)
2348 }
2349 };
2350 prefix.push(serde_json::json!([role, text]));
2351 }
2352 let bytes = serde_json::to_vec(&prefix).ok()?;
2353 let h = prompt_hash(salt, &bytes);
2354 Some(format!("conv:{}", h.get(..16)?))
2355}
2356
2357fn trajectory_signals(body: &[u8]) -> TrajectorySignals {
2371 let Ok(json) = serde_json::from_slice::<Value>(body) else {
2372 return TrajectorySignals::default();
2373 };
2374 let Some(messages) = json.get("messages").and_then(Value::as_array) else {
2375 return TrajectorySignals::default();
2376 };
2377
2378 const WINDOW: usize = 12;
2383 let recent = &messages[messages.len().saturating_sub(WINDOW)..];
2384
2385 let assistant_turns = u32::try_from(
2394 messages
2395 .iter()
2396 .filter(|m| m.get("role").and_then(Value::as_str) == Some("assistant"))
2397 .count(),
2398 )
2399 .unwrap_or(u32::MAX);
2400 let mut s = TrajectorySignals {
2401 assistant_turns,
2402 ..Default::default()
2403 };
2404 let mut seen: Vec<u64> = Vec::new();
2408
2409 for m in recent {
2410 match m.get("role").and_then(Value::as_str) {
2411 Some("assistant") => {}
2413 Some("tool") => {
2416 s.tool_results += 1;
2417 if openai_tool_content_looks_like_error(m) {
2418 s.tool_errors += 1;
2419 }
2420 }
2421 _ => {}
2422 }
2423
2424 if let Some(blocks) = m.get("content").and_then(Value::as_array) {
2427 for b in blocks {
2428 match b.get("type").and_then(Value::as_str) {
2429 Some("tool_result") => {
2430 s.tool_results += 1;
2431 if b.get("is_error").and_then(Value::as_bool).unwrap_or(false) {
2432 s.tool_errors += 1;
2433 }
2434 }
2435 Some("tool_use") => {
2436 let fp = tool_call_fingerprint(
2437 b.get("name").and_then(Value::as_str).unwrap_or_default(),
2438 b.get("input"),
2439 );
2440 if seen.contains(&fp) {
2441 s.repeated_tool_calls += 1;
2442 } else {
2443 seen.push(fp);
2444 }
2445 }
2446 _ => {}
2447 }
2448 }
2449 }
2450
2451 if let Some(calls) = m.get("tool_calls").and_then(Value::as_array) {
2453 for c in calls {
2454 let f = c.get("function");
2455 let fp = tool_call_fingerprint(
2456 f.and_then(|f| f.get("name"))
2457 .and_then(Value::as_str)
2458 .unwrap_or_default(),
2459 f.and_then(|f| f.get("arguments")),
2460 );
2461 if seen.contains(&fp) {
2462 s.repeated_tool_calls += 1;
2463 } else {
2464 seen.push(fp);
2465 }
2466 }
2467 }
2468 }
2469 s
2470}
2471
2472fn tool_call_fingerprint(name: &str, args: Option<&Value>) -> u64 {
2477 use std::hash::{Hash, Hasher};
2478 let mut h = std::collections::hash_map::DefaultHasher::new();
2479 name.hash(&mut h);
2480 if let Some(a) = args {
2485 a.to_string().hash(&mut h);
2486 }
2487 h.finish()
2488}
2489
2490fn openai_tool_content_looks_like_error(message: &Value) -> bool {
2499 let content = message.get("content");
2503 let text: &str = match content.and_then(Value::as_str) {
2508 Some(t) => t,
2509 None => {
2510 let Some(t) = content.and_then(Value::as_array).and_then(|parts| {
2511 parts
2512 .iter()
2513 .find_map(|p| p.get("text").and_then(Value::as_str))
2514 }) else {
2515 return false;
2516 };
2517 t
2518 }
2519 };
2520 let lowered: String = text
2536 .chars()
2537 .take(128)
2538 .collect::<String>()
2539 .trim_start()
2540 .chars()
2541 .take(64)
2542 .collect::<String>()
2543 .to_ascii_lowercase();
2544 lowered.starts_with("error")
2545 || lowered.starts_with("exception")
2546 || lowered.starts_with("traceback")
2547 || lowered.starts_with("failed")
2548 || lowered.starts_with("fatal")
2549}
2550
2551fn extract_features(headers: &HeaderMap, body: &[u8]) -> Features {
2552 let (_model, tool_count, has_images) = request_features(body);
2553 let mut f = Features::new(TaskKind::Other);
2554 f.agent = header_str(headers, AGENT_HEADER);
2555 f.subagent = header_str(headers, SUBAGENT_HEADER);
2556 f.tool_count = tool_count;
2557 f.has_images = has_images;
2558 f.prompt_token_bucket = token_bucket(body.len() as u64);
2561 f.hour_bucket = hour_bucket(jiff::Timestamp::now());
2562 f.difficulty_hint = DifficultyHint::score(trajectory_signals(body)).as_u8();
2566 f
2567}
2568
2569#[allow(clippy::too_many_arguments)]
2572async fn handle_enforce(
2573 state: &AppState,
2574 headers: &HeaderMap,
2575 body: &Bytes,
2576 features: Features,
2577 route: &Route,
2578 route_ix: usize,
2581 session_header: Option<String>,
2582 tenant: String,
2583 routing_mode: RoutingMode,
2584) -> Response {
2585 if is_stream_request(body) {
2593 let (tx, rx) = tokio::sync::oneshot::channel::<Result<Value, ProxyError>>();
2594 let (state_c, headers_c, body_c, route_c) =
2595 (state.clone(), headers.clone(), body.clone(), route.clone());
2596 tokio::spawn(async move {
2597 let out = enforce_pipeline(
2598 &state_c,
2599 &headers_c,
2600 &body_c,
2601 features,
2602 &route_c,
2603 route_ix,
2604 session_header,
2605 tenant,
2606 routing_mode,
2607 )
2608 .await;
2609 let _ = tx.send(out);
2610 });
2611 return sse_keepalive_response(rx, Box::new(anthropic_sse_from_message), sse_error_event);
2612 }
2613 match enforce_pipeline(
2614 state,
2615 headers,
2616 body,
2617 features,
2618 route,
2619 route_ix,
2620 session_header,
2621 tenant,
2622 routing_mode,
2623 )
2624 .await
2625 {
2626 Ok(message) => (axum::http::StatusCode::OK, Json(message)).into_response(),
2627 Err(e) => e.into_response(),
2628 }
2629}
2630
2631#[allow(clippy::too_many_arguments)] async fn enforce_pipeline_inner(
2636 state: &AppState,
2637 body: &Bytes,
2638 base_request: ModelRequest,
2639 auth: Auth,
2640 features: Features,
2641 route: &Route,
2642 session_header: Option<String>,
2643 tenant: String,
2644 api: &str,
2645 routing_mode: RoutingMode,
2646 route_ix: usize,
2647 allow_challenger: bool,
2651) -> Result<ModelResponse, ProxyError> {
2652 let gate_defs = state
2653 .config
2654 .routing
2655 .as_ref()
2656 .map_or(&[][..], |cfg| &cfg.gate_defs);
2657 let gates = resolve_gates(
2658 &route.gates,
2659 gate_defs,
2660 &state.providers,
2661 &auth,
2662 &state.config.prices,
2663 &state.http,
2664 );
2665 let session_id = session_header.unwrap_or_else(|| Uuid::now_v7().to_string());
2666
2667 let live_arm_info: Option<(firstpass_core::Arm, String)> = route
2690 .challenger
2691 .as_ref()
2692 .filter(|ch| allow_challenger && !ch.is_paired())
2693 .map(|ch| {
2694 let bucket = firstpass_core::Challenger::bucket(&state.config.prompt_salt, &session_id);
2695 (ch.arm_of(bucket), ch.fingerprint(&route.ladder))
2696 });
2697
2698 let live_sync_ok = if let Some((firstpass_core::Arm::Challenger, ref fp)) = live_arm_info {
2699 state.experiments.ensure_synced(&tenant, route_ix, fp).await
2700 } else {
2701 true
2702 };
2703
2704 let experiment = route
2705 .challenger
2706 .as_ref()
2707 .filter(|_| allow_challenger)
2708 .map(|ch| {
2709 let bucket = firstpass_core::Challenger::bucket(&state.config.prompt_salt, &session_id);
2710 let assigned = ch.arm_of(bucket);
2711 let arm = if ch.is_paired()
2712 || (!live_sync_ok && !ch.is_paired() && assigned == firstpass_core::Arm::Challenger)
2713 || (assigned == firstpass_core::Arm::Challenger
2714 && state.experiments.challenger_blocked(&tenant, route_ix))
2715 {
2716 firstpass_core::Arm::Incumbent
2717 } else {
2718 assigned
2719 };
2720 (
2721 ch,
2722 firstpass_core::trace::ExperimentRecord {
2723 arm,
2724 percent: ch.percent,
2725 bucket,
2726 fingerprint: ch.fingerprint(&route.ladder),
2727 },
2728 )
2729 });
2730 let on_challenger = experiment
2731 .as_ref()
2732 .is_some_and(|(_, rec)| rec.arm == firstpass_core::Arm::Challenger);
2733 let ladder: &[String] = match &experiment {
2736 Some((ch, rec)) if rec.arm == firstpass_core::Arm::Challenger => &ch.ladder,
2737 _ => &route.ladder,
2738 };
2739 let (budget, max_rungs, speculation, serve_threshold, elastic) =
2740 match state.config.routing.as_ref() {
2741 Some(cfg) => (
2742 cfg.budget.per_request_usd,
2743 cfg.escalation.max_rungs_per_request,
2744 cfg.escalation.speculation,
2745 cfg.escalation.serve_threshold,
2746 cfg.escalation.elastic.as_ref(),
2747 ),
2748 None => (None, 3, 0, None, None),
2749 };
2750 let certified = state.eprocess.as_ref().and_then(|e| {
2765 e.lock()
2766 .ok()
2767 .and_then(|g| g.certified_threshold().map(|c| (c, g.alpha(), g.delta())))
2768 });
2769 if let Some((c, _, _)) = &certified {
2770 metrics::gauge!("firstpass_eprocess_certified_threshold").set(c.threshold);
2771 metrics::gauge!("firstpass_eprocess_e_value").set(c.e_value);
2772 }
2773 let aci = state
2774 .adaptive
2775 .as_ref()
2776 .and_then(|a| a.lock().ok().map(|g| (g.threshold(), g.alpha())));
2777 let risk = risk_certificate(certified, aci, serve_threshold);
2780 let serve_threshold = risk.as_ref().map(|r| r.threshold);
2781
2782 let preset = routing_mode.preset();
2785 let max_rungs = if let Some(delta) = preset.max_rungs_delta {
2786 (max_rungs as i32 + delta).max(1) as u32
2787 } else {
2788 max_rungs
2789 };
2790 let speculation = preset.speculation.unwrap_or(speculation);
2791
2792 let bandit_ctx = crate::bandit::ContextBucket::from_features(&features);
2796
2797 let decision_prior: Option<Vec<f64>> = match state.prior.as_ref().filter(|_| !on_challenger) {
2803 Some(prior_client) => {
2804 prior_client
2805 .fetch(&crate::prior::query_text(&base_request))
2806 .await
2807 }
2808 None => None,
2809 };
2810
2811 let (greedy_rung, base_policy_id, ts_propensity) = if on_challenger {
2822 (0, "challenger-ladder@v0".to_owned(), None)
2823 } else {
2824 let (chosen, ts_p) = match state.bandit.as_ref() {
2825 Some(bandit) => bandit
2826 .lock()
2827 .ok()
2828 .map(|mut b| {
2829 b.choose_start_with_prior(
2830 &bandit_ctx,
2831 &route.ladder,
2832 &state.config.prices,
2833 decision_prior.as_deref(),
2834 )
2835 })
2836 .unwrap_or((0, None)),
2837 None => match (state.prior.as_ref(), decision_prior.as_deref()) {
2840 (Some(prior_client), Some(prior)) => crate::bandit::StartRungBandit::new(0, 1.0)
2841 .with_prior_strength(prior_client.strength())
2842 .choose_start_with_prior(
2843 &bandit_ctx,
2844 &route.ladder,
2845 &state.config.prices,
2846 Some(prior),
2847 ),
2848 _ => (0, None),
2849 },
2850 };
2851 let policy = if decision_prior.is_some() {
2852 "bandit@v3-prior".to_owned()
2853 } else if ts_p.is_some() {
2854 "bandit@v2-ts".to_owned()
2855 } else if chosen > 0 {
2856 "bandit@v1".to_owned()
2857 } else {
2858 "static-ladder@v0".to_owned()
2859 };
2860 (chosen, policy, ts_p)
2861 };
2862
2863 let exploration_epsilon = state
2868 .config
2869 .routing
2870 .as_ref()
2871 .and_then(|cfg| cfg.escalation.exploration.as_ref())
2872 .map(|e| e.epsilon)
2873 .filter(|_| !on_challenger);
2874
2875 let (start_rung, policy_id, explore_flag, propensity) = if ts_propensity.is_some() {
2876 if exploration_epsilon.is_some() {
2879 tracing::warn!(
2880 "bandit.algorithm = thompson already logs propensities; \
2881 [escalation.exploration] epsilon is ignored"
2882 );
2883 }
2884 (greedy_rung, base_policy_id, false, ts_propensity)
2885 } else if let Some(epsilon) = exploration_epsilon {
2886 let k = ladder.len().max(1);
2887 let u = u01(Uuid::now_v7().as_u128());
2889 let (chosen, eps_branch) = if u < epsilon {
2890 let idx = ((u / epsilon) * k as f64) as u32;
2892 (idx.min(k as u32 - 1), true)
2893 } else {
2894 (greedy_rung, false)
2895 };
2896 let p = epsilon_propensity(chosen, greedy_rung, epsilon, k);
2897 (chosen, format!("{base_policy_id}+eps"), eps_branch, Some(p))
2898 } else {
2899 (greedy_rung, base_policy_id, false, None)
2900 };
2901
2902 let promotion = state.promoter.as_ref().filter(|_| !on_challenger).cloned();
2907 let promotion = match promotion {
2910 Some(p) => {
2911 p.decide_async(&tenant, &session_id, std::time::Instant::now())
2912 .await
2913 }
2914 None => crate::affinity::Decision::Cold,
2915 };
2916 let start_rung = start_rung.max(promotion.start_rung());
2917 let promo_key = state
2920 .promoter
2921 .as_ref()
2922 .filter(|_| !on_challenger)
2923 .map(|_| (tenant.clone(), session_id.clone()));
2924
2925 let start_rung = if preset.start_at_top {
2928 ladder.len().saturating_sub(1) as u32
2929 } else {
2930 start_rung
2931 };
2932
2933 let speculation = match state
2939 .config
2940 .routing
2941 .as_ref()
2942 .and_then(|cfg| cfg.escalation.speculation_band)
2943 {
2944 Some([lo, hi]) if speculation > 0 && !on_challenger => {
2945 let estimate = state
2946 .bandit
2947 .as_ref()
2948 .and_then(|b| b.lock().ok())
2949 .and_then(|b| b.pass_estimate(&bandit_ctx, start_rung));
2950 match estimate {
2951 Some(p) if p < lo || p > hi => {
2952 metrics::counter!("firstpass_speculation_skipped_total").increment(1);
2953 0
2954 }
2955 _ => speculation,
2956 }
2957 }
2958 _ => speculation,
2959 };
2960
2961 if state.bandit.is_some() && !on_challenger {
2963 metrics::counter!(
2964 "firstpass_bandit_start_rung",
2965 "rung" => start_rung.to_string()
2966 )
2967 .increment(1);
2968 }
2969
2970 let mut base_request = base_request;
2971 base_request.cache_prefix = state
2975 .config
2976 .routing
2977 .as_ref()
2978 .is_some_and(|r| r.escalation.prompt_cache);
2979 let paired_spawn_data: Option<(Features, String, String)> = experiment
2982 .as_ref()
2983 .filter(|(ch, _)| ch.is_paired())
2984 .map(|_| (features.clone(), tenant.clone(), session_id.clone()));
2985
2986 let ctx = EnforceCtx {
2987 condense: routing_cfg_condense(state),
2988 ladder,
2989 gates: &gates,
2990 health: &state.gate_health,
2991 base_request: &base_request,
2992 providers: &state.providers,
2993 auth: &auth,
2994 prices: &state.config.prices,
2995 budget_per_request_usd: budget,
2996 max_rungs,
2997 speculation,
2998 serve_threshold,
2999 elastic,
3000 features,
3001 start_rung,
3002 tenant_id: tenant,
3005 session_id,
3006 prompt_hash: prompt_hash(&state.config.prompt_salt, body),
3007 api: api.to_owned(),
3008 policy_id,
3009 };
3010
3011 if let Some(cache) = state.verified_cache.as_ref()
3016 && let Some(entry) = cache
3017 .get(
3018 &crate::verified_cache::VerifiedCache::compose_key(
3019 &ctx.tenant_id,
3020 &ctx.prompt_hash,
3021 ladder,
3024 ),
3025 std::time::Instant::now(),
3026 )
3027 .await
3028 {
3029 metrics::counter!("firstpass_cache_hits_total").increment(1);
3030 let mut trace = cache_hit_trace(&ctx, &entry, mode_now());
3031 if let Some((_, rec)) = &experiment {
3032 trace.route_ix = Some(u32::try_from(route_ix).unwrap_or(0));
3037 trace.experiment = Some(rec.clone());
3038 state.experiments.observe_cost(
3039 &trace.tenant_id,
3040 route_ix,
3041 &rec.fingerprint,
3042 rec.arm,
3043 trace.final_.total_cost_usd,
3044 );
3045 }
3046 offer_trace(&state.traces, state.spill.as_ref(), trace);
3047 return Ok(entry.response);
3048 }
3049
3050 let (outcome, mut trace) = route_enforce(ctx).await;
3051
3052 trace.route_ix = Some(u32::try_from(route_ix).unwrap_or(0));
3056 trace.policy.explore = explore_flag;
3059 trace.policy.propensity = propensity;
3060 if routing_mode != RoutingMode::Balanced {
3063 trace.policy.mode_profile = Some(routing_mode.as_str().to_owned());
3064 }
3065 trace.decision_prior = decision_prior;
3068 trace.risk = risk;
3069 if let Some((_, rec)) = &experiment {
3072 state.experiments.observe_cost(
3073 &trace.tenant_id,
3074 route_ix,
3075 &rec.fingerprint,
3076 rec.arm,
3077 trace.final_.total_cost_usd,
3078 );
3079 state.experiments.observe_health(
3080 &trace.tenant_id,
3081 route_ix,
3082 &rec.fingerprint,
3083 rec.arm,
3084 trace.final_.served_from == firstpass_core::ServedFrom::Error,
3085 crate::experiment::truncated_attempts(&trace),
3086 );
3087 trace.experiment = Some(rec.clone());
3088 }
3089
3090 if let Some(bandit) = state.bandit.as_ref().filter(|_| !on_challenger)
3094 && let Ok(mut b) = bandit.lock()
3095 {
3096 for attempt in &trace.attempts {
3097 b.observe(&bandit_ctx, attempt.rung, attempt.verdict);
3098 }
3099 }
3100
3101 if let Some(promoter) = state.promoter.as_ref()
3106 && let Some((tenant_key, session_key)) = promo_key.as_ref()
3107 && let Some(served) = trace.final_.served_rung
3108 {
3109 promoter
3110 .record_async(
3111 tenant_key,
3112 session_key,
3113 served,
3114 served > start_rung,
3115 std::time::Instant::now(),
3116 )
3117 .await;
3118 }
3119
3120 if let Some(predictor) = state.predictor.as_ref().filter(|_| !on_challenger)
3126 && let Ok(mut p) = predictor.lock()
3127 {
3128 let predicted = p.predict(&trace.request.features, start_rung);
3130 for attempt in &trace.attempts {
3131 match attempt.verdict {
3132 Verdict::Pass => p.update(&trace.request.features, attempt.rung, true),
3133 Verdict::Fail => p.update(&trace.request.features, attempt.rung, false),
3134 Verdict::Abstain => {} }
3136 }
3137 trace.predicted_pass = Some(predicted);
3138 metrics::histogram!("firstpass_predictor_pass_prob").record(predicted);
3139 }
3140
3141 if let Some(probe_cfg) = state
3148 .config
3149 .routing
3150 .as_ref()
3151 .and_then(|c| c.escalation.probe)
3152 && u01(Uuid::now_v7().as_u128()) < probe_cfg.sample_rate
3153 {
3154 let probe_rung = (start_rung as usize).min(ladder.len().saturating_sub(1));
3156 if let Some(probe_model_str) = ladder.get(probe_rung).cloned() {
3157 let probe_provider = ModelRef::parse(&probe_model_str)
3158 .ok()
3159 .and_then(|m| state.providers.get(&m.provider));
3160
3161 if let Some(probe_provider) = probe_provider {
3162 let mut join_set = tokio::task::JoinSet::new();
3165 for _ in 0..probe_cfg.k {
3166 let mut probe_req = base_request.clone();
3167 probe_req.model = probe_model_str.clone();
3168 let probe_auth = auth.clone();
3169 let prov = probe_provider.clone();
3170 join_set.spawn(async move { prov.complete(&probe_req, &probe_auth).await });
3171 }
3172
3173 let fail_closed_owned: std::collections::HashSet<String> = gates
3176 .iter()
3177 .filter(|g| g.abstain_fails_closed())
3178 .map(|g| g.id().to_owned())
3179 .collect();
3180
3181 let mut gate_pass_count = 0u32;
3182 let mut probe_cost_usd = 0.0f64;
3183
3184 while let Some(task_result) = join_set.join_next().await {
3185 let Ok(Ok(probe_resp)) = task_result else {
3186 continue; };
3188 probe_cost_usd += state
3191 .config
3192 .prices
3193 .cost_usd_with_cache(
3194 &probe_model_str,
3195 probe_resp.in_tokens,
3196 probe_resp.cache_write_tokens,
3197 probe_resp.cache_read_tokens,
3198 probe_resp.out_tokens,
3199 )
3200 .unwrap_or(0.0);
3201
3202 let mut probe_gate_req = base_request.clone();
3203 probe_gate_req.model = probe_model_str.clone();
3204
3205 let mut probe_gate_results = Vec::with_capacity(gates.len());
3208 for g in &gates {
3209 if !state.gate_health.enabled(&trace.tenant_id, g.id()) {
3211 continue;
3212 }
3213 let r = g.evaluate(&probe_gate_req, &probe_resp).await;
3214 probe_gate_results.push(r);
3216 }
3217
3218 let fail_closed_refs: std::collections::HashSet<&str> =
3219 fail_closed_owned.iter().map(|s| s.as_str()).collect();
3220 let verdict = aggregate_with_policy(&probe_gate_results, &fail_closed_refs);
3221 let passes = match serve_threshold {
3224 None => verdict == Verdict::Pass,
3225 Some(t) => crate::calibrate::gate_score(&probe_gate_results, verdict) >= t,
3226 };
3227 if passes {
3228 gate_pass_count += 1;
3229 }
3230 }
3231
3232 let regime = ProbeRegime::classify(gate_pass_count, probe_cfg.k);
3233 let regime_label = match regime {
3234 ProbeRegime::ConfidentPass => "confident_pass",
3235 ProbeRegime::ConfidentFail => "confident_fail",
3236 ProbeRegime::Ambiguous => "ambiguous",
3237 };
3238 metrics::counter!(
3239 "firstpass_probe_regime_total",
3240 "regime" => regime_label
3241 )
3242 .increment(1);
3243 metrics::gauge!("firstpass_probe_cost_usd_total").increment(probe_cost_usd);
3244 trace.probe = Some(ProbeSignal {
3245 k: probe_cfg.k,
3246 gate_pass_count,
3247 regime,
3248 probe_cost_usd,
3249 });
3250 }
3251 }
3252 }
3253 if let Some(cache) = state.verified_cache.as_ref()
3266 && let EngineOutcome::Served(resp) = &outcome
3267 && crate::verified_cache::is_cacheable(&trace)
3268 {
3269 let now = std::time::Instant::now();
3270 cache
3271 .put(
3272 &crate::verified_cache::VerifiedCache::compose_key(
3273 &trace.tenant_id,
3274 &trace.request.prompt_hash,
3275 ladder,
3276 ),
3277 trace.trace_id,
3278 crate::verified_cache::Entry::new(resp.clone(), &trace, now),
3279 now,
3280 )
3281 .await;
3282 metrics::counter!("firstpass_cache_stores_total").increment(1);
3283 }
3284
3285 if let Some((ch, rec)) = &experiment
3289 && ch.is_paired()
3290 {
3291 let trace_id_str = trace.trace_id.to_string();
3292 if ch.paired_sampled(&state.config.prompt_salt, &trace_id_str)
3293 && let Some((spawn_feats, spawn_tenant, spawn_session)) = paired_spawn_data
3294 {
3295 let served_from = trace.final_.served_from;
3296 let incumbent_cost = trace.final_.total_cost_usd;
3297 spawn_paired_challenger(
3298 state,
3299 route,
3300 ch,
3301 rec.fingerprint.clone(),
3302 route_ix,
3303 body.clone(),
3304 auth.clone(),
3305 spawn_feats,
3306 spawn_tenant,
3307 spawn_session,
3308 api,
3309 served_from,
3310 incumbent_cost,
3311 trace.trace_id,
3312 );
3313 }
3314 }
3315
3316 offer_trace(&state.traces, state.spill.as_ref(), trace);
3318
3319 match outcome {
3320 EngineOutcome::Served(resp) => Ok(resp),
3321 EngineOutcome::Failed(msg) => Err(ProxyError::Engine(msg)),
3322 }
3323}
3324
3325#[allow(clippy::too_many_arguments)]
3328async fn enforce_pipeline(
3329 state: &AppState,
3330 headers: &HeaderMap,
3331 body: &Bytes,
3332 features: Features,
3333 route: &Route,
3334 route_ix: usize,
3336 session_header: Option<String>,
3337 tenant: String,
3338 routing_mode: RoutingMode,
3339) -> Result<Value, ProxyError> {
3340 let Some(base_request) = parse_model_request(body) else {
3341 return Err(ProxyError::BadRequest(
3342 "request body is not a valid Anthropic Messages request".to_owned(),
3343 ));
3344 };
3345 let auth = Auth::from_headers(headers, firstpass_core::Dialect::Anthropic);
3346 let resp = enforce_pipeline_inner(
3347 state,
3348 body,
3349 base_request,
3350 auth,
3351 features,
3352 route,
3353 session_header,
3354 tenant,
3355 "anthropic.messages",
3356 routing_mode,
3357 route_ix,
3358 true,
3359 )
3360 .await?;
3361 Ok(anthropic_response_json(&resp))
3362}
3363
3364#[allow(clippy::too_many_arguments)]
3367async fn enforce_pipeline_openai(
3368 state: &AppState,
3369 headers: &HeaderMap,
3370 body: &Bytes,
3371 features: Features,
3372 route: &Route,
3373 route_ix: usize,
3375 session_header: Option<String>,
3376 tenant: String,
3377 routing_mode: RoutingMode,
3378) -> Result<Value, ProxyError> {
3379 enforce_pipeline_openai_as(
3380 state,
3381 headers,
3382 body,
3383 body,
3384 "openai.chat_completions",
3385 features,
3386 route,
3387 route_ix,
3388 session_header,
3389 tenant,
3390 routing_mode,
3391 )
3392 .await
3393}
3394
3395#[allow(clippy::too_many_arguments)]
3404async fn enforce_pipeline_openai_as(
3405 state: &AppState,
3406 headers: &HeaderMap,
3407 body: &Bytes,
3408 receipt_body: &Bytes,
3409 api: &str,
3410 features: Features,
3411 route: &Route,
3412 route_ix: usize,
3413 session_header: Option<String>,
3414 tenant: String,
3415 routing_mode: RoutingMode,
3416) -> Result<Value, ProxyError> {
3417 let providers = &state.providers;
3419 let all_openai_of = |ladder: &[String]| {
3420 ladder.iter().all(|rung| {
3421 let pid = rung.split('/').next().unwrap_or_default();
3422 providers
3423 .get(pid)
3424 .is_some_and(|p| p.carries_structured_verbatim(Dialect::Openai))
3425 })
3426 };
3427 let all_openai = all_openai_of(&route.ladder);
3428 let allow_challenger = route
3433 .challenger
3434 .as_ref()
3435 .is_none_or(|c| all_openai_of(&c.ladder) == all_openai);
3436 let Some(base_request) = parse_openai_request(body, all_openai) else {
3437 return Err(ProxyError::BadRequest(
3438 "request body is not a valid OpenAI Chat Completions request".to_owned(),
3439 ));
3440 };
3441 let auth = Auth::from_headers(headers, firstpass_core::Dialect::Openai);
3442 let resp = enforce_pipeline_inner(
3443 state,
3444 receipt_body,
3445 base_request,
3446 auth,
3447 features,
3448 route,
3449 session_header,
3450 tenant,
3451 api,
3452 routing_mode,
3453 route_ix,
3454 allow_challenger,
3455 )
3456 .await?;
3457 Ok(openai_response_json(&resp))
3458}
3459
3460const SSE_KEEPALIVE_EVERY: Duration = Duration::from_secs(5);
3462
3463fn sse_keepalive_response(
3473 rx: tokio::sync::oneshot::Receiver<Result<Value, ProxyError>>,
3474 format_message: Box<dyn Fn(&Value) -> String + Send>,
3475 format_error: fn(&ProxyError) -> String,
3476) -> Response {
3477 let mut ticks = tokio::time::interval(SSE_KEEPALIVE_EVERY);
3478 ticks.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
3479 ticks.reset(); let stream = KeepaliveStream {
3481 rx: Some(rx),
3482 ticks,
3483 format_message,
3484 format_error,
3485 };
3486 (
3487 axum::http::StatusCode::OK,
3488 [(
3489 axum::http::header::CONTENT_TYPE,
3490 "text/event-stream; charset=utf-8",
3491 )],
3492 axum::body::Body::from_stream(stream),
3493 )
3494 .into_response()
3495}
3496
3497struct KeepaliveStream {
3500 rx: Option<tokio::sync::oneshot::Receiver<Result<Value, ProxyError>>>,
3502 ticks: tokio::time::Interval,
3503 format_message: Box<dyn Fn(&Value) -> String + Send>,
3505 format_error: fn(&ProxyError) -> String,
3507}
3508
3509impl futures_core::Stream for KeepaliveStream {
3510 type Item = Result<Bytes, std::convert::Infallible>;
3511
3512 fn poll_next(
3513 mut self: std::pin::Pin<&mut Self>,
3514 cx: &mut std::task::Context<'_>,
3515 ) -> std::task::Poll<Option<Self::Item>> {
3516 use std::task::Poll;
3517 let Some(rx) = self.rx.as_mut() else {
3518 return Poll::Ready(None); };
3520 if let Poll::Ready(out) = std::pin::Pin::new(rx).poll(cx) {
3521 let frame = match out {
3522 Ok(Ok(message)) => (self.format_message)(&message),
3523 Ok(Err(e)) => (self.format_error)(&e),
3524 Err(_) => (self.format_error)(&ProxyError::Internal(
3525 "enforce pipeline task dropped".to_owned(),
3526 )),
3527 };
3528 self.rx = None;
3529 return Poll::Ready(Some(Ok(Bytes::from(frame))));
3530 }
3531 if self.ticks.poll_tick(cx).is_ready() {
3532 return Poll::Ready(Some(Ok(Bytes::from_static(b": firstpass routing\n\n"))));
3533 }
3534 Poll::Pending
3535 }
3536}
3537
3538fn sse_error_event(e: &ProxyError) -> String {
3541 let mut out = String::new();
3542 sse_event(
3543 &mut out,
3544 "error",
3545 &serde_json::json!({
3546 "type": "error",
3547 "error": { "type": "api_error", "message": e.client_message() }
3548 }),
3549 );
3550 out
3551}
3552
3553fn parse_model_request(body: &[u8]) -> Option<ModelRequest> {
3562 let json: Value = serde_json::from_slice(body).ok()?;
3563 let raw = json.clone();
3564 let messages_json = json.get("messages")?.as_array()?;
3565 let messages = messages_json
3566 .iter()
3567 .map(|m| ChatMessage {
3568 role: m
3569 .get("role")
3570 .and_then(Value::as_str)
3571 .unwrap_or("user")
3572 .to_owned(),
3573 content: m
3574 .get("content")
3575 .cloned()
3576 .unwrap_or_else(|| Value::String(String::new())),
3577 })
3578 .collect();
3579 let system = json
3580 .get("system")
3581 .and_then(Value::as_str)
3582 .map(str::to_owned);
3583 let max_tokens = json
3584 .get("max_tokens")
3585 .and_then(Value::as_u64)
3586 .and_then(|n| u32::try_from(n).ok())
3587 .unwrap_or(1024);
3588 let tools = json.get("tools").cloned().unwrap_or(Value::Null);
3589 Some(ModelRequest {
3590 model: json
3591 .get("model")
3592 .and_then(Value::as_str)
3593 .unwrap_or_default()
3594 .to_owned(),
3595 system,
3596 messages,
3597 max_tokens,
3598 tools,
3599 raw,
3600 cache_prefix: false,
3601 })
3602}
3603
3604fn openai_finish_reason_to_anthropic(finish_reason: &str) -> &str {
3614 match finish_reason {
3615 "tool_calls" => "tool_use",
3616 "length" => "max_tokens",
3617 "stop" => "end_turn",
3618 "content_filter" => "end_turn",
3619 other => other,
3620 }
3621}
3622
3623fn openai_message_to_anthropic_content(msg: &Value) -> Value {
3628 let mut blocks: Vec<Value> = Vec::new();
3629
3630 if let Some(text) = msg
3631 .get("content")
3632 .and_then(Value::as_str)
3633 .filter(|s| !s.is_empty())
3634 {
3635 blocks.push(serde_json::json!({ "type": "text", "text": text }));
3636 }
3637
3638 if let Some(tool_calls) = msg.get("tool_calls").and_then(Value::as_array) {
3639 for tc in tool_calls {
3640 let id = tc
3642 .get("id")
3643 .and_then(Value::as_str)
3644 .filter(|s| !s.is_empty())
3645 .map_or_else(
3646 || format!("toolu_{}", Uuid::now_v7().simple()),
3647 str::to_owned,
3648 );
3649 let name = tc
3650 .pointer("/function/name")
3651 .and_then(Value::as_str)
3652 .unwrap_or("");
3653 let input = tc
3654 .pointer("/function/arguments")
3655 .and_then(Value::as_str)
3656 .and_then(|s| serde_json::from_str::<Value>(s).ok())
3657 .filter(Value::is_object)
3658 .unwrap_or_else(|| serde_json::json!({}));
3659 if !name.is_empty() {
3660 blocks.push(serde_json::json!({
3661 "type": "tool_use",
3662 "id": id,
3663 "name": name,
3664 "input": input,
3665 }));
3666 }
3667 }
3668 }
3669
3670 if blocks.is_empty() {
3671 serde_json::json!([{ "type": "text", "text": "" }])
3672 } else {
3673 Value::Array(blocks)
3674 }
3675}
3676
3677fn gemini_finish_reason_to_anthropic(reason: &str) -> &str {
3679 match reason {
3680 "MAX_TOKENS" => "max_tokens",
3681 "STOP" => "end_turn",
3682 other => other,
3683 }
3684}
3685
3686fn anthropic_response_json(resp: &ModelResponse) -> Value {
3698 let content = if let Some(c) = resp.raw.get("content").filter(|c| c.is_array()) {
3700 c.clone()
3701 } else if let Some(msg) = resp.raw.pointer("/choices/0/message") {
3702 openai_message_to_anthropic_content(msg)
3703 } else {
3704 serde_json::json!([{ "type": "text", "text": resp.text }])
3706 };
3707
3708 let has_tool_use = content.as_array().is_some_and(|a| {
3709 a.iter()
3710 .any(|b| b.get("type") == Some(&Value::String("tool_use".to_owned())))
3711 });
3712
3713 let stop_reason_str = resp
3717 .raw
3718 .get("stop_reason")
3719 .and_then(Value::as_str)
3720 .or_else(|| {
3721 resp.raw
3722 .pointer("/choices/0/finish_reason")
3723 .and_then(Value::as_str)
3724 .map(openai_finish_reason_to_anthropic)
3725 })
3726 .or_else(|| {
3727 resp.raw
3728 .pointer("/candidates/0/finishReason")
3729 .and_then(Value::as_str)
3730 .map(gemini_finish_reason_to_anthropic)
3731 })
3732 .unwrap_or("end_turn");
3733 let stop_reason_str = if stop_reason_str == "tool_use" && !has_tool_use {
3734 "end_turn"
3735 } else {
3736 stop_reason_str
3737 };
3738 let stop_reason = Value::String(stop_reason_str.to_owned());
3739 let stop_sequence = resp
3740 .raw
3741 .get("stop_sequence")
3742 .cloned()
3743 .unwrap_or(Value::Null);
3744
3745 let mut usage = serde_json::json!({
3747 "input_tokens": resp.in_tokens,
3748 "output_tokens": resp.out_tokens,
3749 });
3750 if resp.cache_write_tokens > 0 {
3751 usage["cache_creation_input_tokens"] = Value::from(resp.cache_write_tokens);
3752 }
3753 if resp.cache_read_tokens > 0 {
3754 usage["cache_read_input_tokens"] = Value::from(resp.cache_read_tokens);
3755 }
3756
3757 serde_json::json!({
3758 "id": format!("msg_{}", Uuid::now_v7()),
3759 "type": "message",
3760 "role": "assistant",
3761 "model": resp.model,
3762 "content": content,
3763 "stop_reason": stop_reason,
3764 "stop_sequence": stop_sequence,
3765 "usage": usage,
3766 })
3767}
3768
3769fn sse_event(out: &mut String, event: &str, data: &Value) {
3771 out.push_str("event: ");
3772 out.push_str(event);
3773 out.push_str("\ndata: ");
3774 out.push_str(&data.to_string());
3775 out.push_str("\n\n");
3776}
3777
3778fn anthropic_sse_from_message(message: &Value) -> String {
3785 let mut out = String::new();
3786
3787 let mut start_msg = message.clone();
3794 start_msg["content"] = Value::Array(Vec::new());
3795 start_msg["stop_reason"] = Value::Null;
3796 start_msg["stop_sequence"] = Value::Null;
3797 start_msg["usage"]["output_tokens"] = Value::from(0u64);
3798 sse_event(
3799 &mut out,
3800 "message_start",
3801 &serde_json::json!({ "type": "message_start", "message": start_msg }),
3802 );
3803
3804 let empty = Vec::new();
3805 let blocks = message
3806 .get("content")
3807 .and_then(Value::as_array)
3808 .unwrap_or(&empty);
3809 for (i, block) in blocks.iter().enumerate() {
3810 match block.get("type").and_then(Value::as_str) {
3811 Some("tool_use") => {
3812 let mut shell = block.clone();
3814 shell["input"] = serde_json::json!({});
3815 sse_event(
3816 &mut out,
3817 "content_block_start",
3818 &serde_json::json!({ "type": "content_block_start", "index": i, "content_block": shell }),
3819 );
3820 let input_json = block
3821 .get("input")
3822 .map_or_else(|| "{}".to_owned(), std::string::ToString::to_string);
3823 sse_event(
3824 &mut out,
3825 "content_block_delta",
3826 &serde_json::json!({ "type": "content_block_delta", "index": i,
3827 "delta": { "type": "input_json_delta", "partial_json": input_json } }),
3828 );
3829 }
3830 Some("thinking") => {
3831 let thinking = block.get("thinking").and_then(Value::as_str).unwrap_or("");
3833 let signature = block.get("signature").and_then(Value::as_str).unwrap_or("");
3834 sse_event(
3835 &mut out,
3836 "content_block_start",
3837 &serde_json::json!({ "type": "content_block_start", "index": i,
3838 "content_block": { "type": "thinking", "thinking": "" } }),
3839 );
3840 sse_event(
3841 &mut out,
3842 "content_block_delta",
3843 &serde_json::json!({ "type": "content_block_delta", "index": i,
3844 "delta": { "type": "thinking_delta", "thinking": thinking } }),
3845 );
3846 if !signature.is_empty() {
3847 sse_event(
3848 &mut out,
3849 "content_block_delta",
3850 &serde_json::json!({ "type": "content_block_delta", "index": i,
3851 "delta": { "type": "signature_delta", "signature": signature } }),
3852 );
3853 }
3854 }
3855 Some("redacted_thinking") => {
3856 sse_event(
3860 &mut out,
3861 "content_block_start",
3862 &serde_json::json!({ "type": "content_block_start", "index": i,
3863 "content_block": block }),
3864 );
3865 }
3867 _ => {
3868 let text = block.get("text").and_then(Value::as_str).unwrap_or("");
3870 sse_event(
3871 &mut out,
3872 "content_block_start",
3873 &serde_json::json!({ "type": "content_block_start", "index": i,
3874 "content_block": { "type": "text", "text": "" } }),
3875 );
3876 sse_event(
3877 &mut out,
3878 "content_block_delta",
3879 &serde_json::json!({ "type": "content_block_delta", "index": i,
3880 "delta": { "type": "text_delta", "text": text } }),
3881 );
3882 }
3883 }
3884 sse_event(
3885 &mut out,
3886 "content_block_stop",
3887 &serde_json::json!({ "type": "content_block_stop", "index": i }),
3888 );
3889 }
3890
3891 let out_tokens = message
3892 .pointer("/usage/output_tokens")
3893 .cloned()
3894 .unwrap_or_else(|| Value::from(0));
3895 let stop_reason = message
3897 .get("stop_reason")
3898 .cloned()
3899 .unwrap_or_else(|| Value::String("end_turn".to_owned()));
3900 let mut message_delta_delta =
3901 serde_json::json!({ "stop_reason": stop_reason, "stop_sequence": Value::Null });
3902 if let Some(seq) = message.get("stop_sequence").filter(|v| !v.is_null()) {
3904 message_delta_delta["stop_sequence"] = seq.clone();
3905 }
3906 sse_event(
3907 &mut out,
3908 "message_delta",
3909 &serde_json::json!({ "type": "message_delta", "delta": message_delta_delta,
3910 "usage": { "output_tokens": out_tokens } }),
3911 );
3912 sse_event(
3913 &mut out,
3914 "message_stop",
3915 &serde_json::json!({ "type": "message_stop" }),
3916 );
3917 out
3918}
3919
3920async fn observe_passthrough(
3922 state: AppState,
3923 headers: HeaderMap,
3924 body: Bytes,
3925 session_header: Option<String>,
3926 tenant: String,
3927) -> Response {
3928 if is_stream_request(&body) {
3930 return observe_stream(state, headers, body, session_header, tenant).await;
3931 }
3932 let start = Instant::now();
3933 let result = forward_anthropic(
3934 &state.http,
3935 &state.config.upstream_anthropic,
3936 &headers,
3937 body.clone(),
3938 )
3939 .await;
3940 let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
3941
3942 match result {
3943 Ok((status, resp_headers, resp_body)) => {
3944 let for_trace = status.is_success().then_some(resp_body.clone());
3949 spawn_trace(
3950 &state,
3951 body,
3952 for_trace,
3953 latency_ms,
3954 session_header,
3955 tenant,
3956 "anthropic",
3957 "anthropic.messages",
3958 None,
3959 );
3960 (status, resp_headers, resp_body).into_response()
3961 }
3962 Err(err) => {
3963 spawn_trace(
3964 &state,
3965 body,
3966 None,
3967 latency_ms,
3968 session_header,
3969 tenant,
3970 "anthropic",
3971 "anthropic.messages",
3972 None,
3973 );
3974 err.into_response()
3975 }
3976 }
3977}
3978
3979async fn observe_stream(
3985 state: AppState,
3986 headers: HeaderMap,
3987 body: Bytes,
3988 session_header: Option<String>,
3989 tenant: String,
3990) -> Response {
3991 let start = Instant::now();
3992 let result = forward_anthropic_streaming(
3993 &state.http,
3994 &state.config.upstream_anthropic,
3995 &headers,
3996 body.clone(),
3997 )
3998 .await;
3999 let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
4000
4001 match result {
4002 Ok((status, resp_headers, response)) => {
4003 let on_done = stream_trace_callback(
4004 &state,
4005 body,
4006 latency_ms,
4007 session_header,
4008 tenant,
4009 "anthropic",
4010 status.is_success(),
4011 "anthropic.messages",
4012 None,
4013 );
4014 let tap = SseTap::new(response.bytes_stream(), on_done);
4015 let stream_body = Body::from_stream(tap);
4016 (status, resp_headers, stream_body).into_response()
4017 }
4018 Err(err) => {
4019 spawn_trace(
4020 &state,
4021 body,
4022 None,
4023 latency_ms,
4024 session_header,
4025 tenant,
4026 "anthropic",
4027 "anthropic.messages",
4028 None,
4029 );
4030 err.into_response()
4031 }
4032 }
4033}
4034
4035#[allow(clippy::too_many_arguments)]
4042fn stream_trace_callback(
4043 state: &AppState,
4044 req_body: Bytes,
4045 latency_ms: u64,
4046 session_header: Option<String>,
4047 tenant: String,
4048 provider: &'static str,
4049 upstream_success: bool,
4050 api: &'static str,
4051 model_hint: Option<String>,
4052) -> impl FnOnce(SseUsage, SseEndReason) + Send + 'static {
4053 let config = state.config.clone();
4054 let traces = state.traces.clone();
4055 let spill = state.spill.clone();
4056 move |usage: SseUsage, reason: SseEndReason| {
4057 let mut trace = if !upstream_success {
4058 build_error_trace(
4061 &config,
4062 &req_body,
4063 latency_ms,
4064 session_header.as_deref(),
4065 provider,
4066 api,
4067 )
4068 } else {
4069 match reason {
4070 SseEndReason::UpstreamError => {
4071 build_stream_trace_from_usage(
4077 &config,
4078 &req_body,
4079 latency_ms,
4080 session_header.as_deref(),
4081 usage,
4082 provider,
4083 StreamOutcome::MidStreamError,
4084 api,
4085 model_hint.as_deref(),
4086 )
4087 }
4088 SseEndReason::Dropped => {
4089 let outcome = if usage.completed {
4094 StreamOutcome::Served
4095 } else {
4096 StreamOutcome::Partial
4097 };
4098 build_stream_trace_from_usage(
4099 &config,
4100 &req_body,
4101 latency_ms,
4102 session_header.as_deref(),
4103 usage,
4104 provider,
4105 outcome,
4106 api,
4107 model_hint.as_deref(),
4108 )
4109 }
4110 SseEndReason::Eof => {
4111 let saw_events =
4116 usage.model.is_some() || usage.in_tokens > 0 || usage.out_tokens > 0;
4117 let outcome = if usage.completed || !saw_events {
4118 StreamOutcome::Served
4119 } else {
4120 StreamOutcome::Partial
4121 };
4122 build_stream_trace_from_usage(
4123 &config,
4124 &req_body,
4125 latency_ms,
4126 session_header.as_deref(),
4127 usage,
4128 provider,
4129 outcome,
4130 api,
4131 model_hint.as_deref(),
4132 )
4133 }
4134 }
4135 };
4136 trace.tenant_id = tenant;
4139 offer_trace(&traces, spill.as_ref(), trace);
4140 }
4141}
4142
4143const SSE_LINE_BUF_CAP: usize = 64 * 1024; #[derive(Debug, Clone, Copy, PartialEq, Eq)]
4151enum SseEndReason {
4152 Eof,
4154 UpstreamError,
4156 Dropped,
4158}
4159
4160#[derive(Debug, Default)]
4164struct SseUsage {
4165 model: Option<String>,
4166 in_tokens: u64,
4167 cache_write_tokens: u64,
4168 cache_read_tokens: u64,
4169 out_tokens: u64,
4170 stop_reason: Option<String>,
4173 completed: bool,
4176}
4177
4178struct SseTap {
4191 inner: std::pin::Pin<Box<dyn Stream<Item = Result<Bytes, reqwest::Error>> + Send>>,
4194 line_buf: Vec<u8>,
4195 overflow: bool,
4197 usage: SseUsage,
4198 on_done: Option<Box<dyn FnOnce(SseUsage, SseEndReason) + Send>>,
4199}
4200
4201impl SseTap {
4202 fn new(
4203 inner: impl Stream<Item = Result<Bytes, reqwest::Error>> + Send + 'static,
4204 on_done: impl FnOnce(SseUsage, SseEndReason) + Send + 'static,
4205 ) -> Self {
4206 Self {
4207 inner: Box::pin(inner),
4208 line_buf: Vec::new(),
4209 overflow: false,
4210 usage: SseUsage::default(),
4211 on_done: Some(Box::new(on_done)),
4212 }
4213 }
4214}
4215
4216impl Stream for SseTap {
4217 type Item = Result<Bytes, reqwest::Error>;
4218
4219 fn poll_next(self: std::pin::Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
4220 let this = self.get_mut();
4222 match this.inner.as_mut().poll_next(cx) {
4223 Poll::Pending => Poll::Pending,
4224 Poll::Ready(Some(Ok(chunk))) => {
4225 parse_sse_chunk_into(
4227 &chunk,
4228 &mut this.line_buf,
4229 &mut this.overflow,
4230 &mut this.usage,
4231 );
4232 Poll::Ready(Some(Ok(chunk)))
4233 }
4234 Poll::Ready(Some(Err(e))) => {
4235 if let Some(on_done) = this.on_done.take() {
4236 on_done(std::mem::take(&mut this.usage), SseEndReason::UpstreamError);
4237 }
4238 Poll::Ready(Some(Err(e)))
4239 }
4240 Poll::Ready(None) => {
4241 if let Some(on_done) = this.on_done.take() {
4242 on_done(std::mem::take(&mut this.usage), SseEndReason::Eof);
4243 }
4244 Poll::Ready(None)
4245 }
4246 }
4247 }
4248}
4249
4250impl Drop for SseTap {
4251 fn drop(&mut self) {
4252 if let Some(on_done) = self.on_done.take() {
4255 on_done(std::mem::take(&mut self.usage), SseEndReason::Dropped);
4256 }
4257 }
4258}
4259
4260fn parse_sse_chunk_into(
4263 chunk: &[u8],
4264 line_buf: &mut Vec<u8>,
4265 overflow: &mut bool,
4266 usage: &mut SseUsage,
4267) {
4268 for &byte in chunk {
4269 if byte == b'\n' {
4270 if !*overflow {
4271 process_sse_line(line_buf, usage);
4272 }
4273 line_buf.clear();
4274 *overflow = false;
4275 } else if *overflow {
4276 } else {
4278 line_buf.push(byte);
4279 if line_buf.len() > SSE_LINE_BUF_CAP {
4280 *overflow = true;
4281 line_buf.clear(); }
4283 }
4284 }
4285 }
4287
4288fn process_sse_line(line: &[u8], usage: &mut SseUsage) {
4291 let Some(data) = line.strip_prefix(b"data: ") else {
4292 return;
4293 };
4294 let data = data.strip_suffix(b"\r").unwrap_or(data);
4296 if data == b"[DONE]" {
4298 usage.completed = true;
4299 return;
4300 }
4301 let Ok(json) = serde_json::from_slice::<Value>(data) else {
4302 return;
4303 };
4304
4305 if usage.model.is_none() {
4307 let model = json
4308 .get("model")
4309 .or_else(|| json.pointer("/message/model"))
4310 .and_then(Value::as_str);
4311 if let Some(m) = model {
4312 usage.model = Some(m.to_owned());
4313 }
4314 }
4315
4316 match json.get("type").and_then(Value::as_str) {
4317 Some("message_start") => {
4318 let u = |ptr: &str| json.pointer(ptr).and_then(Value::as_u64).unwrap_or(0);
4320 usage.in_tokens = u("/message/usage/input_tokens");
4321 usage.cache_write_tokens = u("/message/usage/cache_creation_input_tokens");
4322 usage.cache_read_tokens = u("/message/usage/cache_read_input_tokens");
4323 }
4324 Some("message_delta") => {
4325 if let Some(n) = json.pointer("/usage/output_tokens").and_then(Value::as_u64) {
4327 usage.out_tokens = n;
4328 }
4329 if let Some(stop) = json.pointer("/delta/stop_reason").and_then(Value::as_str) {
4330 usage.stop_reason = Some(stop.to_owned());
4331 usage.completed = true;
4332 }
4333 }
4334 Some("message_stop") => usage.completed = true,
4335 _ => {
4336 let is_gemini = json.get("candidates").is_some() || json.get("usageMetadata").is_some();
4341 if let Some(fr) = json
4342 .pointer("/candidates/0/finishReason")
4343 .and_then(Value::as_str)
4344 .filter(|fr| !fr.is_empty() && *fr != "FINISH_REASON_UNSPECIFIED")
4345 {
4346 usage.stop_reason = Some(fr.to_owned());
4347 usage.completed = true;
4348 }
4349 if json.pointer("/usageMetadata/promptTokenCount").is_some() {
4350 let u = |ptr: &str| json.pointer(ptr).and_then(Value::as_u64).unwrap_or(0);
4351 let cached = u("/usageMetadata/cachedContentTokenCount");
4353 usage.in_tokens = u("/usageMetadata/promptTokenCount").saturating_sub(cached);
4354 usage.cache_read_tokens = cached;
4355 usage.out_tokens = u("/usageMetadata/candidatesTokenCount")
4357 + u("/usageMetadata/thoughtsTokenCount");
4358 }
4359 if is_gemini {
4360 return;
4361 }
4362 if let Some(pt) = json.pointer("/usage/prompt_tokens").and_then(Value::as_u64) {
4365 let cached = json
4367 .pointer("/usage/prompt_tokens_details/cached_tokens")
4368 .and_then(Value::as_u64)
4369 .unwrap_or(0);
4370 usage.in_tokens = pt.saturating_sub(cached);
4371 usage.cache_read_tokens = cached;
4372 }
4373 if let Some(ct) = json
4374 .pointer("/usage/completion_tokens")
4375 .and_then(Value::as_u64)
4376 {
4377 usage.out_tokens = ct;
4378 }
4379 if let Some(fr) = json
4381 .pointer("/choices/0/finish_reason")
4382 .and_then(Value::as_str)
4383 .filter(|fr| !fr.is_empty())
4384 {
4385 usage.stop_reason = Some(fr.to_owned());
4386 usage.completed = true;
4387 }
4388 }
4389 }
4390}
4391
4392#[allow(clippy::too_many_arguments)]
4397fn spawn_trace(
4398 state: &AppState,
4399 req_body: Bytes,
4400 resp_body: Option<Bytes>,
4401 latency_ms: u64,
4402 session_header: Option<String>,
4403 tenant: String,
4404 provider: &'static str,
4405 api: &'static str,
4406 model_hint: Option<String>,
4407) {
4408 let config = state.config.clone();
4409 let traces = state.traces.clone();
4410 let spill = state.spill.clone();
4411 tokio::spawn(async move {
4412 let mut trace = match resp_body {
4413 Some(resp) => build_trace(
4414 &config,
4415 &req_body,
4416 &resp,
4417 latency_ms,
4418 session_header.as_deref(),
4419 provider,
4420 api,
4421 model_hint.as_deref(),
4422 ),
4423 None => build_error_trace(
4424 &config,
4425 &req_body,
4426 latency_ms,
4427 session_header.as_deref(),
4428 provider,
4429 api,
4430 ),
4431 };
4432 trace.tenant_id = tenant;
4434 offer_trace(&traces, spill.as_ref(), trace);
4435 });
4436}
4437
4438fn session_id(session_header: Option<&str>, trace_id: Uuid) -> String {
4440 session_header
4441 .map(str::to_owned)
4442 .unwrap_or_else(|| trace_id.to_string())
4443}
4444
4445fn prompt_hash(salt: &str, body: &[u8]) -> String {
4448 let mut salted = Vec::with_capacity(salt.len() + body.len());
4449 salted.extend_from_slice(salt.as_bytes());
4450 salted.extend_from_slice(body);
4451 sha256_hex(&salted)
4452}
4453
4454fn request_features(body: &[u8]) -> (Option<String>, u32, bool) {
4458 let Ok(json) = serde_json::from_slice::<Value>(body) else {
4459 return (None, 0, false);
4460 };
4461 let model = json.get("model").and_then(Value::as_str).map(str::to_owned);
4462 let tool_count = json
4463 .get("tools")
4464 .and_then(Value::as_array)
4465 .map_or(0, |tools| u32::try_from(tools.len()).unwrap_or(u32::MAX));
4466 let has_images = json
4467 .get("messages")
4468 .and_then(Value::as_array)
4469 .is_some_and(|messages| messages.iter().any(message_has_image));
4470 (model, tool_count, has_images)
4471}
4472
4473fn message_has_image(message: &Value) -> bool {
4475 message
4476 .get("content")
4477 .and_then(Value::as_array)
4478 .is_some_and(|blocks| {
4479 blocks
4480 .iter()
4481 .any(|block| block.get("type").and_then(Value::as_str) == Some("image"))
4482 })
4483}
4484
4485#[derive(Debug, Default, Clone, PartialEq, Eq)]
4487struct ResponseUsage {
4488 model: Option<String>,
4489 in_tokens: u64,
4490 cache_write_tokens: u64,
4491 cache_read_tokens: u64,
4492 out_tokens: u64,
4493}
4494
4495fn response_usage(body: &[u8]) -> ResponseUsage {
4503 let Ok(json) = serde_json::from_slice::<Value>(body) else {
4504 return ResponseUsage::default();
4505 };
4506 let u = |ptr: &str| json.pointer(ptr).and_then(Value::as_u64).unwrap_or(0);
4507 let model = json.get("model").and_then(Value::as_str).map(str::to_owned);
4508 if json.pointer("/usageMetadata/promptTokenCount").is_some() {
4511 let cached = u("/usageMetadata/cachedContentTokenCount");
4513 return ResponseUsage {
4514 model, in_tokens: u("/usageMetadata/promptTokenCount").saturating_sub(cached),
4516 cache_write_tokens: 0,
4517 cache_read_tokens: cached,
4518 out_tokens: u("/usageMetadata/candidatesTokenCount")
4519 + u("/usageMetadata/thoughtsTokenCount"),
4520 };
4521 }
4522 if json.pointer("/usage/prompt_tokens").is_some() {
4525 let cached = u("/usage/prompt_tokens_details/cached_tokens");
4526 return ResponseUsage {
4527 model,
4528 in_tokens: u("/usage/prompt_tokens").saturating_sub(cached),
4529 cache_write_tokens: 0,
4530 cache_read_tokens: cached,
4531 out_tokens: u("/usage/completion_tokens"),
4532 };
4533 }
4534 ResponseUsage {
4535 model,
4536 in_tokens: u("/usage/input_tokens"),
4537 cache_write_tokens: u("/usage/cache_creation_input_tokens"),
4538 cache_read_tokens: u("/usage/cache_read_input_tokens"),
4539 out_tokens: u("/usage/output_tokens"),
4540 }
4541}
4542
4543#[allow(clippy::too_many_arguments)]
4549fn build_trace(
4550 config: &ProxyConfig,
4551 req_body: &Bytes,
4552 resp_body: &Bytes,
4553 latency_ms: u64,
4554 session_header: Option<&str>,
4555 provider: &str,
4556 api: &str,
4557 model_hint: Option<&str>,
4558) -> Trace {
4559 let (req_model, tool_count, has_images) = request_features(req_body);
4560 let usage = response_usage(resp_body);
4561 let (cache_write_tokens, cache_read_tokens) =
4562 (usage.cache_write_tokens, usage.cache_read_tokens);
4563 let (in_tokens, out_tokens) = (usage.in_tokens, usage.out_tokens);
4564 let model = usage
4567 .model
4568 .as_deref()
4569 .or(req_model.as_deref())
4570 .or(model_hint)
4571 .unwrap_or("unknown")
4572 .to_owned();
4573
4574 let cost_usd = config
4575 .prices
4576 .cost_usd_with_cache(
4577 &format!(
4579 "{provider}/{}",
4580 model
4581 .strip_prefix(&format!("{provider}/"))
4582 .unwrap_or(&model)
4583 ),
4584 in_tokens,
4585 cache_write_tokens,
4586 cache_read_tokens,
4587 out_tokens,
4588 )
4589 .or_else(|_| {
4590 let bare_req = req_model.as_deref().unwrap_or("unknown");
4597 let bare_req = bare_req
4598 .strip_prefix(&format!("{provider}/"))
4599 .unwrap_or(bare_req);
4600 let bare_resp = model
4601 .strip_prefix(&format!("{provider}/"))
4602 .unwrap_or(&model);
4603 if is_snapshot_of(bare_resp, bare_req) {
4604 config.prices.cost_usd_with_cache(
4605 &format!("{provider}/{bare_req}"),
4606 in_tokens,
4607 cache_write_tokens,
4608 cache_read_tokens,
4609 out_tokens,
4610 )
4611 } else {
4612 Err(firstpass_core::error::Error::UnknownModel(model.clone()))
4613 }
4614 })
4615 .unwrap_or(0.0);
4616
4617 let attempt = Attempt {
4618 rung: 0,
4619 model,
4620 provider: provider.to_owned(),
4621 in_tokens,
4622 cache_write_tokens,
4623 cache_read_tokens,
4624 out_tokens,
4625 cost_usd,
4626 latency_ms,
4627 gates: Vec::new(),
4628 verdict: Verdict::Pass,
4629 truncated: serde_json::from_slice::<Value>(resp_body)
4630 .is_ok_and(|v| crate::provider::hit_output_limit(&v)),
4631 };
4632
4633 let mut trace = base_trace(config, req_body, latency_ms, session_header, api);
4634 trace.request.features.prompt_token_bucket = token_bucket(in_tokens);
4635 trace.request.features.tool_count = tool_count;
4636 trace.request.features.has_images = has_images;
4637 trace.attempts.push(attempt);
4638 trace.final_ = FinalOutcome {
4639 served_rung: Some(0),
4640 served_from: ServedFrom::Attempt,
4641 total_cost_usd: cost_usd,
4642 gate_cost_usd: 0.0,
4643 total_latency_ms: latency_ms,
4644 escalations: 0,
4645 counterfactual_baseline_usd: cost_usd,
4646 savings_usd: 0.0,
4647 cache_source: None,
4648 };
4649 trace.recompute_savings();
4650 trace
4651}
4652
4653#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4662enum StreamOutcome {
4663 Served,
4666 MidStreamError,
4670 Partial,
4674}
4675
4676#[allow(clippy::too_many_arguments)]
4677fn build_stream_trace_from_usage(
4678 config: &ProxyConfig,
4679 req_body: &Bytes,
4680 latency_ms: u64,
4681 session_header: Option<&str>,
4682 usage: SseUsage,
4683 provider: &str,
4684 outcome: StreamOutcome,
4685 api: &str,
4686 model_hint: Option<&str>,
4687) -> Trace {
4688 let (req_model, tool_count, has_images) = request_features(req_body);
4689 let (in_tokens, cache_write_tokens, cache_read_tokens, out_tokens) = (
4690 usage.in_tokens,
4691 usage.cache_write_tokens,
4692 usage.cache_read_tokens,
4693 usage.out_tokens,
4694 );
4695 let model = usage
4699 .model
4700 .as_deref()
4701 .or(req_model.as_deref())
4702 .or(model_hint)
4703 .unwrap_or("unknown")
4704 .to_owned();
4705
4706 let cost_usd = config
4707 .prices
4708 .cost_usd_with_cache(
4709 &format!(
4711 "{provider}/{}",
4712 model
4713 .strip_prefix(&format!("{provider}/"))
4714 .unwrap_or(&model)
4715 ),
4716 in_tokens,
4717 cache_write_tokens,
4718 cache_read_tokens,
4719 out_tokens,
4720 )
4721 .or_else(|_| {
4722 let bare_req = req_model.as_deref().unwrap_or("unknown");
4731 let bare_req = bare_req
4732 .strip_prefix(&format!("{provider}/"))
4733 .unwrap_or(bare_req);
4734 let bare_resp = model
4736 .strip_prefix(&format!("{provider}/"))
4737 .unwrap_or(&model);
4738 if is_snapshot_of(bare_resp, bare_req) {
4739 config.prices.cost_usd_with_cache(
4740 &format!("{provider}/{bare_req}"),
4741 in_tokens,
4742 cache_write_tokens,
4743 cache_read_tokens,
4744 out_tokens,
4745 )
4746 } else {
4747 Err(firstpass_core::error::Error::UnknownModel(model.clone()))
4748 }
4749 })
4750 .unwrap_or(0.0);
4751
4752 let truncated = matches!(
4755 usage.stop_reason.as_deref(),
4756 Some("max_tokens") | Some("length") | Some("MAX_TOKENS")
4757 );
4758
4759 let (verdict, served_from, served_rung) = match outcome {
4760 StreamOutcome::Served => (Verdict::Pass, ServedFrom::Attempt, Some(0u32)),
4761 StreamOutcome::MidStreamError => (Verdict::Abstain, ServedFrom::Error, None),
4762 StreamOutcome::Partial => (Verdict::Abstain, ServedFrom::Attempt, Some(0u32)),
4763 };
4764
4765 let attempt = Attempt {
4766 rung: 0,
4767 model,
4768 provider: provider.to_owned(),
4769 in_tokens,
4770 cache_write_tokens,
4771 cache_read_tokens,
4772 out_tokens,
4773 cost_usd,
4774 latency_ms,
4775 gates: Vec::new(),
4776 verdict,
4777 truncated,
4778 };
4779
4780 let mut trace = base_trace(config, req_body, latency_ms, session_header, api);
4781 trace.request.features.prompt_token_bucket = token_bucket(in_tokens);
4782 trace.request.features.tool_count = tool_count;
4783 trace.request.features.has_images = has_images;
4784 trace.attempts.push(attempt);
4785 trace.final_ = FinalOutcome {
4786 served_rung,
4787 served_from,
4788 total_cost_usd: cost_usd,
4789 gate_cost_usd: 0.0,
4790 total_latency_ms: latency_ms,
4791 escalations: 0,
4792 counterfactual_baseline_usd: cost_usd,
4793 savings_usd: 0.0,
4794 cache_source: None,
4795 };
4796 trace.recompute_savings();
4797 trace
4798}
4799
4800fn is_snapshot_of(resp_model: &str, req_model: &str) -> bool {
4808 let Some(suffix) = resp_model.strip_prefix(&format!("{req_model}-")) else {
4809 return false;
4810 };
4811 is_date_version_suffix(suffix)
4812}
4813
4814fn is_date_version_suffix(s: &str) -> bool {
4817 if s.len() == 8 && s.bytes().all(|b| b.is_ascii_digit()) {
4819 return true;
4820 }
4821 if s.len() == 10 {
4823 let b = s.as_bytes();
4824 return b[4] == b'-'
4825 && b[7] == b'-'
4826 && b[..4].iter().all(|c| c.is_ascii_digit())
4827 && b[5..7].iter().all(|c| c.is_ascii_digit())
4828 && b[8..].iter().all(|c| c.is_ascii_digit());
4829 }
4830 false
4831}
4832
4833fn build_error_trace(
4837 config: &ProxyConfig,
4838 req_body: &Bytes,
4839 latency_ms: u64,
4840 session_header: Option<&str>,
4841 provider: &str,
4842 api: &str,
4843) -> Trace {
4844 let (_, tool_count, has_images) = request_features(req_body);
4845 let mut trace = base_trace(config, req_body, latency_ms, session_header, api);
4846 trace.request.features.tool_count = tool_count;
4847 trace.request.features.has_images = has_images;
4848 trace.upstream_provider = provider.to_owned();
4849 trace.final_ = FinalOutcome {
4850 served_rung: None,
4851 served_from: ServedFrom::Error,
4852 total_cost_usd: 0.0,
4853 gate_cost_usd: 0.0,
4854 total_latency_ms: latency_ms,
4855 escalations: 0,
4856 counterfactual_baseline_usd: 0.0,
4857 savings_usd: 0.0,
4858 cache_source: None,
4859 };
4860 trace.recompute_savings();
4861 trace
4862}
4863
4864fn base_trace(
4867 config: &ProxyConfig,
4868 req_body: &Bytes,
4869 latency_ms: u64,
4870 session_header: Option<&str>,
4871 api: &str,
4872) -> Trace {
4873 let trace_id = Uuid::now_v7();
4874 let mut features = Features::new(TaskKind::Other);
4875 features.hour_bucket = hour_bucket(jiff::Timestamp::now());
4876
4877 Trace {
4878 trace_id,
4879 prev_hash: GENESIS_HASH.to_owned(),
4880 tenant_id: config.tenant_id.clone(),
4881 session_id: session_id(session_header, trace_id),
4882 ts: jiff::Timestamp::now(),
4883 mode: Mode::Observe,
4884 policy: PolicyRef {
4885 id: "observe-passthrough@v0".to_owned(),
4886 explore: false,
4887 propensity: None,
4888 mode_profile: None,
4889 },
4890 request: RequestInfo {
4891 api: api.to_owned(),
4892 prompt_hash: prompt_hash(&config.prompt_salt, req_body),
4893 features,
4894 },
4895 attempts: Vec::new(),
4896 deferred: Vec::new(),
4897 final_: FinalOutcome {
4898 served_rung: None,
4899 served_from: ServedFrom::Error,
4900 total_cost_usd: 0.0,
4901 gate_cost_usd: 0.0,
4902 total_latency_ms: latency_ms,
4903 escalations: 0,
4904 counterfactual_baseline_usd: 0.0,
4905 savings_usd: 0.0,
4906 cache_source: None,
4907 },
4908 probe: None,
4909 rollout: None,
4910 shadow: None,
4911 route_ix: None,
4912 predicted_pass: None,
4913 decision_prior: None,
4914 elastic: None,
4915 risk: None,
4916 experiment: None,
4917 upstream_provider: String::new(),
4918 }
4919}
4920
4921fn openai_messages_have_tool_calls(body: &[u8]) -> bool {
4926 serde_json::from_slice::<Value>(body)
4927 .ok()
4928 .and_then(|json| {
4929 json.get("messages").and_then(Value::as_array).map(|msgs| {
4930 msgs.iter().any(|m| {
4931 m.get("tool_calls").is_some()
4932 || m.get("role").and_then(Value::as_str) == Some("tool")
4933 })
4934 })
4935 })
4936 .unwrap_or(false)
4937}
4938
4939fn openai_has_http_images(body: &[u8]) -> bool {
4943 serde_json::from_slice::<Value>(body)
4944 .ok()
4945 .and_then(|json| {
4946 json.get("messages").and_then(Value::as_array).map(|msgs| {
4947 msgs.iter().any(|m| {
4948 m.get("content")
4949 .and_then(Value::as_array)
4950 .is_some_and(|parts| {
4951 parts.iter().any(|p| {
4952 p.get("type").and_then(Value::as_str) == Some("image_url")
4953 && p.pointer("/image_url/url")
4954 .and_then(Value::as_str)
4955 .is_some_and(|u| {
4956 u.starts_with("http://") || u.starts_with("https://")
4957 })
4958 })
4959 })
4960 })
4961 })
4962 })
4963 .unwrap_or(false)
4964}
4965
4966fn openai_messages_have_images(body: &[u8]) -> bool {
4969 serde_json::from_slice::<Value>(body)
4970 .ok()
4971 .and_then(|json| {
4972 json.get("messages").and_then(Value::as_array).map(|msgs| {
4973 msgs.iter().any(|m| {
4974 m.get("content")
4975 .and_then(Value::as_array)
4976 .is_some_and(|parts| {
4977 parts
4978 .iter()
4979 .any(|p| p.get("type").and_then(Value::as_str) == Some("image_url"))
4980 })
4981 })
4982 })
4983 })
4984 .unwrap_or(false)
4985}
4986
4987fn extract_openai_features(headers: &HeaderMap, body: &[u8]) -> Features {
4990 let Ok(json) = serde_json::from_slice::<Value>(body) else {
4991 let mut f = Features::new(TaskKind::Other);
4992 f.hour_bucket = hour_bucket(jiff::Timestamp::now());
4993 return f;
4994 };
4995 let tool_count = json
4996 .get("tools")
4997 .and_then(Value::as_array)
4998 .map_or(0, |tools| u32::try_from(tools.len()).unwrap_or(u32::MAX));
4999 let has_images = openai_messages_have_images(body);
5000 let mut f = Features::new(TaskKind::Other);
5001 f.agent = header_str(headers, AGENT_HEADER);
5002 f.subagent = header_str(headers, SUBAGENT_HEADER);
5003 f.tool_count = tool_count;
5004 f.has_images = has_images;
5005 f.prompt_token_bucket = token_bucket(body.len() as u64);
5006 f.hour_bucket = hour_bucket(jiff::Timestamp::now());
5007 f.difficulty_hint = DifficultyHint::score(trajectory_signals(body)).as_u8();
5011 f
5012}
5013
5014fn parse_data_url(url: &str) -> Option<(&str, &str)> {
5018 let rest = url.strip_prefix("data:")?;
5019 let (meta, data) = rest.split_once(',')?;
5020 let media_type = meta.strip_suffix(";base64")?;
5021 Some((media_type, data))
5022}
5023
5024fn translate_openai_user_content(content: &Value) -> Option<Value> {
5027 match content {
5028 Value::String(_) => Some(content.clone()),
5030 Value::Array(parts) => {
5031 let mut blocks: Vec<Value> = Vec::with_capacity(parts.len());
5032 for part in parts {
5033 match part.get("type").and_then(Value::as_str) {
5034 Some("text") => {
5035 let text = part.get("text").and_then(Value::as_str).unwrap_or("");
5036 blocks.push(serde_json::json!({ "type": "text", "text": text }));
5037 }
5038 Some("image_url") => {
5039 let url = part.pointer("/image_url/url").and_then(Value::as_str)?;
5040 if url.starts_with("http://") || url.starts_with("https://") {
5041 return None; }
5043 let (media_type, data) = parse_data_url(url)?;
5045 blocks.push(serde_json::json!({
5046 "type": "image",
5047 "source": { "type": "base64", "media_type": media_type, "data": data }
5048 }));
5049 }
5050 _ => {} }
5052 }
5053 Some(Value::Array(blocks))
5054 }
5055 _ => Some(Value::String(String::new())),
5056 }
5057}
5058
5059fn translate_openai_tools(tools: &Value) -> Value {
5063 let Some(arr) = tools.as_array() else {
5064 return Value::Null;
5065 };
5066 let converted: Vec<Value> = arr
5067 .iter()
5068 .map(|tool| {
5069 let func = tool.get("function").unwrap_or(&Value::Null);
5070 let mut out = serde_json::json!({
5071 "name": func.get("name").cloned().unwrap_or(Value::String(String::new())),
5072 "input_schema": func.get("parameters").cloned()
5073 .unwrap_or_else(|| serde_json::json!({ "type": "object" })),
5074 });
5075 if let Some(desc) = func.get("description") {
5076 out["description"] = desc.clone();
5077 }
5078 out
5079 })
5080 .collect();
5081 Value::Array(converted)
5082}
5083
5084fn translate_openai_tool_choice(tc: &Value) -> Value {
5086 match tc {
5087 Value::String(s) => match s.as_str() {
5088 "auto" => serde_json::json!({ "type": "auto" }),
5089 "required" => serde_json::json!({ "type": "any" }),
5090 _ => serde_json::json!({ "type": "auto" }),
5092 },
5093 Value::Object(_) => {
5094 if tc.get("type").and_then(Value::as_str) == Some("function") {
5096 let name = tc.pointer("/function/name").cloned().unwrap_or(Value::Null);
5097 serde_json::json!({ "type": "tool", "name": name })
5098 } else {
5099 serde_json::json!({ "type": "auto" })
5100 }
5101 }
5102 _ => serde_json::json!({ "type": "auto" }),
5103 }
5104}
5105
5106pub fn parse_openai_request(body: &[u8], carry_raw: bool) -> Option<ModelRequest> {
5119 let json: Value = serde_json::from_slice(body).ok()?;
5120 let raw = if carry_raw { json.clone() } else { Value::Null };
5121
5122 let messages_json = json.get("messages")?.as_array()?;
5123
5124 let mut system: Option<String> = None;
5125 let mut messages: Vec<ChatMessage> = Vec::with_capacity(messages_json.len());
5126 let mut tools = Value::Null;
5127 let mut tool_choice_override: Option<Value> = None;
5128
5129 for msg in messages_json {
5130 let role = msg.get("role").and_then(Value::as_str).unwrap_or("user");
5131 match role {
5132 "system" => {
5133 if let Some(s) = msg.get("content").and_then(Value::as_str) {
5138 system = Some(s.to_owned());
5139 }
5140 }
5141 "user" => {
5142 let content_val = msg.get("content").unwrap_or(&Value::Null);
5143 let translated = translate_openai_user_content(content_val)?;
5144 messages.push(ChatMessage {
5145 role: "user".to_owned(),
5146 content: translated,
5147 });
5148 }
5149 "assistant" => {
5150 if let Some(tc_arr) = msg.get("tool_calls").and_then(Value::as_array) {
5151 let mut blocks: Vec<Value> = Vec::new();
5153 if let Some(text) = msg.get("content").and_then(Value::as_str)
5155 && !text.is_empty()
5156 {
5157 blocks.push(serde_json::json!({ "type": "text", "text": text }));
5158 }
5159 for tc in tc_arr {
5160 let id = tc.get("id").and_then(Value::as_str).unwrap_or("");
5161 let func = tc.get("function").unwrap_or(&Value::Null);
5162 let name = func.get("name").and_then(Value::as_str).unwrap_or("");
5163 let args_str = func
5164 .get("arguments")
5165 .and_then(Value::as_str)
5166 .unwrap_or("{}");
5167 let input: Value = serde_json::from_str(args_str)
5168 .unwrap_or_else(|_| serde_json::json!({}));
5169 blocks.push(serde_json::json!({
5170 "type": "tool_use",
5171 "id": id,
5172 "name": name,
5173 "input": input,
5174 }));
5175 }
5176 messages.push(ChatMessage {
5177 role: "assistant".to_owned(),
5178 content: Value::Array(blocks),
5179 });
5180 } else {
5181 let content = msg
5183 .get("content")
5184 .cloned()
5185 .unwrap_or_else(|| Value::String(String::new()));
5186 messages.push(ChatMessage {
5187 role: "assistant".to_owned(),
5188 content,
5189 });
5190 }
5191 }
5192 "tool" => {
5193 let tool_call_id = msg
5195 .get("tool_call_id")
5196 .and_then(Value::as_str)
5197 .unwrap_or("");
5198 let content = msg
5199 .get("content")
5200 .cloned()
5201 .unwrap_or_else(|| Value::String(String::new()));
5202 let result_block = serde_json::json!({
5203 "type": "tool_result",
5204 "tool_use_id": tool_call_id,
5205 "content": content,
5206 });
5207 messages.push(ChatMessage {
5208 role: "user".to_owned(),
5209 content: Value::Array(vec![result_block]),
5210 });
5211 }
5212 _ => {} }
5214 }
5215
5216 if !carry_raw {
5218 if let Some(t) = json.get("tools") {
5219 tools = translate_openai_tools(t);
5220 }
5221 if let Some(tc) = json.get("tool_choice") {
5222 tool_choice_override = Some(translate_openai_tool_choice(tc));
5223 }
5224 } else {
5225 tools = json.get("tools").cloned().unwrap_or(Value::Null);
5226 }
5227
5228 let max_tokens = json
5229 .get("max_tokens")
5230 .and_then(Value::as_u64)
5231 .and_then(|n| u32::try_from(n).ok())
5232 .unwrap_or(1024);
5233
5234 let model = json
5235 .get("model")
5236 .and_then(Value::as_str)
5237 .unwrap_or_default()
5238 .to_owned();
5239
5240 let _ = tool_choice_override; Some(ModelRequest {
5251 model,
5252 system,
5253 messages,
5254 max_tokens,
5255 tools,
5256 raw,
5257 cache_prefix: false,
5258 })
5259}
5260
5261fn extract_openai_content_and_tools(raw: &Value, text: &str) -> (Value, Option<Value>) {
5268 if let Some(blocks) = raw.get("content").and_then(Value::as_array) {
5270 let mut text_parts: Vec<&str> = Vec::new();
5271 let mut tool_calls: Vec<Value> = Vec::new();
5272 for block in blocks {
5273 match block.get("type").and_then(Value::as_str) {
5274 Some("text") => {
5275 if let Some(t) = block.get("text").and_then(Value::as_str) {
5276 text_parts.push(t);
5277 }
5278 }
5279 Some("tool_use") => {
5280 let id = block.get("id").and_then(Value::as_str).unwrap_or("");
5281 let name = block.get("name").and_then(Value::as_str).unwrap_or("");
5282 let input_str = block
5283 .get("input")
5284 .map_or_else(|| "{}".to_owned(), std::string::ToString::to_string);
5285 tool_calls.push(serde_json::json!({
5286 "id": id,
5287 "type": "function",
5288 "function": { "name": name, "arguments": input_str },
5289 }));
5290 }
5291 _ => {}
5292 }
5293 }
5294 let content_text = if tool_calls.is_empty() || !text_parts.is_empty() {
5295 Value::String(text_parts.join(""))
5296 } else {
5297 Value::Null };
5299 let tc = if tool_calls.is_empty() {
5300 None
5301 } else {
5302 Some(Value::Array(tool_calls))
5303 };
5304 return (content_text, tc);
5305 }
5306
5307 if let Some(msg) = raw.pointer("/choices/0/message") {
5309 let content = msg
5310 .get("content")
5311 .cloned()
5312 .unwrap_or(Value::String(text.to_owned()));
5313 let tc = msg.get("tool_calls").cloned();
5314 return (content, tc);
5315 }
5316
5317 (Value::String(text.to_owned()), None)
5319}
5320
5321fn openai_response_json(resp: &ModelResponse) -> Value {
5324 let (content_text, tool_calls) = extract_openai_content_and_tools(&resp.raw, &resp.text);
5325 let finish_reason = if resp.truncated() {
5326 "length"
5327 } else if tool_calls.is_some() {
5328 "tool_calls"
5329 } else {
5330 "stop"
5331 };
5332 let mut message = serde_json::json!({
5333 "role": "assistant",
5334 "content": content_text,
5335 });
5336 if let Some(tc) = tool_calls {
5337 message["tool_calls"] = tc;
5338 }
5339 serde_json::json!({
5340 "id": format!("chatcmpl-{}", Uuid::now_v7()),
5341 "object": "chat.completion",
5342 "created": jiff::Timestamp::now().as_second(),
5343 "model": resp.model,
5344 "choices": [{
5345 "index": 0,
5346 "message": message,
5347 "finish_reason": finish_reason,
5348 }],
5349 "usage": {
5350 "prompt_tokens": resp.in_tokens,
5351 "completion_tokens": resp.out_tokens,
5352 "total_tokens": resp.in_tokens + resp.out_tokens,
5353 }
5354 })
5355}
5356
5357fn stream_options_include_usage(body: &[u8]) -> bool {
5362 serde_json::from_slice::<Value>(body)
5363 .ok()
5364 .and_then(|v| v.pointer("/stream_options/include_usage").cloned())
5365 .and_then(|v| v.as_bool())
5366 .unwrap_or(false)
5367}
5368
5369fn openai_sse_from_message(message: &Value, include_usage: bool) -> String {
5370 let id = message
5371 .get("id")
5372 .and_then(Value::as_str)
5373 .unwrap_or("chatcmpl-unknown")
5374 .to_owned();
5375 let created = message.get("created").cloned().unwrap_or(Value::from(0));
5376 let model = message
5377 .get("model")
5378 .and_then(Value::as_str)
5379 .unwrap_or("unknown");
5380
5381 let choices = message.get("choices").and_then(Value::as_array);
5382 let msg = choices
5383 .and_then(|c| c.first())
5384 .and_then(|c| c.get("message"));
5385 let content = msg
5386 .and_then(|m| m.get("content"))
5387 .cloned()
5388 .unwrap_or(Value::Null);
5389 let tool_calls = msg.and_then(|m| m.get("tool_calls")).cloned();
5390 let finish_reason = choices
5391 .and_then(|c| c.first())
5392 .and_then(|c| c.get("finish_reason"))
5393 .cloned()
5394 .unwrap_or_else(|| Value::String("stop".to_owned()));
5395
5396 let mut out = String::new();
5397 let chunk = |delta: Value| {
5398 serde_json::json!({
5399 "id": id,
5400 "object": "chat.completion.chunk",
5401 "created": created,
5402 "model": model,
5403 "choices": [{ "index": 0, "delta": delta, "finish_reason": Value::Null }]
5404 })
5405 };
5406
5407 let role_chunk = chunk(serde_json::json!({ "role": "assistant", "content": "" }));
5409 out.push_str("data: ");
5410 out.push_str(&role_chunk.to_string());
5411 out.push_str("\n\n");
5412
5413 if let Value::String(text) = &content
5415 && !text.is_empty()
5416 {
5417 let content_chunk = chunk(serde_json::json!({ "content": text }));
5418 out.push_str("data: ");
5419 out.push_str(&content_chunk.to_string());
5420 out.push_str("\n\n");
5421 }
5422
5423 if let Some(tc) = tool_calls {
5425 let tc_with_index: Vec<Value> = tc
5426 .as_array()
5427 .map(|arr| {
5428 arr.iter()
5429 .enumerate()
5430 .map(|(idx, call)| {
5431 let mut c = call.clone();
5432 c["index"] = Value::from(idx);
5433 c
5434 })
5435 .collect()
5436 })
5437 .unwrap_or_default();
5438 let tc_chunk = chunk(serde_json::json!({ "tool_calls": tc_with_index }));
5439 out.push_str("data: ");
5440 out.push_str(&tc_chunk.to_string());
5441 out.push_str("\n\n");
5442 }
5443
5444 let finish_chunk = serde_json::json!({
5446 "id": id,
5447 "object": "chat.completion.chunk",
5448 "created": created,
5449 "model": model,
5450 "choices": [{ "index": 0, "delta": {}, "finish_reason": finish_reason }]
5451 });
5452 out.push_str("data: ");
5453 out.push_str(&finish_chunk.to_string());
5454 out.push_str("\n\n");
5455
5456 if include_usage && let Some(usage) = message.get("usage") {
5459 let usage_chunk = serde_json::json!({
5460 "id": id,
5461 "object": "chat.completion.chunk",
5462 "created": created,
5463 "model": model,
5464 "choices": [],
5465 "usage": usage
5466 });
5467 out.push_str("data: ");
5468 out.push_str(&usage_chunk.to_string());
5469 out.push_str("\n\n");
5470 }
5471
5472 out.push_str("data: [DONE]\n\n");
5473 out
5474}
5475
5476#[allow(clippy::too_many_arguments)]
5481async fn handle_enforce_openai(
5482 state: &AppState,
5483 headers: &HeaderMap,
5484 body: &Bytes,
5485 features: Features,
5486 route: &Route,
5487 route_ix: usize,
5490 session_header: Option<String>,
5491 tenant: String,
5492 routing_mode: RoutingMode,
5493) -> Response {
5494 if is_stream_request(body) {
5495 let include_usage = stream_options_include_usage(body);
5496 let (tx, rx) = tokio::sync::oneshot::channel::<Result<Value, ProxyError>>();
5497 let (state_c, headers_c, body_c, route_c) =
5498 (state.clone(), headers.clone(), body.clone(), route.clone());
5499 tokio::spawn(async move {
5500 let out = enforce_pipeline_openai(
5501 &state_c,
5502 &headers_c,
5503 &body_c,
5504 features,
5505 &route_c,
5506 route_ix,
5507 session_header,
5508 tenant,
5509 routing_mode,
5510 )
5511 .await;
5512 let _ = tx.send(out);
5513 });
5514 return sse_keepalive_response(
5515 rx,
5516 Box::new(move |v| openai_sse_from_message(v, include_usage)),
5517 sse_error_event,
5518 );
5519 }
5520 match enforce_pipeline_openai(
5521 state,
5522 headers,
5523 body,
5524 features,
5525 route,
5526 route_ix,
5527 session_header,
5528 tenant,
5529 routing_mode,
5530 )
5531 .await
5532 {
5533 Ok(message) => (axum::http::StatusCode::OK, Json(message)).into_response(),
5534 Err(e) => e.into_response(),
5535 }
5536}
5537
5538async fn observe_passthrough_openai(
5544 state: AppState,
5545 headers: HeaderMap,
5546 body: Bytes,
5547 session_header: Option<String>,
5548 tenant: String,
5549) -> Response {
5550 observe_passthrough_openai_path(
5551 state,
5552 headers,
5553 body,
5554 session_header,
5555 tenant,
5556 crate::upstream::OPENAI_CHAT_PATH,
5557 )
5558 .await
5559}
5560
5561async fn observe_passthrough_openai_path(
5567 state: AppState,
5568 headers: HeaderMap,
5569 body: Bytes,
5570 session_header: Option<String>,
5571 tenant: String,
5572 path: &'static str,
5573) -> Response {
5574 if is_stream_request(&body) {
5575 return observe_stream_openai_path(state, headers, body, session_header, tenant, path)
5576 .await;
5577 }
5578 let start = Instant::now();
5579 let result = crate::upstream::forward_openai_path(
5580 &state.http,
5581 &state.config.upstream_openai,
5582 path,
5583 &headers,
5584 body.clone(),
5585 )
5586 .await;
5587 let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
5588 match result {
5589 Ok((status, resp_headers, resp_body)) => {
5590 let for_trace = status.is_success().then_some(resp_body.clone());
5591 spawn_trace(
5592 &state,
5593 body,
5594 for_trace,
5595 latency_ms,
5596 session_header,
5597 tenant,
5598 "openai",
5599 "openai.chat_completions",
5600 None,
5601 );
5602 (status, resp_headers, resp_body).into_response()
5603 }
5604 Err(err) => {
5605 spawn_trace(
5606 &state,
5607 body,
5608 None,
5609 latency_ms,
5610 session_header,
5611 tenant,
5612 "openai",
5613 "openai.chat_completions",
5614 None,
5615 );
5616 err.into_response()
5617 }
5618 }
5619}
5620
5621async fn observe_stream_openai_path(
5625 state: AppState,
5626 headers: HeaderMap,
5627 body: Bytes,
5628 session_header: Option<String>,
5629 tenant: String,
5630 path: &'static str,
5631) -> Response {
5632 let start = Instant::now();
5633 let result = crate::upstream::forward_openai_streaming_path(
5634 &state.http,
5635 &state.config.upstream_openai,
5636 path,
5637 &headers,
5638 body.clone(),
5639 )
5640 .await;
5641 let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
5642 match result {
5643 Ok((status, resp_headers, response)) => {
5644 let on_done = stream_trace_callback(
5645 &state,
5646 body,
5647 latency_ms,
5648 session_header,
5649 tenant,
5650 "openai",
5651 status.is_success(),
5652 "openai.chat_completions",
5653 None,
5654 );
5655 let tap = SseTap::new(response.bytes_stream(), on_done);
5656 let stream_body = Body::from_stream(tap);
5657 (status, resp_headers, stream_body).into_response()
5658 }
5659 Err(err) => {
5660 spawn_trace(
5661 &state,
5662 body,
5663 None,
5664 latency_ms,
5665 session_header,
5666 tenant,
5667 "openai",
5668 "openai.chat_completions",
5669 None,
5670 );
5671 err.into_response()
5672 }
5673 }
5674}
5675
5676async fn chat_completions(
5681 State(state): State<AppState>,
5682 Extension(TenantId(tenant)): Extension<TenantId>,
5683 headers: HeaderMap,
5684 body: Bytes,
5685) -> Response {
5686 let mut session_header = header_session(&headers, &state.config.prompt_salt);
5687
5688 if let Some(routing) = state.config.routing.as_ref() {
5689 let features = extract_openai_features(&headers, &body);
5690 if let Some(route) = routing
5691 .route_for(&features)
5692 .filter(|r| r.mode == Mode::Enforce && !r.ladder.is_empty())
5693 {
5694 session_header = session_header
5696 .take()
5697 .or_else(|| body_session_bytes(&body, &state.config.prompt_salt));
5698 let route_ix = routing
5700 .routes
5701 .iter()
5702 .position(|r| std::ptr::eq(r, route))
5703 .unwrap_or(0);
5704 let route = route.clone();
5705 let routing_mode = resolve_mode(&headers, &route, &state.config);
5707 if routing_mode == RoutingMode::Observe {
5709 return observe_passthrough_openai(state, headers, body, session_header, tenant)
5710 .await;
5711 }
5712 if enforce_can_handle_route(
5713 &features,
5714 &body,
5715 routing.escalation.enforce_structured,
5716 &route,
5717 &state.providers,
5718 Dialect::Openai,
5719 ) {
5720 return handle_enforce_openai(
5721 &state,
5722 &headers,
5723 &body,
5724 features,
5725 &route,
5726 route_ix,
5727 session_header,
5728 tenant,
5729 routing_mode,
5730 )
5731 .await;
5732 }
5733 tracing::info!(
5734 "enforce route matched but OpenAI structured request can't be routed faithfully (flag/ladder); serving via observe passthrough"
5735 );
5736 }
5737 }
5738 observe_passthrough_openai(state, headers, body, session_header, tenant).await
5739}
5740
5741async fn responses(
5754 State(state): State<AppState>,
5755 Extension(TenantId(tenant)): Extension<TenantId>,
5756 headers: HeaderMap,
5757 body: Bytes,
5758) -> Response {
5759 let mut session_header = header_session(&headers, &state.config.prompt_salt);
5760
5761 let Ok(parsed) = serde_json::from_slice::<Value>(&body) else {
5762 return observe_passthrough_openai_path(
5763 state,
5764 headers,
5765 body,
5766 session_header,
5767 tenant,
5768 crate::upstream::OPENAI_RESPONSES_PATH,
5769 )
5770 .await;
5771 };
5772 if crate::responses::has_untranslatable_content(&parsed) {
5776 return observe_passthrough_openai_path(
5777 state,
5778 headers,
5779 body,
5780 session_header,
5781 tenant,
5782 crate::upstream::OPENAI_RESPONSES_PATH,
5783 )
5784 .await;
5785 }
5786 let chat_body = Bytes::from(crate::responses::request_to_chat(&parsed).to_string());
5787
5788 if let Some(routing) = state.config.routing.as_ref() {
5789 let features = extract_openai_features(&headers, &chat_body);
5790 if let Some(route) = routing
5791 .route_for(&features)
5792 .filter(|r| r.mode == Mode::Enforce && !r.ladder.is_empty())
5793 {
5794 session_header = session_header
5796 .take()
5797 .or_else(|| body_session(&parsed, &state.config.prompt_salt));
5798 let route_ix = routing
5799 .routes
5800 .iter()
5801 .position(|r| std::ptr::eq(r, route))
5802 .unwrap_or(0);
5803 let route = route.clone();
5804 let routing_mode = resolve_mode(&headers, &route, &state.config);
5805 if routing_mode != RoutingMode::Observe
5806 && enforce_can_handle_route(
5807 &features,
5808 &chat_body,
5809 routing.escalation.enforce_structured,
5810 &route,
5811 &state.providers,
5812 Dialect::Openai,
5813 )
5814 {
5815 if is_stream_request(&body) {
5816 let (tx, rx) = tokio::sync::oneshot::channel::<Result<Value, ProxyError>>();
5817 tokio::spawn(async move {
5818 let out = enforce_pipeline_openai_as(
5819 &state,
5820 &headers,
5821 &chat_body,
5822 &body,
5823 "openai.responses",
5824 features,
5825 &route,
5826 route_ix,
5827 session_header,
5828 tenant,
5829 routing_mode,
5830 )
5831 .await;
5832 let _ = tx.send(out);
5833 });
5834 return sse_keepalive_response(
5835 rx,
5836 Box::new(|chat| {
5837 crate::responses::sse_from_response(
5838 &crate::responses::response_from_chat(chat),
5839 )
5840 }),
5841 responses_sse_error,
5842 );
5843 }
5844 return match enforce_pipeline_openai_as(
5845 &state,
5846 &headers,
5847 &chat_body,
5848 &body,
5849 "openai.responses",
5850 features,
5851 &route,
5852 route_ix,
5853 session_header,
5854 tenant,
5855 routing_mode,
5856 )
5857 .await
5858 {
5859 Ok(chat) => Json(crate::responses::response_from_chat(&chat)).into_response(),
5860 Err(e) => e.into_response(),
5861 };
5862 }
5863 }
5864 }
5865 observe_passthrough_openai_path(
5866 state,
5867 headers,
5868 body,
5869 session_header,
5870 tenant,
5871 crate::upstream::OPENAI_RESPONSES_PATH,
5872 )
5873 .await
5874}
5875
5876fn responses_sse_error(e: &ProxyError) -> String {
5879 crate::responses::sse_failed(&format!("resp_{}", Uuid::now_v7()), &e.client_message())
5880}
5881
5882fn parse_gemini_query(raw: Option<&str>) -> (Option<String>, String) {
5901 let Some(q) = raw.filter(|s| !s.is_empty()) else {
5902 return (None, String::new());
5903 };
5904 let mut key_val: Option<String> = None;
5905 let scrubbed: Vec<&str> = q
5906 .split('&')
5907 .filter(|pair| {
5908 if let Some(v) = pair.strip_prefix("key=") {
5909 if !v.is_empty() {
5910 key_val = Some(percent_decode_ascii(v));
5913 }
5914 false } else {
5916 !pair.is_empty()
5917 }
5918 })
5919 .collect();
5920 (key_val, scrubbed.join("&"))
5921}
5922
5923fn percent_decode_ascii(s: &str) -> String {
5927 let bytes = s.as_bytes();
5928 let mut out: Vec<u8> = Vec::with_capacity(bytes.len());
5929 let mut i = 0;
5930 while i < bytes.len() {
5931 if bytes[i] == b'%'
5932 && i + 2 < bytes.len()
5933 && let (Some(hi), Some(lo)) = (hex_val(bytes[i + 1]), hex_val(bytes[i + 2]))
5934 {
5935 out.push((hi << 4) | lo);
5936 i += 3;
5937 continue;
5938 }
5939 out.push(bytes[i]);
5940 i += 1;
5941 }
5942 String::from_utf8(out).unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned())
5943}
5944
5945fn hex_val(b: u8) -> Option<u8> {
5946 match b {
5947 b'0'..=b'9' => Some(b - b'0'),
5948 b'a'..=b'f' => Some(b - b'a' + 10),
5949 b'A'..=b'F' => Some(b - b'A' + 10),
5950 _ => None,
5951 }
5952}
5953
5954async fn gemini_generate_content(
5965 State(state): State<AppState>,
5966 Extension(TenantId(tenant)): Extension<TenantId>,
5967 axum::extract::Path(model_action): axum::extract::Path<String>,
5968 axum::extract::RawQuery(raw_query): axum::extract::RawQuery,
5969 headers: HeaderMap,
5970 body: Bytes,
5971) -> Response {
5972 let (key_from_query, scrubbed_query) = parse_gemini_query(raw_query.as_deref());
5975 let headers: HeaderMap = if let (Some(k), None) = (
5976 &key_from_query,
5977 headers.get("x-goog-api-key").filter(|v| !v.is_empty()),
5978 ) {
5979 match axum::http::HeaderValue::from_str(k) {
5980 Ok(v) => {
5981 let mut h = headers;
5982 h.insert("x-goog-api-key", v);
5983 h
5984 }
5985 Err(_) => headers,
5986 }
5987 } else {
5988 headers
5989 };
5990
5991 let (model, action) = model_action
5994 .rsplit_once(':')
5995 .unwrap_or((model_action.as_str(), ""));
5996 let model_owned = model.to_owned();
5998
5999 if !matches!(action, "generateContent" | "streamGenerateContent") {
6003 let path = if scrubbed_query.is_empty() {
6004 format!("/v1beta/models/{model_action}")
6005 } else {
6006 format!("/v1beta/models/{model_action}?{scrubbed_query}")
6007 };
6008 let result = crate::upstream::forward_gemini_path(
6009 &state.http,
6010 &state.config.upstream_gemini,
6011 &path,
6012 &headers,
6013 body,
6014 )
6015 .await;
6016 return match result {
6017 Ok((status, resp_headers, resp_body)) => {
6018 (status, resp_headers, resp_body).into_response()
6019 }
6020 Err(err) => err.into_response(),
6021 };
6022 }
6023
6024 let is_streaming = action == "streamGenerateContent";
6025
6026 let mut session_header = header_session(&headers, &state.config.prompt_salt);
6027
6028 let Ok(parsed) = serde_json::from_slice::<Value>(&body) else {
6029 return observe_passthrough_gemini(
6030 state,
6031 headers,
6032 body,
6033 session_header,
6034 tenant,
6035 model_action,
6036 is_streaming,
6037 scrubbed_query,
6038 Some(model_owned),
6039 )
6040 .await;
6041 };
6042
6043 if crate::gemini::has_untranslatable_content(&parsed) {
6047 return observe_passthrough_gemini(
6048 state,
6049 headers,
6050 body,
6051 session_header,
6052 tenant,
6053 model_action,
6054 is_streaming,
6055 scrubbed_query,
6056 Some(model_owned),
6057 )
6058 .await;
6059 }
6060
6061 let chat_body = Bytes::from(crate::gemini::request_to_chat(&parsed, model).to_string());
6063
6064 if let Some(routing) = state.config.routing.as_ref() {
6065 let features = extract_openai_features(&headers, &chat_body);
6066 if let Some(route) = routing
6067 .route_for(&features)
6068 .filter(|r| r.mode == Mode::Enforce && !r.ladder.is_empty())
6069 {
6070 session_header = session_header
6076 .take()
6077 .or_else(|| body_session_bytes(&chat_body, &state.config.prompt_salt));
6078 let route_ix = routing
6079 .routes
6080 .iter()
6081 .position(|r| std::ptr::eq(r, route))
6082 .unwrap_or(0);
6083 let route = route.clone();
6084 let routing_mode = resolve_mode(&headers, &route, &state.config);
6085 if routing_mode == RoutingMode::Observe {
6087 return observe_passthrough_gemini(
6088 state,
6089 headers,
6090 body,
6091 session_header,
6092 tenant,
6093 model_action,
6094 is_streaming,
6095 scrubbed_query,
6096 Some(model_owned.clone()),
6097 )
6098 .await;
6099 }
6100 if state
6103 .guardrails
6104 .is_demoted(&tenant, route_ix, jiff::Timestamp::now().as_second())
6105 {
6106 return observe_passthrough_gemini(
6107 state,
6108 headers,
6109 body,
6110 session_header,
6111 tenant,
6112 model_action,
6113 is_streaming,
6114 scrubbed_query,
6115 Some(model_owned.clone()),
6116 )
6117 .await;
6118 }
6119 if let Some(rollout) = route.rollout.as_ref() {
6121 let key_value = match rollout.key {
6122 firstpass_core::RolloutKey::Session => session_header
6123 .clone()
6124 .unwrap_or_else(|| firstpass_core::rollout::request_identity(&body)),
6125 firstpass_core::RolloutKey::Request => {
6126 firstpass_core::rollout::request_identity(&body)
6127 }
6128 firstpass_core::RolloutKey::Tenant => tenant.to_string(),
6129 };
6130 let decision =
6131 firstpass_core::rollout::decide(&state.config.prompt_salt, rollout, &key_value);
6132 if !decision.enforced {
6133 return observe_passthrough_gemini(
6137 state,
6138 headers,
6139 body,
6140 session_header,
6141 tenant,
6142 model_action,
6143 is_streaming,
6144 scrubbed_query,
6145 Some(model_owned.clone()),
6146 )
6147 .await;
6148 }
6149 }
6150 if enforce_can_handle_route(
6151 &features,
6152 &chat_body,
6153 routing.escalation.enforce_structured,
6154 &route,
6155 &state.providers,
6156 Dialect::Openai,
6157 ) {
6158 if is_streaming {
6159 let (tx, rx) = tokio::sync::oneshot::channel::<Result<Value, ProxyError>>();
6160 tokio::spawn(async move {
6161 let out = enforce_pipeline_gemini_as(
6162 &state,
6163 &headers,
6164 &chat_body,
6165 &body,
6166 "gemini.generate_content",
6167 features,
6168 &route,
6169 route_ix,
6170 session_header,
6171 tenant,
6172 routing_mode,
6173 )
6174 .await;
6175 let _ = tx.send(out);
6176 });
6177 return sse_keepalive_response(
6178 rx,
6179 Box::new(|chat| {
6180 crate::gemini::sse_from_response(&crate::gemini::response_from_chat(
6181 chat,
6182 ))
6183 }),
6184 gemini_sse_error,
6185 );
6186 }
6187 return match enforce_pipeline_gemini_as(
6188 &state,
6189 &headers,
6190 &chat_body,
6191 &body,
6192 "gemini.generate_content",
6193 features,
6194 &route,
6195 route_ix,
6196 session_header,
6197 tenant,
6198 routing_mode,
6199 )
6200 .await
6201 {
6202 Ok(chat) => Json(crate::gemini::response_from_chat(&chat)).into_response(),
6203 Err(e) => e.into_response(),
6204 };
6205 }
6206 }
6207 }
6208
6209 observe_passthrough_gemini(
6210 state,
6211 headers,
6212 body,
6213 session_header,
6214 tenant,
6215 model_action,
6216 is_streaming,
6217 scrubbed_query,
6218 Some(model_owned),
6219 )
6220 .await
6221}
6222
6223#[allow(clippy::too_many_arguments)]
6231async fn observe_passthrough_gemini(
6232 state: AppState,
6233 headers: HeaderMap,
6234 body: Bytes,
6235 session_header: Option<String>,
6236 tenant: String,
6237 model_action: String,
6238 is_streaming: bool,
6239 scrubbed_query: String,
6240 model_hint: Option<String>,
6241) -> Response {
6242 let path = if scrubbed_query.is_empty() {
6243 format!("/v1beta/models/{model_action}")
6244 } else {
6245 format!("/v1beta/models/{model_action}?{scrubbed_query}")
6246 };
6247 if is_streaming {
6248 let start = Instant::now();
6249 let result = crate::upstream::forward_gemini_streaming_path(
6250 &state.http,
6251 &state.config.upstream_gemini,
6252 &path,
6253 &headers,
6254 body.clone(),
6255 )
6256 .await;
6257 let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
6258 match result {
6259 Ok((status, resp_headers, response)) => {
6260 let on_done = stream_trace_callback(
6261 &state,
6262 body,
6263 latency_ms,
6264 session_header,
6265 tenant,
6266 "google", status.is_success(),
6268 "gemini.generate_content",
6269 model_hint,
6270 );
6271 let tap = SseTap::new(response.bytes_stream(), on_done);
6272 let stream_body = Body::from_stream(tap);
6273 return (status, resp_headers, stream_body).into_response();
6274 }
6275 Err(err) => {
6276 spawn_trace(
6277 &state,
6278 body,
6279 None,
6280 latency_ms,
6281 session_header,
6282 tenant,
6283 "google", "gemini.generate_content",
6285 model_hint,
6286 );
6287 return err.into_response();
6288 }
6289 }
6290 }
6291 let start = Instant::now();
6292 let result = crate::upstream::forward_gemini_path(
6293 &state.http,
6294 &state.config.upstream_gemini,
6295 &path,
6296 &headers,
6297 body.clone(),
6298 )
6299 .await;
6300 let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
6301 match result {
6302 Ok((status, resp_headers, resp_body)) => {
6303 let for_trace = status.is_success().then_some(resp_body.clone());
6304 spawn_trace(
6305 &state,
6306 body,
6307 for_trace,
6308 latency_ms,
6309 session_header,
6310 tenant,
6311 "google", "gemini.generate_content",
6313 model_hint,
6314 );
6315 (status, resp_headers, resp_body).into_response()
6316 }
6317 Err(err) => {
6318 spawn_trace(
6319 &state,
6320 body,
6321 None,
6322 latency_ms,
6323 session_header,
6324 tenant,
6325 "google", "gemini.generate_content",
6327 model_hint,
6328 );
6329 err.into_response()
6330 }
6331 }
6332}
6333
6334#[allow(clippy::too_many_arguments)]
6341async fn enforce_pipeline_gemini_as(
6342 state: &AppState,
6343 headers: &HeaderMap,
6344 chat_body: &Bytes,
6345 receipt_body: &Bytes,
6346 api: &str,
6347 features: Features,
6348 route: &Route,
6349 route_ix: usize,
6350 session_header: Option<String>,
6351 tenant: String,
6352 routing_mode: RoutingMode,
6353) -> Result<Value, ProxyError> {
6354 let providers = &state.providers;
6355 let all_openai_of = |ladder: &[String]| {
6356 ladder.iter().all(|rung| {
6357 let pid = rung.split('/').next().unwrap_or_default();
6358 providers
6359 .get(pid)
6360 .is_some_and(|p| p.carries_structured_verbatim(Dialect::Openai))
6361 })
6362 };
6363 let all_openai = all_openai_of(&route.ladder);
6364 let allow_challenger = route
6365 .challenger
6366 .as_ref()
6367 .is_none_or(|c| all_openai_of(&c.ladder) == all_openai);
6368 let Some(base_request) = parse_openai_request(chat_body, all_openai) else {
6369 return Err(ProxyError::BadRequest(
6370 "Gemini request could not be translated to a valid Chat Completions request".to_owned(),
6371 ));
6372 };
6373 let auth = Auth::from_headers(headers, Dialect::Gemini);
6376 let resp = enforce_pipeline_inner(
6377 state,
6378 receipt_body,
6379 base_request,
6380 auth,
6381 features,
6382 route,
6383 session_header,
6384 tenant,
6385 api,
6386 routing_mode,
6387 route_ix,
6388 allow_challenger,
6389 )
6390 .await?;
6391 Ok(openai_response_json(&resp))
6392}
6393
6394fn gemini_sse_error(e: &ProxyError) -> String {
6397 crate::gemini::sse_error_response(&e.client_message())
6398}
6399
6400#[cfg(test)]
6401mod tests {
6402 use bytes::Bytes;
6403 use serde_json::json;
6404
6405 fn sess(h: &[(&str, &str)], body: serde_json::Value) -> Option<String> {
6406 let mut headers = HeaderMap::new();
6407 for (k, v) in h {
6408 headers.insert(
6409 axum::http::HeaderName::from_bytes(k.as_bytes()).unwrap(),
6410 v.parse().unwrap(),
6411 );
6412 }
6413 header_session(&headers, "salt")
6415 .or_else(|| body_session_bytes(body.to_string().as_bytes(), "salt"))
6416 }
6417
6418 #[test]
6419 fn caller_session_priority_order() {
6420 let body = json!({
6421 "metadata": {"user_id": "u_session_abc"}, "prompt_cache_key": "k",
6422 "tools": [{"name": "t"}], "messages": [{"role": "user", "content": "hi"}],
6423 });
6424 let all = [("x-firstpass-session", "explicit"), ("session_id", "ag")];
6425 assert_eq!(sess(&all, body.clone()).as_deref(), Some("explicit"));
6426 let agent = sess(&all[1..], body.clone()).unwrap();
6427 assert!(agent.starts_with("agent:"));
6428 let meta = sess(&[], body.clone()).unwrap();
6429 assert!(meta.starts_with("meta:"));
6430 let mut b = body.clone();
6431 b["metadata"] = json!({"user_id": "plain-user"});
6432 let pck = sess(&[], b.clone()).unwrap();
6433 assert!(pck.starts_with("pck:"));
6434 b.as_object_mut().unwrap().remove("prompt_cache_key");
6435 assert!(sess(&[], b).unwrap().starts_with("conv:"));
6436 for (id, raw) in [(&agent, "ag"), (&meta, "u_session_abc"), (&pck, "k")] {
6438 assert_eq!(id.len(), id.find(':').unwrap() + 1 + 16, "{id}");
6439 assert!(
6440 !id[id.find(':').unwrap() + 1..].contains(raw),
6441 "{id} leaks {raw}"
6442 );
6443 }
6444 let plain =
6446 json!({"prompt_cache_key": "k", "messages": [{"role": "user", "content": "hi"}]});
6447 assert_eq!(sess(&[], plain), None);
6448 }
6449
6450 #[test]
6451 fn caller_session_caps_and_trims_header_values() {
6452 let long = "a".repeat(300);
6453 assert_eq!(
6454 sess(&[("conversation_id", &long)], json!({}))
6455 .unwrap()
6456 .len(),
6457 "agent:".len() + 16
6458 );
6459 let v = sess(&[("x-firstpass-session", &long)], json!({})).unwrap();
6460 assert_eq!(v.len(), 256, "explicit header is verbatim but capped");
6461 assert_eq!(cap_session(&"é".repeat(300)).len(), 256); assert_eq!(sess(&[("session_id", " ")], json!({})), None);
6463 }
6464
6465 #[test]
6466 fn blank_explicit_header_is_absent() {
6467 let body =
6468 json!({"tools": [{"name": "t"}], "messages": [{"role": "user", "content": "x"}]});
6469 for blank in ["", " "] {
6470 let id = sess(&[("x-firstpass-session", blank)], body.clone()).unwrap();
6471 assert!(
6472 id.starts_with("conv:"),
6473 "blank header must fall through, got {id:?}"
6474 );
6475 }
6476 assert_eq!(
6477 sess(&[("x-firstpass-session", " a ")], json!({})).as_deref(),
6478 Some("a")
6479 );
6480 }
6481
6482 #[test]
6483 fn responses_string_input_turn_one_matches_array_turn_two() {
6484 let tools = json!([{"type": "function", "name": "shell"}]);
6485 let t1 = json!({"instructions": "i", "tools": tools, "input": "list files"});
6486 let t2 = json!({"instructions": "i", "tools": tools, "input": [
6487 {"role": "user", "content": [{"type": "input_text", "text": "list files"}]},
6488 {"type": "function_call_output", "call_id": "c", "output": "a b"}]});
6489 let a = sess(&[], t1).unwrap();
6490 assert!(a.starts_with("conv:"));
6491 assert_eq!(Some(a), sess(&[], t2));
6492 let p = json!({"input": "more", "previous_response_id": "resp_1"});
6494 assert!(sess(&[], p).unwrap().starts_with("prev:"));
6495 }
6496
6497 #[test]
6498 fn agent_turn_one_and_two_share_a_conv_id_and_tool_names_split_it() {
6499 let u1 = json!({"role": "user", "content": "fix it"});
6500 let turn1 =
6501 json!({"system": "s", "tools": [{"name": "bash"}, {"name": "edit"}], "messages": [u1]});
6502 let turn2 = json!({"system": "s", "tools": [{"name": "edit"}, {"name": "bash"}], "messages": [
6503 u1, {"role": "assistant", "content": "ok"}, {"role": "user", "content": "go on"}]});
6504 let a = sess(&[], turn1.clone()).unwrap();
6505 assert_eq!(
6506 Some(a.clone()),
6507 sess(&[], turn2),
6508 "turn 1 and 2 group, tool order irrelevant"
6509 );
6510 let mut other = turn1;
6511 other["tools"] = json!([{"name": "bash"}, {"name": "grep"}]);
6512 assert_ne!(
6513 Some(a),
6514 sess(&[], other),
6515 "different toolset, different session"
6516 );
6517 }
6518
6519 #[tokio::test]
6522 async fn observe_only_derives_session_from_headers_never_the_body() {
6523 let app = axum::Router::new().route(
6524 "/v1/messages",
6525 axum::routing::post(|| async { (axum::http::StatusCode::OK, "{}") }),
6526 );
6527 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
6528 let base_url = format!("http://{}", listener.local_addr().unwrap());
6529 tokio::spawn(async move {
6530 let _ = axum::serve(listener, app).await;
6531 });
6532 let config = ProxyConfig::from_lookup(|k| match k {
6533 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some(base_url.clone()),
6534 _ => None,
6535 })
6536 .unwrap();
6537 let (traces, mut rx) = mpsc::channel(4);
6538 let state = AppState {
6539 config: Arc::new(config),
6540 http: reqwest::Client::new(),
6541 providers: ProviderRegistry::from_map(HashMap::new()),
6542 gate_health: Arc::new(GateHealthRegistry::new()),
6543 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
6544 paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
6545 experiments: Arc::default(),
6546 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
6547 traces,
6548 adaptive: None,
6549 eprocess: None,
6550 bandit: None,
6551 promoter: None,
6552 verified_cache: None,
6553 predictor: None,
6554 prior: None,
6555 tenant_rate_limiter: None,
6556 spill: None,
6557 };
6558 let body = Bytes::from_static(
6559 br#"{"model":"claude-haiku-4-5","tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
6560 );
6561 let mut got = Vec::new();
6562 for hdr in [None, Some("abc")] {
6563 let mut headers = HeaderMap::new();
6564 if let Some(h) = hdr {
6565 headers.insert("session_id", h.parse().unwrap());
6566 }
6567 let _ = messages(
6568 State(state.clone()),
6569 Extension(TenantId("default".to_owned())),
6570 headers,
6571 body.clone(),
6572 )
6573 .await;
6574 got.push(rx.recv().await.unwrap());
6575 }
6576 assert!(
6577 !got[0].session_id.starts_with("conv:"),
6578 "{}",
6579 got[0].session_id
6580 );
6581 assert!(
6582 got[1].session_id.starts_with("agent:"),
6583 "{}",
6584 got[1].session_id
6585 );
6586 }
6587
6588 #[test]
6589 fn caller_session_anthropic_conversation_is_stable_across_turns() {
6590 let u1 = json!({"role": "user", "content": "fix the bug"});
6591 let t1 = json!({"system": "sys", "tools": [{"name": "t"}], "messages": [u1]});
6592 let t2 = json!({"system": "sys", "tools": [{"name": "t"}], "messages": [
6593 u1, {"role": "assistant", "content": "ok"}, {"role": "user", "content": "more"}]});
6594 let other = json!({"system": "sys", "tools": [{"name": "t"}],
6595 "messages": [{"role": "user", "content": "different task"}]});
6596 let a = sess(&[], t1).unwrap();
6597 assert!(a.starts_with("conv:") && a.len() == 5 + 16);
6598 assert_eq!(Some(a.clone()), sess(&[], t2));
6599 assert_ne!(Some(a), sess(&[], other));
6600 let plain =
6602 json!({"metadata": {"user_id": "bob"}, "messages": [{"role": "user", "content": "x"}]});
6603 assert_eq!(sess(&[], plain), None);
6604 assert_eq!(sess(&[], json!("not an object")), None);
6605 }
6606
6607 #[test]
6608 fn caller_session_openai_chat_and_responses_shapes() {
6609 let chat = |tail: &str| {
6610 json!({"messages": [
6611 {"role": "system", "content": "s"}, {"role": "user", "content": "q"},
6612 {"role": "assistant", "content": "a"}, {"role": "user", "content": tail}]})
6613 };
6614 let c = sess(&[], chat("one")).unwrap();
6615 assert_eq!(Some(c.clone()), sess(&[], chat("two")));
6616 let diff = json!({"messages": [
6617 {"role": "system", "content": "s"}, {"role": "user", "content": "other"},
6618 {"role": "assistant", "content": "a"}]});
6619 assert_ne!(Some(c), sess(&[], diff));
6620 assert_eq!(
6621 sess(&[], json!({"messages": [{"role": "user", "content": "q"}]})),
6622 None
6623 );
6624
6625 let resp = |items: serde_json::Value| json!({"instructions": "i", "input": items});
6626 let first = json!({"role": "user", "content": "q"});
6627 let r1 = sess(
6628 &[],
6629 resp(json!([first, {"role": "user", "content": "more"}])),
6630 )
6631 .unwrap();
6632 assert!(r1.starts_with("conv:"));
6633 assert_eq!(
6634 Some(r1.clone()),
6635 sess(
6636 &[],
6637 resp(
6638 json!([first, {"role": "assistant", "content": "x"}, {"role": "user", "content": "y"}])
6639 )
6640 )
6641 );
6642 assert_ne!(
6643 Some(r1),
6644 sess(&[], resp(json!([{"role": "user", "content": "z"}, first])))
6645 );
6646 assert_eq!(sess(&[], resp(json!("hello"))), None);
6648 assert!(
6649 sess(
6650 &[],
6651 json!({"input": "hello", "tools": [{"type": "function"}]})
6652 )
6653 .is_some()
6654 );
6655 }
6656
6657 #[test]
6660 fn risk_certificate_follows_adr_0011_precedence() {
6661 use super::{ThresholdSource, risk_certificate};
6662 let c = firstpass_core::eprocess::Certification {
6663 threshold: 0.6,
6664 e_value: 30.0,
6665 certified_at_round: 200,
6666 };
6667 let r = risk_certificate(Some((c, 0.1, 0.05)), Some((0.4, 0.1)), Some(0.9)).unwrap();
6668 assert_eq!(r.source, ThresholdSource::Eprocess);
6669 assert_eq!(
6670 (r.threshold, r.e_value, r.delta, r.certified_at_round),
6671 (0.6, Some(30.0), Some(0.05), Some(200))
6672 );
6673
6674 let r = risk_certificate(None, Some((0.4, 0.1)), Some(0.9)).unwrap();
6675 assert_eq!((r.source, r.threshold), (ThresholdSource::Aci, 0.4));
6676 assert_eq!((r.alpha, r.e_value), (Some(0.1), None));
6677
6678 let r = risk_certificate(None, None, Some(0.9)).unwrap();
6679 assert_eq!(
6680 (r.source, r.threshold, r.alpha),
6681 (ThresholdSource::Fixed, 0.9, None)
6682 );
6683
6684 assert!(risk_certificate(None, None, None).is_none());
6685 }
6686
6687 use super::*;
6688
6689 #[test]
6693 fn anthropic_tool_errors_are_extracted() {
6694 let body = br#"{"messages":[
6695 {"role":"assistant","content":[{"type":"tool_use","name":"bash","input":{"cmd":"a"}}]},
6696 {"role":"user","content":[{"type":"tool_result","is_error":true,"content":"boom"}]},
6697 {"role":"assistant","content":[{"type":"tool_use","name":"bash","input":{"cmd":"b"}}]},
6698 {"role":"user","content":[{"type":"tool_result","is_error":true,"content":"boom"}]}
6699 ]}"#;
6700 let s = trajectory_signals(body);
6701 assert_eq!(
6702 s.tool_errors, 2,
6703 "both failing tool_results must be counted"
6704 );
6705 assert_eq!(s.tool_results, 2);
6706 assert_eq!(s.assistant_turns, 2);
6707 assert!(
6708 DifficultyHint::score(s) >= DifficultyHint::Medium,
6709 "an all-failing window must not read as an easy session"
6710 );
6711 }
6712
6713 #[test]
6718 fn openai_tool_errors_are_extracted_from_a_different_shape() {
6719 let body = br#"{"messages":[
6720 {"role":"assistant","tool_calls":[{"function":{"name":"run","arguments":"{\"x\":1}"}}]},
6721 {"role":"tool","content":"Error: command not found"},
6722 {"role":"assistant","tool_calls":[{"function":{"name":"run","arguments":"{\"x\":2}"}}]},
6723 {"role":"tool","content":"Traceback (most recent call last): ..."}
6724 ]}"#;
6725 let s = trajectory_signals(body);
6726 assert_eq!(s.tool_results, 2, "role:tool messages are tool results");
6727 assert_eq!(
6728 s.tool_errors, 2,
6729 "both error-shaped tool outputs must count"
6730 );
6731 assert_eq!(s.assistant_turns, 2);
6732 }
6733
6734 #[test]
6741 fn a_flood_of_leading_whitespace_does_not_defeat_the_bound() {
6742 let flood = format!("{}Error: boom", " ".repeat(4_000_000));
6743 let small = format!("{}Error: boom", " ".repeat(400_000));
6744 let m_big = serde_json::json!({ "role": "tool", "content": flood });
6745 let m_small = serde_json::json!({ "role": "tool", "content": small });
6746
6747 let t = |m: &serde_json::Value| {
6748 let start = std::time::Instant::now();
6749 for _ in 0..20 {
6750 std::hint::black_box(openai_tool_content_looks_like_error(m));
6751 }
6752 start.elapsed()
6753 };
6754 let _warm = t(&m_small);
6755 let small_t = t(&m_small);
6756 let big_t = t(&m_big);
6757 assert!(
6758 big_t < small_t * 5 + std::time::Duration::from_millis(5),
6759 "10x the leading whitespace took {big_t:?} vs {small_t:?} — trimming is scanning the \
6760 whole payload before the bound applies"
6761 );
6762
6763 let indented = serde_json::json!({ "role": "tool", "content": " Error: boom" });
6765 assert!(
6766 openai_tool_content_looks_like_error(&indented),
6767 "ordinary leading whitespace must still be trimmed"
6768 );
6769 }
6770
6771 #[test]
6780 fn a_long_real_conversation_can_actually_reach_the_deep_signal() {
6781 let mut msgs = Vec::new();
6783 for i in 0..40 {
6784 msgs.push(format!(
6785 r#"{{"role":"assistant","content":[{{"type":"tool_use","name":"b","input":{{"i":{i}}}}}]}}"#
6786 ));
6787 msgs.push(
6788 r#"{"role":"user","content":[{"type":"tool_result","is_error":true}]}"#.to_owned(),
6789 );
6790 }
6791 let body = format!(r#"{{"messages":[{}]}}"#, msgs.join(","));
6792 let s = trajectory_signals(body.as_bytes());
6793 assert!(
6794 s.assistant_turns >= 8,
6795 "a 40-turn failing conversation must register as deep, got {} assistant turns — the \
6796 window is counting turns it cannot see",
6797 s.assistant_turns
6798 );
6799 assert_eq!(
6800 DifficultyHint::score(s),
6801 DifficultyHint::High,
6802 "the hardest possible conversation must reach the top bucket, or High is dead code"
6803 );
6804 }
6805
6806 #[test]
6813 fn openai_array_form_tool_content_is_read() {
6814 let arr = serde_json::json!({
6815 "role": "tool",
6816 "content": [{"type": "text", "text": "Error: command not found"}]
6817 });
6818 assert!(
6819 openai_tool_content_looks_like_error(&arr),
6820 "an error in array-form content must be detected"
6821 );
6822
6823 let clean = serde_json::json!({
6824 "role": "tool",
6825 "content": [{"type": "text", "text": "All 42 tests passed"}]
6826 });
6827 assert!(
6828 !openai_tool_content_looks_like_error(&clean),
6829 "clean array-form content must not be a failure"
6830 );
6831
6832 for weird in [
6834 serde_json::json!({"role": "tool", "content": []}),
6835 serde_json::json!({"role": "tool", "content": [{"type": "image"}]}),
6836 serde_json::json!({"role": "tool", "content": [null, 42]}),
6837 ] {
6838 assert!(!openai_tool_content_looks_like_error(&weird));
6839 }
6840
6841 let body = br#"{"messages":[
6843 {"role":"tool","content":[{"type":"text","text":"Error: boom"}]},
6844 {"role":"tool","content":[{"type":"text","text":"Error: boom again"}]},
6845 {"role":"tool","content":[{"type":"text","text":"Error: still broken"}]}
6846 ]}"#;
6847 let s = trajectory_signals(body);
6848 assert_eq!(s.tool_errors, 3, "array-form errors must reach the signals");
6849 assert!(DifficultyHint::score(s) >= DifficultyHint::Medium);
6850 }
6851
6852 #[test]
6858 fn tool_output_merely_mentioning_errors_is_not_counted_as_failure() {
6859 let body = br#"{"messages":[
6860 {"role":"tool","content":"0 errors, 0 warnings"},
6861 {"role":"tool","content":"All 42 tests passed (no failures)"},
6862 {"role":"tool","content":"lint: checked 12 files for errors"}
6863 ]}"#;
6864 let s = trajectory_signals(body);
6865 assert_eq!(s.tool_results, 3);
6866 assert_eq!(
6867 s.tool_errors, 0,
6868 "clean output that mentions the word 'error' must not be scored as a failure"
6869 );
6870 }
6871
6872 #[test]
6882 fn a_multibyte_character_does_not_defeat_the_prefix_bound() {
6883 let padded = format!("{}é{}", "x".repeat(63), "y".repeat(200_000));
6885 let msg = serde_json::json!({ "role": "tool", "content": padded });
6886 assert!(
6887 !openai_tool_content_looks_like_error(&msg),
6888 "a huge non-error payload must not be classified as a failure"
6889 );
6890
6891 let big = format!("{}é{}", "x".repeat(63), "y".repeat(2_000_000));
6899 let small = format!("{}é{}", "x".repeat(63), "y".repeat(200_000));
6900 let m_big = serde_json::json!({ "role": "tool", "content": big });
6901 let m_small = serde_json::json!({ "role": "tool", "content": small });
6902
6903 let t = |m: &serde_json::Value| {
6904 let start = std::time::Instant::now();
6905 for _ in 0..20 {
6906 std::hint::black_box(openai_tool_content_looks_like_error(m));
6907 }
6908 start.elapsed()
6909 };
6910 let _warm = t(&m_small);
6911 let small_t = t(&m_small);
6912 let big_t = t(&m_big);
6913 assert!(
6914 big_t < small_t * 5 + std::time::Duration::from_millis(5),
6915 "scanning a 10x larger payload took {big_t:?} vs {small_t:?} — the prefix bound is not \
6916 holding, so work scales with attacker-controlled input size"
6917 );
6918
6919 let err = format!("Error: boom 💥{}", "z".repeat(200_000));
6921 let msg2 = serde_json::json!({ "role": "tool", "content": err });
6922 assert!(
6923 openai_tool_content_looks_like_error(&msg2),
6924 "bounding the scan must not cost the detection it exists to perform"
6925 );
6926
6927 let late = format!("{}Error: too late", "w".repeat(200));
6930 let msg3 = serde_json::json!({ "role": "tool", "content": late });
6931 assert!(
6932 !openai_tool_content_looks_like_error(&msg3),
6933 "the scan must stay bounded to the head of the message"
6934 );
6935 }
6936
6937 #[test]
6940 fn identical_repeated_tool_calls_are_detected() {
6941 let body = br#"{"messages":[
6942 {"role":"assistant","content":[{"type":"tool_use","name":"ls","input":{"p":"/x"}}]},
6943 {"role":"assistant","content":[{"type":"tool_use","name":"ls","input":{"p":"/x"}}]},
6944 {"role":"assistant","content":[{"type":"tool_use","name":"ls","input":{"p":"/x"}}]},
6945 {"role":"user","content":[{"type":"tool_result","content":"ok"}]}
6946 ]}"#;
6947 let s = trajectory_signals(body);
6948 assert_eq!(
6949 s.repeated_tool_calls, 2,
6950 "three identical calls are two repeats"
6951 );
6952 let distinct = br#"{"messages":[
6954 {"role":"assistant","content":[{"type":"tool_use","name":"ls","input":{"p":"/x"}}]},
6955 {"role":"assistant","content":[{"type":"tool_use","name":"ls","input":{"p":"/y"}}]}
6956 ]}"#;
6957 assert_eq!(trajectory_signals(distinct).repeated_tool_calls, 0);
6958 }
6959
6960 #[test]
6963 fn malformed_bodies_yield_no_signal_and_never_panic() {
6964 for body in [
6965 &b""[..],
6966 b"not json at all",
6967 b"{}",
6968 br#"{"messages":"not-an-array"}"#,
6969 br#"{"messages":[{"role":"assistant","content":"plain text"}]}"#,
6970 br#"{"messages":[null,42,"x"]}"#,
6971 ] {
6972 let s = trajectory_signals(body);
6973 assert_eq!(
6974 DifficultyHint::score(s),
6975 DifficultyHint::None,
6976 "a body with no usable trajectory must score None, not a difficulty level"
6977 );
6978 }
6979
6980 let quiet_success = br#"{"messages":[{"content":[{"type":"tool_result"}]}]}"#;
6986 let s = trajectory_signals(quiet_success);
6987 assert_eq!(s.tool_results, 1);
6988 assert_eq!(s.tool_errors, 0);
6989 assert_eq!(
6990 DifficultyHint::score(s),
6991 DifficultyHint::Low,
6992 "a tool result that reported no error is evidence of health, not absence of evidence"
6993 );
6994 }
6995
6996 #[test]
7000 fn only_the_recent_window_counts_so_old_failures_do_not_ratchet() {
7001 let mut msgs = String::from("{\"messages\":[");
7002 for _ in 0..30 {
7004 msgs.push_str(r#"{"role":"user","content":[{"type":"tool_result","is_error":true}]},"#);
7005 }
7006 for i in 0..12 {
7008 msgs.push_str(&format!(
7009 r#"{{"role":"user","content":[{{"type":"tool_result","is_error":false,"content":"ok{i}"}}]}}{}"#,
7010 if i == 11 { "" } else { "," }
7011 ));
7012 }
7013 msgs.push_str("]}");
7014 let s = trajectory_signals(msgs.as_bytes());
7015 assert_eq!(
7016 s.tool_errors, 0,
7017 "failures outside the window must not count; a recovered session is not a hard one"
7018 );
7019 assert!(s.tool_results > 0, "the recent clean results must be seen");
7020 }
7021
7022 #[test]
7025 fn the_hint_reaches_the_feature_vector_for_both_dialects() {
7026 let hard = br#"{"messages":[
7027 {"role":"assistant","content":[{"type":"tool_use","name":"b","input":{"c":1}}]},
7028 {"role":"user","content":[{"type":"tool_result","is_error":true}]},
7029 {"role":"assistant","content":[{"type":"tool_use","name":"b","input":{"c":1}}]},
7030 {"role":"user","content":[{"type":"tool_result","is_error":true}]},
7031 {"role":"assistant","content":[{"type":"tool_use","name":"b","input":{"c":1}}]},
7032 {"role":"user","content":[{"type":"tool_result","is_error":true}]}
7033 ]}"#;
7034 let h = HeaderMap::new();
7035 assert!(
7036 extract_features(&h, hard).difficulty_hint > 0,
7037 "a visibly struggling Anthropic conversation must carry a non-zero hint"
7038 );
7039 assert!(
7040 extract_openai_features(&h, hard).difficulty_hint > 0,
7041 "the OpenAI extractor must populate the hint too, not silently leave it zero"
7042 );
7043 let plain = br#"{"messages":[{"role":"user","content":"hello"}]}"#;
7044 assert_eq!(
7045 extract_features(&h, plain).difficulty_hint,
7046 0,
7047 "a single-shot request must be unchanged by this feature"
7048 );
7049 }
7050
7051 fn test_config() -> ProxyConfig {
7052 ProxyConfig::from_lookup(|_| None).unwrap()
7053 }
7054
7055 #[test]
7059 fn openai_observe_receipt_uses_openai_usage_and_price() {
7060 let config = test_config();
7061 let req = Bytes::from_static(br#"{"model":"gpt-4.1-mini","messages":[]}"#);
7062 let resp = Bytes::from_static(
7063 br#"{"model":"gpt-4.1-mini","choices":[{"finish_reason":"length"}],
7064 "usage":{"prompt_tokens":1000,"completion_tokens":200,
7065 "prompt_tokens_details":{"cached_tokens":400}}}"#,
7066 );
7067 let trace = build_trace(
7068 &config,
7069 &req,
7070 &resp,
7071 7,
7072 None,
7073 "openai",
7074 "openai.chat_completions",
7075 None,
7076 );
7077 let a = &trace.attempts[0];
7078 assert_eq!(a.provider, "openai");
7079 assert_eq!(a.in_tokens, 600, "uncached prompt tokens");
7080 assert_eq!(a.cache_read_tokens, 400);
7081 assert_eq!(a.out_tokens, 200);
7082 assert!(a.truncated, "finish_reason length");
7083 let want = config
7084 .prices
7085 .cost_usd_with_cache("openai/gpt-4.1-mini", 600, 0, 400, 200)
7086 .expect("gpt-4.1-mini is priced");
7087 assert!(want > 0.0);
7088 assert!(
7089 (a.cost_usd - want).abs() < 1e-12,
7090 "{} vs {want}",
7091 a.cost_usd
7092 );
7093
7094 let mut u = SseUsage::default();
7095 let mut buf = Vec::new();
7096 let mut overflow = false;
7097 parse_sse_chunk_into(
7098 b"data: {\"model\":\"gpt-4.1-mini\",\"choices\":[],\"usage\":{\"prompt_tokens\":1000,\"completion_tokens\":200,\"prompt_tokens_details\":{\"cached_tokens\":400}}}\n",
7099 &mut buf,
7100 &mut overflow,
7101 &mut u,
7102 );
7103 assert_eq!(
7104 (u.in_tokens, u.cache_read_tokens, u.out_tokens),
7105 (600, 400, 200)
7106 );
7107 }
7108
7109 #[test]
7110 fn build_trace_maps_request_and_response_fields() {
7111 let config = test_config();
7112 let req = Bytes::from_static(
7113 br#"{"model":"claude-haiku-4-5","tools":[{"name":"a"}],"messages":[]}"#,
7114 );
7115 let resp = Bytes::from_static(
7116 br#"{"model":"claude-haiku-4-5","usage":{"input_tokens":1200,"output_tokens":300}}"#,
7117 );
7118
7119 let trace = build_trace(
7120 &config,
7121 &req,
7122 &resp,
7123 42,
7124 Some("sess-1"),
7125 "anthropic",
7126 "anthropic.messages",
7127 None,
7128 );
7129
7130 assert_eq!(trace.request.api, "anthropic.messages");
7131 assert_eq!(trace.session_id, "sess-1");
7132 assert_eq!(trace.attempts.len(), 1);
7133 let attempt = &trace.attempts[0];
7134 assert_eq!(attempt.model, "claude-haiku-4-5");
7135 assert_eq!(attempt.provider, "anthropic");
7136 assert_eq!(attempt.in_tokens, 1200);
7137 assert_eq!(attempt.out_tokens, 300);
7138 assert!(attempt.cost_usd > 0.0);
7139 assert_eq!(trace.request.features.tool_count, 1);
7140 assert!(!trace.request.features.has_images);
7141 assert_eq!(trace.final_.served_rung, Some(0));
7142 }
7143
7144 #[test]
7145 fn build_trace_falls_back_to_trace_id_session_when_header_absent() {
7146 let config = test_config();
7147 let req = Bytes::from_static(b"{}");
7148 let resp = Bytes::from_static(b"{}");
7149
7150 let trace = build_trace(
7151 &config,
7152 &req,
7153 &resp,
7154 1,
7155 None,
7156 "anthropic",
7157 "anthropic.messages",
7158 None,
7159 );
7160
7161 assert_eq!(trace.session_id, trace.trace_id.to_string());
7162 }
7163
7164 #[test]
7165 fn build_error_trace_has_no_attempts_and_served_from_error() {
7166 let config = test_config();
7167 let req = Bytes::from_static(br#"{"model":"claude-haiku-4-5"}"#);
7168
7169 let trace = build_error_trace(&config, &req, 7, None, "anthropic", "anthropic.messages");
7170
7171 assert!(trace.attempts.is_empty());
7172 assert_eq!(trace.final_.served_from, ServedFrom::Error);
7173 assert_eq!(trace.final_.served_rung, None);
7174 }
7175
7176 #[test]
7177 fn message_with_image_block_sets_has_images() {
7178 let req = br#"{"messages":[{"role":"user","content":[{"type":"image"}]}]}"#;
7179 let (_, _, has_images) = request_features(req);
7180 assert!(has_images);
7181 }
7182
7183 #[test]
7184 fn prompt_hash_never_contains_raw_prompt_text() {
7185 let hash = prompt_hash("salt", b"super secret prompt");
7186 assert!(!hash.contains("secret"));
7187 assert_eq!(hash.len(), 64);
7188 }
7189
7190 #[test]
7191 fn parse_model_request_preserves_content_verbatim_and_projects_text() {
7192 let body = br#"{"model":"m","system":"sys","max_tokens":50,
7193 "messages":[{"role":"user","content":[{"type":"text","text":"a"},{"type":"text","text":"b"}]},
7194 {"role":"assistant","content":"c"}]}"#;
7195 let req = parse_model_request(body).unwrap();
7196 assert_eq!(req.system.as_deref(), Some("sys"));
7197 assert_eq!(req.max_tokens, 50);
7198 assert_eq!(req.messages.len(), 2);
7199 assert_eq!(
7201 req.messages[0].content,
7202 serde_json::json!([{"type":"text","text":"a"},{"type":"text","text":"b"}])
7203 );
7204 assert_eq!(req.messages[1].content, Value::String("c".to_owned()));
7206 assert_eq!(req.messages[0].text_view(), "a\nb");
7208 assert_eq!(req.messages[1].text_view(), "c");
7209 }
7210
7211 #[test]
7212 fn tool_and_image_blocks_survive_the_request_round_trip() {
7213 let body = br#"{"model":"m","max_tokens":50,"messages":[
7215 {"role":"assistant","content":[{"type":"tool_use","id":"t1","name":"calc","input":{"x":1}}]},
7216 {"role":"user","content":[
7217 {"type":"tool_result","tool_use_id":"t1","content":"2"},
7218 {"type":"image","source":{"type":"base64","media_type":"image/png","data":"AA=="}}
7219 ]}]}"#;
7220 let req = parse_model_request(body).unwrap();
7221 let round_tripped = serde_json::to_value(&req.messages).unwrap();
7222 assert_eq!(
7223 round_tripped,
7224 serde_json::json!([
7225 {"role":"assistant","content":[{"type":"tool_use","id":"t1","name":"calc","input":{"x":1}}]},
7226 {"role":"user","content":[
7227 {"type":"tool_result","tool_use_id":"t1","content":"2"},
7228 {"type":"image","source":{"type":"base64","media_type":"image/png","data":"AA=="}}
7229 ]}
7230 ])
7231 );
7232 }
7233
7234 #[test]
7235 fn text_message_serializes_byte_identical_to_a_plain_string() {
7236 let m = ChatMessage::text("user", "hello");
7238 assert_eq!(
7239 serde_json::to_string(&m).unwrap(),
7240 r#"{"role":"user","content":"hello"}"#
7241 );
7242 }
7243
7244 #[test]
7245 fn parse_model_request_rejects_non_message_bodies() {
7246 assert!(parse_model_request(b"not json").is_none());
7247 assert!(parse_model_request(br#"{"no":"messages"}"#).is_none());
7248 }
7249
7250 use crate::provider::{MockProvider, ModelResponse, Provider, ProviderError, ProviderRegistry};
7253 use axum::extract::State;
7254 use std::collections::HashMap;
7255 use std::sync::Arc;
7256 use tokio::sync::mpsc;
7257
7258 fn model_resp(model: &str, text: &str) -> ModelResponse {
7259 ModelResponse {
7260 model: model.to_owned(),
7261 text: text.to_owned(),
7262 in_tokens: 1000,
7263 cache_write_tokens: 0,
7264 cache_read_tokens: 0,
7265 out_tokens: 400,
7266 raw: serde_json::Value::Null,
7267 }
7268 }
7269
7270 fn enforce_state(
7273 ladder: &[&str],
7274 gates: &[&str],
7275 outcomes: Vec<(&str, Result<ModelResponse, ProviderError>)>,
7276 ) -> (AppState, mpsc::Receiver<Trace>) {
7277 let toml = format!(
7278 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\nladder = [{}]\ngates = [{}]\n",
7279 ladder
7280 .iter()
7281 .map(|m| format!("\"{m}\""))
7282 .collect::<Vec<_>>()
7283 .join(", "),
7284 gates
7285 .iter()
7286 .map(|g| format!("\"{g}\""))
7287 .collect::<Vec<_>>()
7288 .join(", "),
7289 );
7290 let config = ProxyConfig::from_lookup(|k| match k {
7291 "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
7292 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
7293 _ => None,
7294 })
7295 .unwrap();
7296
7297 let mut outs = HashMap::new();
7298 for (model, out) in outcomes {
7299 outs.insert(model.to_owned(), out);
7300 }
7301 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
7302 map.insert(
7303 "anthropic".to_owned(),
7304 Arc::new(MockProvider::new("anthropic", outs)),
7305 );
7306 let providers = ProviderRegistry::from_map(map);
7307
7308 let (traces, rx) = mpsc::channel(64);
7309 let state = AppState {
7310 config: Arc::new(config),
7311 http: reqwest::Client::new(),
7312 providers,
7313 gate_health: Arc::new(GateHealthRegistry::new()),
7314 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
7315 paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
7316 experiments: Arc::default(),
7317 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
7318 traces,
7319 adaptive: None,
7320 eprocess: None,
7321 bandit: None,
7322 promoter: None,
7323 verified_cache: None,
7324 predictor: None,
7325 prior: None,
7326 tenant_rate_limiter: None,
7327 spill: None,
7328 };
7329 (state, rx)
7330 }
7331
7332 fn user_body() -> Bytes {
7333 Bytes::from_static(
7334 br#"{"model":"ignored","max_tokens":64,"messages":[{"role":"user","content":"hi"}]}"#,
7335 )
7336 }
7337
7338 async fn body_json(resp: Response) -> Value {
7339 let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
7340 .await
7341 .unwrap();
7342 serde_json::from_slice(&bytes).unwrap()
7343 }
7344
7345 #[tokio::test]
7346 async fn mode_profile_stamped_on_trace_when_non_balanced() {
7347 let (state, mut rx) = enforce_state(
7348 &["anthropic/claude-haiku-4-5"],
7349 &["non-empty"],
7350 vec![(
7351 "anthropic/claude-haiku-4-5",
7352 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
7353 )],
7354 );
7355 let mut headers = HeaderMap::new();
7356 headers.insert("x-firstpass-mode", "quality".parse().unwrap());
7357 let resp = messages(
7358 State(state),
7359 Extension(TenantId("default".to_owned())),
7360 headers,
7361 user_body(),
7362 )
7363 .await;
7364 assert_eq!(resp.status(), axum::http::StatusCode::OK);
7365 let trace = rx.try_recv().expect("trace enqueued");
7366 assert_eq!(
7367 trace.policy.mode_profile.as_deref(),
7368 Some("quality"),
7369 "mode_profile must be stamped when quality mode is active"
7370 );
7371 }
7372
7373 #[tokio::test]
7374 async fn mode_profile_absent_from_trace_when_balanced() {
7375 let (state, mut rx) = enforce_state(
7376 &["anthropic/claude-haiku-4-5"],
7377 &["non-empty"],
7378 vec![(
7379 "anthropic/claude-haiku-4-5",
7380 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
7381 )],
7382 );
7383 let resp = messages(
7385 State(state),
7386 Extension(TenantId("default".to_owned())),
7387 HeaderMap::new(),
7388 user_body(),
7389 )
7390 .await;
7391 assert_eq!(resp.status(), axum::http::StatusCode::OK);
7392 let trace = rx.try_recv().expect("trace enqueued");
7393 assert!(
7394 trace.policy.mode_profile.is_none(),
7395 "mode_profile must be absent when Balanced (byte-identical invariant)"
7396 );
7397 }
7398
7399 #[tokio::test]
7403 async fn enforce_serves_a_tool_only_turn_from_the_cheap_rung() {
7404 let haiku = "anthropic/claude-haiku-4-5";
7405 let mut tool_turn = model_resp(haiku, "");
7406 tool_turn.raw = serde_json::json!({
7407 "model": "claude-haiku-4-5",
7408 "stop_reason": "tool_use",
7409 "content": [{"type": "tool_use", "id": "toolu_1", "name": "bash",
7410 "input": {"command": "ls"}}],
7411 "usage": {"input_tokens": 10, "output_tokens": 5}
7412 });
7413 let (state, mut rx) = enforce_state(
7414 &[haiku, "anthropic/claude-sonnet-5"],
7415 &["non-empty"],
7416 vec![
7417 (haiku, Ok(tool_turn)),
7418 (
7419 "anthropic/claude-sonnet-5",
7420 Ok(model_resp(
7421 "anthropic/claude-sonnet-5",
7422 "should not be called",
7423 )),
7424 ),
7425 ],
7426 );
7427 let body_with_tools = Bytes::from_static(br#"{
7430 "model": "ignored",
7431 "max_tokens": 64,
7432 "messages": [{"role": "user", "content": "run ls"}],
7433 "tools": [{"name": "bash", "input_schema": {"type": "object", "properties": {"command": {"type": "string"}}}}]
7434 }"#);
7435 let resp = messages(
7436 State(state),
7437 Extension(TenantId("default".to_owned())),
7438 HeaderMap::new(),
7439 body_with_tools,
7440 )
7441 .await;
7442 assert_eq!(resp.status(), axum::http::StatusCode::OK);
7443 let body = axum::body::to_bytes(resp.into_body(), 1 << 20)
7444 .await
7445 .unwrap();
7446 let json: serde_json::Value = serde_json::from_slice(&body).unwrap();
7447 assert_eq!(json["stop_reason"], "tool_use", "{json}");
7448 let trace = rx.try_recv().expect("trace enqueued");
7449 assert_eq!(trace.attempts.len(), 1, "no escalation on a tool step");
7450 assert_eq!(trace.final_.served_rung, Some(0));
7451 }
7452
7453 #[tokio::test]
7454 async fn enforce_serves_first_pass_and_returns_anthropic_shape() {
7455 let (state, mut rx) = enforce_state(
7456 &["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"],
7457 &["non-empty"],
7458 vec![(
7459 "anthropic/claude-haiku-4-5",
7460 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
7461 )],
7462 );
7463 let resp = messages(
7464 State(state),
7465 Extension(TenantId("default".to_owned())),
7466 HeaderMap::new(),
7467 user_body(),
7468 )
7469 .await;
7470 assert_eq!(resp.status(), axum::http::StatusCode::OK);
7471 let json = body_json(resp).await;
7472 assert_eq!(json["type"], "message");
7473 assert_eq!(json["content"][0]["text"], "hello");
7474 assert_eq!(json["model"], "anthropic/claude-haiku-4-5");
7475
7476 let trace = rx.try_recv().expect("a trace was enqueued");
7477 assert_eq!(trace.mode, Mode::Enforce);
7478 assert_eq!(trace.final_.served_rung, Some(0));
7479 assert_eq!(trace.attempts.len(), 1);
7480 }
7481
7482 #[tokio::test]
7483 async fn enforce_escalates_then_serves_and_traces_two_attempts() {
7484 let (state, mut rx) = enforce_state(
7485 &["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"],
7486 &["non-empty"],
7487 vec![
7488 (
7489 "anthropic/claude-haiku-4-5",
7490 Ok(model_resp("anthropic/claude-haiku-4-5", " ")),
7491 ), (
7493 "anthropic/claude-sonnet-5",
7494 Ok(model_resp("anthropic/claude-sonnet-5", "answer")),
7495 ),
7496 ],
7497 );
7498 let resp = messages(
7499 State(state),
7500 Extension(TenantId("default".to_owned())),
7501 HeaderMap::new(),
7502 user_body(),
7503 )
7504 .await;
7505 let json = body_json(resp).await;
7506 assert_eq!(json["content"][0]["text"], "answer");
7507
7508 let trace = rx.try_recv().expect("trace enqueued");
7509 assert_eq!(trace.attempts.len(), 2);
7510 assert_eq!(trace.final_.escalations, 1);
7511 assert_eq!(trace.final_.served_rung, Some(1));
7512 }
7513
7514 #[tokio::test]
7515 async fn enforce_all_rungs_error_returns_502() {
7516 let (state, mut rx) = enforce_state(
7517 &["anthropic/claude-haiku-4-5"],
7518 &["non-empty"],
7519 vec![(
7520 "anthropic/claude-haiku-4-5",
7521 Err(ProviderError::Transport("down".into())),
7522 )],
7523 );
7524 let resp = messages(
7525 State(state),
7526 Extension(TenantId("default".to_owned())),
7527 HeaderMap::new(),
7528 user_body(),
7529 )
7530 .await;
7531 assert_eq!(resp.status(), axum::http::StatusCode::BAD_GATEWAY);
7532 assert!(rx.try_recv().is_ok());
7534 }
7535
7536 #[tokio::test]
7537 async fn no_routing_config_falls_through_to_observe_not_enforce() {
7538 let config = ProxyConfig::from_lookup(|k| match k {
7542 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
7543 _ => None,
7544 })
7545 .unwrap();
7546 let (traces, _rx) = mpsc::channel(64);
7547 let state = AppState {
7548 config: Arc::new(config),
7549 http: reqwest::Client::new(),
7550 providers: ProviderRegistry::new("http://127.0.0.1:1", "http://127.0.0.1:1"),
7551 gate_health: Arc::new(GateHealthRegistry::new()),
7552 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
7553 paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
7554 experiments: Arc::default(),
7555 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
7556 traces,
7557 adaptive: None,
7558 eprocess: None,
7559 bandit: None,
7560 promoter: None,
7561 verified_cache: None,
7562 predictor: None,
7563 prior: None,
7564 tenant_rate_limiter: None,
7565 spill: None,
7566 };
7567 let resp = messages(
7568 State(state),
7569 Extension(TenantId("default".to_owned())),
7570 HeaderMap::new(),
7571 user_body(),
7572 )
7573 .await;
7574 assert_ne!(resp.status(), axum::http::StatusCode::OK);
7576 }
7577
7578 fn bare_enforce_route() -> Route {
7581 use firstpass_core::config::{Match, Mode};
7582 Route {
7583 match_: Match::default(),
7584 mode: Mode::Enforce,
7585 ladder: vec!["anthropic/claude-haiku-4-5".to_owned()],
7586 gates: vec![],
7587 deferred_gates: vec![],
7588 routing_mode: None,
7589 rollout: None,
7590 shadow: None,
7591 challenger: None,
7592 }
7593 }
7594
7595 #[test]
7596 fn balanced_preset_application_is_noop() {
7597 let base_max_rungs = 3u32;
7599 let base_speculation = 2u32;
7600 let preset = RoutingMode::Balanced.preset();
7601 let max_rungs = if let Some(d) = preset.max_rungs_delta {
7602 (base_max_rungs as i32 + d).max(1) as u32
7603 } else {
7604 base_max_rungs
7605 };
7606 let speculation = preset.speculation.unwrap_or(base_speculation);
7607 let start_at_top = preset.start_at_top;
7608 assert_eq!(max_rungs, 3, "Balanced must not change max_rungs");
7609 assert_eq!(speculation, 2, "Balanced must not change speculation");
7610 assert!(!start_at_top, "Balanced must not set start_at_top");
7611 }
7612
7613 #[test]
7614 fn resolve_mode_header_wins_over_route_and_global() {
7615 let mut headers = HeaderMap::new();
7616 headers.insert("x-firstpass-mode", "cost".parse().unwrap());
7617 let mut route = bare_enforce_route();
7618 route.routing_mode = Some(RoutingMode::Quality); let mut config = test_config();
7620 config.default_routing_mode = RoutingMode::Max; assert_eq!(
7622 resolve_mode(&headers, &route, &config),
7623 RoutingMode::Cost,
7624 "header must win"
7625 );
7626 }
7627
7628 #[test]
7629 fn resolve_mode_route_wins_over_global_when_no_header() {
7630 let mut route = bare_enforce_route();
7631 route.routing_mode = Some(RoutingMode::Latency);
7632 let mut config = test_config();
7633 config.default_routing_mode = RoutingMode::Max;
7634 assert_eq!(
7635 resolve_mode(&HeaderMap::new(), &route, &config),
7636 RoutingMode::Latency,
7637 "route must beat global default"
7638 );
7639 }
7640
7641 #[test]
7642 fn resolve_mode_global_when_header_and_route_absent() {
7643 let mut config = test_config();
7644 config.default_routing_mode = RoutingMode::Quality;
7645 assert_eq!(
7646 resolve_mode(&HeaderMap::new(), &bare_enforce_route(), &config),
7647 RoutingMode::Quality
7648 );
7649 }
7650
7651 #[test]
7652 fn resolve_mode_unknown_header_falls_through_to_route() {
7653 let mut headers = HeaderMap::new();
7654 headers.insert("x-firstpass-mode", "turbo-mode".parse().unwrap());
7656 let mut route = bare_enforce_route();
7657 route.routing_mode = Some(RoutingMode::Cost);
7658 let config = test_config();
7659 assert_eq!(
7660 resolve_mode(&headers, &route, &config),
7661 RoutingMode::Cost,
7662 "unknown header value must fall through to route"
7663 );
7664 }
7665
7666 #[test]
7667 fn resolve_mode_header_case_insensitive() {
7668 let mut headers = HeaderMap::new();
7669 headers.insert("x-firstpass-mode", "QUALITY".parse().unwrap());
7670 assert_eq!(
7671 resolve_mode(&headers, &bare_enforce_route(), &test_config()),
7672 RoutingMode::Quality
7673 );
7674 }
7675
7676 #[test]
7677 fn resolve_mode_no_mode_set_returns_balanced() {
7678 assert_eq!(
7680 resolve_mode(&HeaderMap::new(), &bare_enforce_route(), &test_config()),
7681 RoutingMode::Balanced
7682 );
7683 }
7684
7685 #[test]
7686 fn capabilities_json_includes_routing_modes() {
7687 let modes: Vec<&'static str> = RoutingMode::ALL.iter().map(|m| m.as_str()).collect();
7688 assert!(modes.contains(&"balanced"));
7689 assert!(modes.contains(&"cost"));
7690 assert!(modes.contains(&"quality"));
7691 assert!(modes.contains(&"latency"));
7692 assert!(modes.contains(&"max"));
7693 assert!(modes.contains(&"observe"));
7694 }
7695
7696 #[test]
7697 fn model_list_dedups_across_ladders_keeping_cheapest_first_order() {
7698 let config = firstpass_core::config::Config::parse(
7701 r#"
7702 [[route]]
7703 mode = "enforce"
7704 ladder = ["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"]
7705 [[route]]
7706 mode = "enforce"
7707 ladder = ["anthropic/claude-sonnet-5", "anthropic/claude-opus-4-8"]
7708 "#,
7709 )
7710 .expect("test config parses");
7711
7712 let out = model_list(Some(&config));
7713
7714 let ids: Vec<&str> = out["data"]
7715 .as_array()
7716 .expect("data is an array")
7717 .iter()
7718 .map(|m| m["id"].as_str().expect("id is a string"))
7719 .collect();
7720 assert_eq!(
7721 ids,
7722 [
7723 "anthropic/claude-haiku-4-5",
7724 "anthropic/claude-sonnet-5",
7725 "anthropic/claude-opus-4-8",
7726 ],
7727 "sonnet appears once, and ladder order is preserved"
7728 );
7729 assert_eq!(out["object"], "list");
7730 assert_eq!(out["data"][0]["owned_by"], "anthropic");
7731 let haiku_in = out["data"][0]["firstpass"]["input_per_mtok"]
7734 .as_f64()
7735 .expect("a first-party rung has a price");
7736 assert!(haiku_in > 0.0, "expected a real price, got {haiku_in}");
7737 let opus_in = out["data"][2]["firstpass"]["input_per_mtok"]
7739 .as_f64()
7740 .expect("a first-party rung has a price");
7741 assert!(haiku_in < opus_in, "{haiku_in} should undercut {opus_in}");
7742 }
7743
7744 #[tokio::test]
7745 async fn healthz_names_the_service_so_a_client_can_tell_us_from_a_stranger() {
7746 let body = healthz().await.into_response();
7750 let bytes = axum::body::to_bytes(body.into_body(), 64 * 1024)
7751 .await
7752 .expect("healthz body");
7753 let json: serde_json::Value = serde_json::from_slice(&bytes).expect("healthz is json");
7754 assert_eq!(json["status"], "ok");
7755 assert_eq!(json["service"], "firstpass");
7756 }
7757
7758 #[test]
7759 fn model_list_is_empty_without_routing_config() {
7760 let out = model_list(None);
7761 assert_eq!(out["object"], "list");
7762 assert_eq!(
7763 out["data"].as_array().expect("data is an array").len(),
7764 0,
7765 "an observe-only deployment advertises no models"
7766 );
7767 }
7768
7769 #[test]
7770 fn detects_stream_requests() {
7771 assert!(is_stream_request(br#"{"stream": true}"#));
7772 assert!(!is_stream_request(br#"{"stream": false}"#));
7773 assert!(!is_stream_request(br#"{"model":"m"}"#));
7774 assert!(!is_stream_request(b"not json"));
7775 }
7776
7777 #[test]
7778 fn detects_tool_blocks_in_messages() {
7779 let with =
7780 br#"{"messages":[{"role":"user","content":[{"type":"tool_result","content":"42"}]}]}"#;
7781 let without = br#"{"messages":[{"role":"user","content":"hi"}]}"#;
7782 assert!(messages_have_tool_blocks(with));
7783 assert!(!messages_have_tool_blocks(without));
7784 }
7785
7786 #[test]
7787 fn enforce_only_handles_plain_text() {
7788 let plain =
7789 Bytes::from_static(br#"{"model":"m","messages":[{"role":"user","content":"hi"}]}"#);
7790 let tools = Bytes::from_static(
7791 br#"{"model":"m","tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
7792 );
7793 let f_plain = extract_features(&HeaderMap::new(), &plain);
7794 let f_tools = extract_features(&HeaderMap::new(), &tools);
7795 let anthropic_ladder = vec!["anthropic/claude-haiku-4-5".to_owned()];
7796 let providers = test_registry();
7797 assert!(enforce_can_handle(
7799 &f_plain,
7800 &plain,
7801 false,
7802 &anthropic_ladder,
7803 &providers,
7804 Dialect::Anthropic,
7805 ));
7806 assert!(!enforce_can_handle(
7807 &f_tools,
7808 &tools,
7809 false,
7810 &anthropic_ladder,
7811 &providers,
7812 Dialect::Anthropic,
7813 ));
7814 }
7815
7816 #[test]
7817 fn structured_enforce_routes_tools_and_streaming() {
7818 let tools = Bytes::from_static(
7821 br#"{"model":"m","tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
7822 );
7823 let streaming_tools = Bytes::from_static(
7824 br#"{"model":"m","stream":true,"tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
7825 );
7826 let f = extract_features(&HeaderMap::new(), &tools);
7827 let anthropic_ladder = vec![
7828 "anthropic/claude-haiku-4-5".to_owned(),
7829 "anthropic/claude-sonnet-5".to_owned(),
7830 ];
7831 let providers = test_registry();
7832 assert!(enforce_can_handle(
7833 &f,
7834 &tools,
7835 true,
7836 &anthropic_ladder,
7837 &providers,
7838 Dialect::Anthropic,
7839 ));
7840 assert!(enforce_can_handle(
7841 &f,
7842 &streaming_tools,
7843 true,
7844 &anthropic_ladder,
7845 &providers,
7846 Dialect::Anthropic,
7847 ));
7848 }
7849
7850 fn test_registry() -> crate::provider::ProviderRegistry {
7852 crate::provider::ProviderRegistry::new("http://localhost", "http://localhost")
7853 }
7854
7855 #[test]
7856 fn fidelity_guard_blocks_structured_on_non_verbatim_ladder() {
7857 let tools = Bytes::from_static(
7861 br#"{"model":"m","tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
7862 );
7863 let plain =
7864 Bytes::from_static(br#"{"model":"m","messages":[{"role":"user","content":"hi"}]}"#);
7865 let f_tools = extract_features(&HeaderMap::new(), &tools);
7866 let f_plain = extract_features(&HeaderMap::new(), &plain);
7867 let providers = test_registry();
7868 let mixed_ladder = vec![
7869 "openai/gpt-4.1-mini".to_owned(),
7870 "anthropic/claude-sonnet-5".to_owned(),
7871 ];
7872 assert!(!enforce_can_handle(
7873 &f_tools,
7874 &tools,
7875 true,
7876 &mixed_ladder,
7877 &providers,
7878 Dialect::Anthropic,
7879 ));
7880 assert!(enforce_can_handle(
7881 &f_plain,
7882 &plain,
7883 true,
7884 &mixed_ladder,
7885 &providers,
7886 Dialect::Anthropic,
7887 ));
7888 }
7889
7890 #[test]
7891 fn enforce_sse_reemission_preserves_text_and_tool_use() {
7892 let resp = ModelResponse {
7895 model: "anthropic/claude-haiku-4-5".to_owned(),
7896 text: "let me check".to_owned(),
7897 in_tokens: 5,
7898 cache_write_tokens: 0,
7899 cache_read_tokens: 0,
7900 out_tokens: 7,
7901 raw: serde_json::json!({
7902 "content": [
7903 { "type": "text", "text": "let me check" },
7904 { "type": "tool_use", "id": "tu_1", "name": "get_weather", "input": { "city": "Paris" } }
7905 ]
7906 }),
7907 };
7908 let sse = anthropic_sse_from_message(&anthropic_response_json(&resp));
7909
7910 let frames: Vec<Value> = sse
7912 .lines()
7913 .filter_map(|l| l.strip_prefix("data: "))
7914 .map(|d| serde_json::from_str::<Value>(d).expect("each SSE data frame is valid JSON"))
7915 .collect();
7916
7917 assert_eq!(frames.first().unwrap()["type"], "message_start");
7919 assert_eq!(frames.last().unwrap()["type"], "message_stop");
7920 assert!(frames.iter().any(|f| f["delta"]["type"] == "text_delta"
7922 && f["delta"]["text"] == "let me check"));
7923 assert!(
7926 frames
7927 .iter()
7928 .any(|f| f["content_block"]["type"] == "tool_use"
7929 && f["content_block"]["name"] == "get_weather"
7930 && f["content_block"]["id"] == "tu_1")
7931 );
7932 assert!(
7933 frames
7934 .iter()
7935 .any(|f| f["delta"]["type"] == "input_json_delta"
7936 && f["delta"]["partial_json"] == r#"{"city":"Paris"}"#)
7937 );
7938 }
7939
7940 #[tokio::test]
7943 async fn derived_session_survives_the_observe_fallback() {
7944 let toml = "[[price]]\nmodel = \"anthropic/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/m\"]\ngates = [\"non-empty\"]\n[escalation]\nenforce_structured = false\n";
7945 let config = ProxyConfig::from_lookup(|k| match k {
7946 "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
7947 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
7948 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
7949 _ => None,
7950 })
7951 .unwrap();
7952 let (traces, mut rx) = mpsc::channel(8);
7953 let state = AppState {
7954 config: Arc::new(config),
7955 http: reqwest::Client::new(),
7956 providers: ProviderRegistry::from_map(HashMap::new()),
7957 gate_health: Arc::new(GateHealthRegistry::new()),
7958 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
7959 paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
7960 experiments: Arc::default(),
7961 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
7962 traces,
7963 adaptive: None,
7964 eprocess: None,
7965 bandit: None,
7966 promoter: None,
7967 verified_cache: None,
7968 predictor: None,
7969 prior: None,
7970 tenant_rate_limiter: None,
7971 spill: None,
7972 };
7973 let body = Bytes::from_static(
7974 br#"{"model":"m","tools":[{"name":"get_weather"}],"messages":[{"role":"user","content":"hi"}]}"#,
7975 );
7976 let _ = messages(
7977 State(state),
7978 Extension(TenantId("default".to_owned())),
7979 HeaderMap::new(),
7980 body,
7981 )
7982 .await;
7983 let t = rx.recv().await.expect("observe fallback records a trace");
7984 assert!(t.session_id.starts_with("conv:"), "{}", t.session_id);
7985 }
7986
7987 #[tokio::test]
7992 async fn enforce_falls_back_to_observe_for_tool_requests() {
7993 let toml = "[[price]]\nmodel = \"anthropic/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/m\"]\ngates = [\"non-empty\"]\n";
7994 let config = ProxyConfig::from_lookup(|k| match k {
7995 "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
7996 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
7997 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
7998 _ => None,
7999 })
8000 .unwrap();
8001 let mut outs = HashMap::new();
8002 outs.insert(
8003 "anthropic/m".to_owned(),
8004 Ok(model_resp("anthropic/m", "hello")),
8005 );
8006 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
8007 map.insert(
8008 "anthropic".to_owned(),
8009 Arc::new(MockProvider::new("anthropic", outs)),
8010 );
8011 let (traces, _rx) = mpsc::channel(64);
8012 let state = AppState {
8013 config: Arc::new(config),
8014 http: reqwest::Client::new(),
8015 providers: ProviderRegistry::from_map(map),
8016 gate_health: Arc::new(GateHealthRegistry::new()),
8017 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
8018 paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
8019 experiments: Arc::default(),
8020 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
8021 traces,
8022 adaptive: None,
8023 eprocess: None,
8024 bandit: None,
8025 promoter: None,
8026 verified_cache: None,
8027 predictor: None,
8028 prior: None,
8029 tenant_rate_limiter: None,
8030 spill: None,
8031 };
8032
8033 let plain =
8035 Bytes::from_static(br#"{"model":"m","messages":[{"role":"user","content":"hi"}]}"#);
8036 let resp = messages(
8037 State(state.clone()),
8038 Extension(TenantId("default".to_owned())),
8039 HeaderMap::new(),
8040 plain,
8041 )
8042 .await;
8043 assert_eq!(
8044 resp.status(),
8045 axum::http::StatusCode::OK,
8046 "plain text should enforce"
8047 );
8048
8049 let tools = Bytes::from_static(
8052 br#"{"model":"m","tools":[{"name":"get_weather"}],"messages":[{"role":"user","content":"hi"}]}"#,
8053 );
8054 let resp = messages(
8055 State(state.clone()),
8056 Extension(TenantId("default".to_owned())),
8057 HeaderMap::new(),
8058 tools.clone(),
8059 )
8060 .await;
8061 assert_eq!(
8062 resp.status(),
8063 axum::http::StatusCode::OK,
8064 "tool request must route through enforce by default (ADR 0005 default-on)"
8065 );
8066
8067 let toml_off = "[[price]]\nmodel = \"anthropic/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/m\"]\ngates = [\"non-empty\"]\n[escalation]\nenforce_structured = false\n";
8070 let config_off = ProxyConfig::from_lookup(|k| match k {
8071 "FIRSTPASS_CONFIG_TOML" => Some(toml_off.to_owned()),
8072 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
8073 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
8074 _ => None,
8075 })
8076 .unwrap();
8077 let state_off = AppState {
8078 config: Arc::new(config_off),
8079 ..state.clone()
8080 };
8081 let resp = messages(
8082 State(state_off),
8083 Extension(TenantId("default".to_owned())),
8084 HeaderMap::new(),
8085 tools,
8086 )
8087 .await;
8088 assert_ne!(
8089 resp.status(),
8090 axum::http::StatusCode::OK,
8091 "with enforce_structured = false a tool request must fall back to observe"
8092 );
8093
8094 let toolres = Bytes::from_static(
8096 br#"{"model":"m","messages":[{"role":"user","content":[{"type":"tool_result","tool_use_id":"x","content":"42"}]}]}"#,
8097 );
8098 let resp = messages(
8099 State(state),
8100 Extension(TenantId("default".to_owned())),
8101 HeaderMap::new(),
8102 toolres,
8103 )
8104 .await;
8105 assert_eq!(
8106 resp.status(),
8107 axum::http::StatusCode::OK,
8108 "tool_result blocks route through enforce by default too (verbatim carry)"
8109 );
8110 }
8111
8112 #[tokio::test]
8116 async fn panel_is_public_markup_but_receipts_stay_behind_auth() {
8117 use tower::ServiceExt;
8118 let (state, _rx) = enforce_state(
8119 &["anthropic/claude-haiku-4-5"],
8120 &["non-empty"],
8121 vec![(
8122 "anthropic/claude-haiku-4-5",
8123 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
8124 )],
8125 );
8126 let router = app(state).expect("prometheus recorder installs");
8127
8128 let page = router
8129 .clone()
8130 .oneshot(
8131 axum::http::Request::builder()
8132 .uri("/panel")
8133 .body(Body::empty())
8134 .unwrap(),
8135 )
8136 .await
8137 .unwrap();
8138 assert_eq!(page.status(), axum::http::StatusCode::OK);
8139 let html = axum::body::to_bytes(page.into_body(), 1 << 20)
8140 .await
8141 .unwrap();
8142 let html = String::from_utf8_lossy(&html);
8143 assert!(
8144 html.contains("firstpass") && html.contains("/v1/receipts"),
8145 "the page must fetch its data rather than embed it"
8146 );
8147 assert!(
8150 !html.contains("http://") && !html.contains("https://"),
8151 "the panel must not reference any external origin"
8152 );
8153 assert!(html.contains("const esc ="), "the escaper must exist");
8158 for raw in [
8159 "${a.model}",
8160 "${a.rung}",
8161 "${a.verdict}",
8162 "${r.trace_id}",
8163 "${r.served_rung}",
8164 ] {
8165 assert!(
8166 !html.contains(raw),
8167 "receipt field interpolated unescaped into innerHTML: {raw}"
8168 );
8169 }
8170
8171 let data = router
8172 .oneshot(
8173 axum::http::Request::builder()
8174 .uri("/v1/receipts?limit=5")
8175 .body(Body::empty())
8176 .unwrap(),
8177 )
8178 .await
8179 .unwrap();
8180 assert_eq!(data.status(), axum::http::StatusCode::OK);
8184 let body = axum::body::to_bytes(data.into_body(), 1 << 20)
8185 .await
8186 .unwrap();
8187 let v: Value = serde_json::from_slice(&body).unwrap();
8188 assert!(v.get("receipts").and_then(Value::as_array).is_some());
8189 }
8190
8191 async fn feedback_state() -> (AppState, std::path::PathBuf, String) {
8195 feedback_state_with(false).await
8196 }
8197
8198 async fn feedback_state_with(served: bool) -> (AppState, std::path::PathBuf, String) {
8204 let db = std::env::temp_dir().join(format!("firstpass-feedback-{}.db", Uuid::now_v7()));
8205 let (tx, handle) = crate::store::open(&db).unwrap();
8206
8207 let mut trace = build_error_trace(
8208 &ProxyConfig::from_lookup(|_| None).unwrap(),
8209 &Bytes::from_static(b"{}"),
8210 5,
8211 Some("sess-fb"),
8212 "anthropic",
8213 "anthropic.messages",
8214 );
8215 trace.attempts.push(Attempt {
8216 rung: 0,
8217 model: "anthropic/claude-haiku-4-5".into(),
8218 provider: "anthropic".into(),
8219 in_tokens: 10,
8220 cache_write_tokens: 0,
8221 cache_read_tokens: 0,
8222 out_tokens: 5,
8223 cost_usd: 0.001,
8224 latency_ms: 5,
8225 gates: vec![],
8226 verdict: Verdict::Pass,
8227 truncated: false,
8228 });
8229 let trace_id = trace.trace_id.to_string();
8230 if served {
8231 trace.final_.served_rung = Some(0);
8232 }
8233 tx.try_send(trace).unwrap();
8234 drop(tx);
8235 handle.await.unwrap();
8236
8237 let db_str = db.to_string_lossy().into_owned();
8238 let config = ProxyConfig::from_lookup(move |k| match k {
8239 "FIRSTPASS_DB" => Some(db_str.clone()),
8240 _ => None,
8241 })
8242 .unwrap();
8243 let (traces, _rx) = mpsc::channel(64);
8244 let state = AppState {
8245 config: Arc::new(config),
8246 http: reqwest::Client::new(),
8247 providers: ProviderRegistry::new("http://127.0.0.1:1", "http://127.0.0.1:1"),
8248 gate_health: Arc::new(GateHealthRegistry::new()),
8249 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
8250 paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
8251 experiments: Arc::default(),
8252 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
8253 traces,
8254 adaptive: None,
8255 eprocess: None,
8256 bandit: None,
8257 promoter: None,
8258 verified_cache: None,
8259 predictor: None,
8260 prior: None,
8261 tenant_rate_limiter: None,
8262 spill: None,
8263 };
8264 (state, db, trace_id)
8265 }
8266
8267 #[tokio::test]
8281 async fn feedback_without_a_score_still_advances_the_eprocess() {
8282 use firstpass_core::eprocess::{DEFAULT_BET, EProcessRiskControl};
8283 let (mut state, _db, trace_id) = feedback_state_with(true).await;
8284 let grid: Vec<f64> = (0..=20).map(|i| f64::from(i) / 20.0).collect();
8285 let ep = Arc::new(std::sync::Mutex::new(EProcessRiskControl::new(
8286 0.2,
8287 0.05,
8288 DEFAULT_BET,
8289 &grid,
8290 )));
8291 state.eprocess = Some(ep.clone());
8292 assert_eq!(ep.lock().unwrap().rounds(), 0);
8293
8294 let body = Bytes::from(
8296 serde_json::json!({
8297 "trace_id": trace_id, "gate_id": "tests", "verdict": "pass", "reporter": "ci"
8298 })
8299 .to_string(),
8300 );
8301 let resp = feedback(
8302 State(state.clone()),
8303 Extension(TenantId("default".to_owned())),
8304 body,
8305 )
8306 .await
8307 .into_response();
8308 assert_eq!(resp.status(), axum::http::StatusCode::ACCEPTED);
8310
8311 assert_eq!(
8312 ep.lock().unwrap().rounds(),
8313 1,
8314 "a scoreless feedback call must still advance the controller — keying on the optional \
8315 payload score left it permanently inert"
8316 );
8317 }
8318
8319 #[tokio::test]
8320 async fn feedback_nudges_the_adaptive_threshold() {
8321 use firstpass_core::conformal::AdaptiveConformal;
8322 let (mut state, _db, trace_id) = feedback_state().await;
8323 let aci = Arc::new(std::sync::Mutex::new(AdaptiveConformal::new(0.1, 0.2, 0.5)));
8324 state.adaptive = Some(aci.clone());
8325 let before = aci.lock().unwrap().threshold();
8326
8327 let fail = Bytes::from(
8329 serde_json::json!({ "trace_id": trace_id, "gate_id": "tests", "verdict": "fail", "reporter": "ci" })
8330 .to_string(),
8331 );
8332 assert_eq!(
8333 feedback(
8334 State(state.clone()),
8335 Extension(TenantId("default".to_owned())),
8336 fail
8337 )
8338 .await
8339 .status(),
8340 axum::http::StatusCode::ACCEPTED
8341 );
8342 let after_fail = aci.lock().unwrap().threshold();
8343 assert!(
8344 after_fail > before,
8345 "served fail should raise the live threshold: {before} -> {after_fail}"
8346 );
8347
8348 let pass = Bytes::from(
8350 serde_json::json!({ "trace_id": trace_id, "gate_id": "tests", "verdict": "pass", "reporter": "ci" })
8351 .to_string(),
8352 );
8353 let _ = feedback(
8354 State(state),
8355 Extension(TenantId("default".to_owned())),
8356 pass,
8357 )
8358 .await;
8359 assert!(aci.lock().unwrap().threshold() < after_fail);
8360 }
8361
8362 #[tokio::test]
8363 async fn feedback_records_a_deferred_verdict_without_breaking_the_chain() {
8364 let (state, db, trace_id) = feedback_state().await;
8365 let body = Bytes::from(
8366 serde_json::json!({
8367 "trace_id": trace_id,
8368 "gate_id": "tests",
8369 "verdict": "pass",
8370 "score": 1.0,
8371 "reporter": "ci",
8372 })
8373 .to_string(),
8374 );
8375 let resp = feedback(
8376 State(state),
8377 Extension(TenantId("default".to_owned())),
8378 body,
8379 )
8380 .await;
8381 assert_eq!(resp.status(), axum::http::StatusCode::ACCEPTED);
8382
8383 let view = crate::store::load_trace_view(&db, "default", &trace_id)
8385 .unwrap()
8386 .unwrap();
8387 assert_eq!(view.deferred.len(), 1);
8388 assert_eq!(view.deferred[0].gate_id, "tests");
8389 let traces = crate::store::load_all_traces(&db).unwrap();
8391 firstpass_core::verify_chain(&traces, GENESIS_HASH).unwrap();
8392
8393 let _ = std::fs::remove_file(&db);
8394 }
8395
8396 #[tokio::test]
8397 async fn feedback_for_unknown_trace_is_404() {
8398 let (state, db, _trace_id) = feedback_state().await;
8399 let body = Bytes::from(
8400 serde_json::json!({
8401 "trace_id": "does-not-exist",
8402 "gate_id": "tests",
8403 "verdict": "pass",
8404 "reporter": "ci",
8405 })
8406 .to_string(),
8407 );
8408 let resp = feedback(
8409 State(state),
8410 Extension(TenantId("default".to_owned())),
8411 body,
8412 )
8413 .await;
8414 assert_eq!(resp.status(), axum::http::StatusCode::NOT_FOUND);
8415 let _ = std::fs::remove_file(&db);
8416 }
8417
8418 #[tokio::test]
8421 async fn feedback_across_tenants_is_404_not_403() {
8422 let (state, db, trace_id) = feedback_state().await;
8423 let body = Bytes::from(
8424 serde_json::json!({
8425 "trace_id": trace_id,
8426 "gate_id": "tests",
8427 "verdict": "pass",
8428 "score": 1.0,
8429 "reporter": "attacker",
8430 })
8431 .to_string(),
8432 );
8433 let resp = feedback(
8435 State(state),
8436 Extension(TenantId("tenant-b".to_owned())),
8437 body,
8438 )
8439 .await;
8440 assert_eq!(
8441 resp.status(),
8442 axum::http::StatusCode::NOT_FOUND,
8443 "cross-tenant feedback must look exactly like a missing trace"
8444 );
8445 let _ = std::fs::remove_file(&db);
8446 }
8447
8448 #[tokio::test]
8449 async fn feedback_rejects_bad_verdict_and_score() {
8450 let (state, db, trace_id) = feedback_state().await;
8451 let bad_verdict = Bytes::from(
8452 serde_json::json!({ "trace_id": trace_id, "gate_id": "g", "verdict": "maybe", "reporter": "x" })
8453 .to_string(),
8454 );
8455 assert_eq!(
8456 feedback(
8457 State(state.clone()),
8458 Extension(TenantId("default".to_owned())),
8459 bad_verdict
8460 )
8461 .await
8462 .status(),
8463 axum::http::StatusCode::BAD_REQUEST
8464 );
8465 let bad_score = Bytes::from(
8466 serde_json::json!({ "trace_id": trace_id, "gate_id": "g", "verdict": "pass", "score": 9.0, "reporter": "x" })
8467 .to_string(),
8468 );
8469 assert_eq!(
8470 feedback(
8471 State(state),
8472 Extension(TenantId("default".to_owned())),
8473 bad_score
8474 )
8475 .await
8476 .status(),
8477 axum::http::StatusCode::BAD_REQUEST
8478 );
8479 let _ = std::fs::remove_file(&db);
8480 }
8481
8482 #[tokio::test]
8483 async fn metrics_endpoint_renders_after_a_real_request() {
8484 use tower::ServiceExt;
8485
8486 let (state, mut rx) = enforce_state(
8487 &["anthropic/claude-haiku-4-5"],
8488 &["non-empty"],
8489 vec![(
8490 "anthropic/claude-haiku-4-5",
8491 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
8492 )],
8493 );
8494 let router = app(state).expect("prometheus recorder installs");
8495
8496 let req = axum::http::Request::builder()
8497 .method("POST")
8498 .uri("/v1/messages")
8499 .header("content-type", "application/json")
8500 .body(Body::from(user_body()))
8501 .unwrap();
8502 let resp = router.clone().oneshot(req).await.unwrap();
8503 assert_eq!(resp.status(), axum::http::StatusCode::OK);
8504 rx.try_recv().expect("a trace was enqueued");
8505
8506 let metrics_req = axum::http::Request::builder()
8507 .method("GET")
8508 .uri("/metrics")
8509 .body(Body::empty())
8510 .unwrap();
8511 let metrics_resp = router.oneshot(metrics_req).await.unwrap();
8512 assert_eq!(metrics_resp.status(), axum::http::StatusCode::OK);
8513 let bytes = axum::body::to_bytes(metrics_resp.into_body(), 1 << 20)
8514 .await
8515 .unwrap();
8516 let body = String::from_utf8(bytes.to_vec()).unwrap();
8517 assert!(
8518 body.contains("firstpass_enforce_latency_ms"),
8519 "metrics body missing enforce latency histogram: {body}"
8520 );
8521 assert!(
8522 body.contains("firstpass_served_total"),
8523 "metrics body missing served counter: {body}"
8524 );
8525 }
8526
8527 #[tokio::test]
8533 async fn metrics_are_dimensioned_by_provider_rung_and_gate() {
8534 use tower::ServiceExt;
8535
8536 let (state, mut rx) = enforce_state(
8537 &["anthropic/claude-haiku-4-5"],
8538 &["non-empty"],
8539 vec![(
8540 "anthropic/claude-haiku-4-5",
8541 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
8542 )],
8543 );
8544 let router = app(state).expect("prometheus recorder installs");
8545
8546 let req = axum::http::Request::builder()
8547 .method("POST")
8548 .uri("/v1/messages")
8549 .header("content-type", "application/json")
8550 .body(Body::from(user_body()))
8551 .unwrap();
8552 assert_eq!(
8553 router.clone().oneshot(req).await.unwrap().status(),
8554 axum::http::StatusCode::OK
8555 );
8556 rx.try_recv().expect("a trace was enqueued");
8557
8558 let metrics_req = axum::http::Request::builder()
8559 .method("GET")
8560 .uri("/metrics")
8561 .body(Body::empty())
8562 .unwrap();
8563 let bytes = axum::body::to_bytes(
8564 router.oneshot(metrics_req).await.unwrap().into_body(),
8565 1 << 20,
8566 )
8567 .await
8568 .unwrap();
8569 let body = String::from_utf8(bytes.to_vec()).unwrap();
8570
8571 assert!(
8573 body.contains(r#"firstpass_attempt_total{provider="anthropic",rung="0"}"#),
8574 "attempt counter is not dimensioned by provider+rung: {body}"
8575 );
8576 assert!(
8577 body.contains("firstpass_attempt_latency_ms")
8578 && body.contains(r#"provider="anthropic""#),
8579 "attempt latency is not dimensioned by provider: {body}"
8580 );
8581 assert!(
8582 body.contains("firstpass_attempt_cost_usd_total"),
8583 "per-attempt cost series missing: {body}"
8584 );
8585 assert!(
8587 body.contains(r#"firstpass_gate_verdict_total{gate_id="non-empty",verdict="pass"}"#),
8588 "gate verdicts are not dimensioned by gate_id+verdict: {body}"
8589 );
8590 }
8591
8592 fn auth_state(require_auth: bool, keys_json: Option<String>) -> AppState {
8596 auth_state_rated(require_auth, keys_json, None)
8597 }
8598
8599 fn auth_state_rated(
8602 require_auth: bool,
8603 keys_json: Option<String>,
8604 rate_per_sec: Option<u32>,
8605 ) -> AppState {
8606 let config = ProxyConfig::from_lookup(|k| match k {
8607 "FIRSTPASS_REQUIRE_AUTH" => require_auth.then(|| "true".to_owned()),
8608 "FIRSTPASS_TENANT_KEYS_JSON" => keys_json.clone(),
8609 "FIRSTPASS_TENANT_RATE_PER_SEC" => rate_per_sec.map(|n| n.to_string()),
8610 _ => None,
8611 })
8612 .unwrap();
8613 let (traces, _rx) = mpsc::channel(64);
8614 std::mem::forget(_rx);
8617 let providers: HashMap<String, Arc<dyn Provider>> = HashMap::new();
8618 let tenant_rate_limiter = build_tenant_rate_limiter(&config);
8619 AppState {
8620 config: Arc::new(config),
8621 http: reqwest::Client::new(),
8622 providers: ProviderRegistry::from_map(providers),
8623 gate_health: Arc::new(GateHealthRegistry::new()),
8624 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
8625 paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
8626 experiments: Arc::default(),
8627 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
8628 traces,
8629 adaptive: None,
8630 eprocess: None,
8631 bandit: None,
8632 promoter: None,
8633 verified_cache: None,
8634 predictor: None,
8635 prior: None,
8636 tenant_rate_limiter,
8637 spill: None,
8638 }
8639 }
8640
8641 fn cap_request(auth_header: Option<&str>) -> axum::http::Request<Body> {
8642 let mut b = axum::http::Request::builder()
8643 .method("GET")
8644 .uri("/v1/capabilities");
8645 if let Some(h) = auth_header {
8646 b = b.header("authorization", h);
8647 }
8648 b.body(Body::empty()).unwrap()
8649 }
8650
8651 #[tokio::test]
8652 async fn auth_off_allows_unauthenticated_request() {
8653 use tower::ServiceExt;
8654 let router = app(auth_state(false, None)).expect("router");
8655 let resp = router.oneshot(cap_request(None)).await.unwrap();
8656 assert_eq!(resp.status(), axum::http::StatusCode::OK);
8658 }
8659
8660 #[tokio::test]
8661 async fn auth_on_missing_key_is_401_opaque() {
8662 use tower::ServiceExt;
8663 let hash = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
8664 let keys = format!("{{\"tenant-a\": {hash:?}}}");
8665 let router = app(auth_state(true, Some(keys))).expect("router");
8666
8667 let resp = router.oneshot(cap_request(None)).await.unwrap();
8668 assert_eq!(resp.status(), axum::http::StatusCode::UNAUTHORIZED);
8669 let json = body_json(resp).await;
8670 assert_eq!(json["error"]["type"], "unauthorized");
8671 let msg = json["error"]["message"].as_str().unwrap();
8673 assert!(!msg.contains("tenant"), "no tenant oracle in body: {msg}");
8674 }
8675
8676 #[tokio::test]
8677 async fn auth_on_invalid_key_is_401() {
8678 use tower::ServiceExt;
8679 let hash = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
8680 let keys = format!("{{\"tenant-a\": {hash:?}}}");
8681 let router = app(auth_state(true, Some(keys))).expect("router");
8682
8683 let resp = router
8684 .oneshot(cap_request(Some("Bearer wrong-key")))
8685 .await
8686 .unwrap();
8687 assert_eq!(resp.status(), axum::http::StatusCode::UNAUTHORIZED);
8688 }
8689
8690 #[tokio::test]
8691 async fn auth_on_valid_key_proceeds() {
8692 use tower::ServiceExt;
8693 let hash = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
8694 let keys = format!("{{\"tenant-a\": {hash:?}}}");
8695 let router = app(auth_state(true, Some(keys))).expect("router");
8696
8697 let resp = router
8699 .oneshot(cap_request(Some("Bearer tenant-a.key-a")))
8700 .await
8701 .unwrap();
8702 assert_eq!(resp.status(), axum::http::StatusCode::OK);
8704 }
8705
8706 fn two_tenant_state(rate_per_sec: Option<u32>) -> AppState {
8708 let hash_a = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
8709 let hash_b = crate::tenant_auth::TenantKeys::hash_key("key-b").unwrap();
8710 let keys = format!("{{\"tenant-a\": {hash_a:?}, \"tenant-b\": {hash_b:?}}}");
8711 let mut state = auth_state_rated(true, Some(keys), rate_per_sec);
8712 if let Some(n) = rate_per_sec.and_then(std::num::NonZeroU32::new) {
8717 state.tenant_rate_limiter = Some(Arc::new(governor::RateLimiter::keyed(
8718 governor::Quota::per_hour(n),
8719 )));
8720 }
8721 state
8722 }
8723
8724 #[test]
8725 fn tenant_rate_limiter_allows_a_burst_of_exactly_the_configured_rate() {
8726 let config = ProxyConfig::from_lookup(|k| {
8727 (k == "FIRSTPASS_TENANT_RATE_PER_SEC").then(|| "2".to_owned())
8728 })
8729 .unwrap();
8730 let limiter = build_tenant_rate_limiter(&config).expect("rate configured");
8731 let t = "tenant-a".to_owned();
8732 assert!(limiter.check_key(&t).is_ok());
8733 assert!(limiter.check_key(&t).is_ok());
8734 assert!(
8735 limiter.check_key(&t).is_err(),
8736 "third in the same instant is over the burst"
8737 );
8738 assert!(
8739 limiter.check_key(&"tenant-b".to_owned()).is_ok(),
8740 "buckets are per tenant"
8741 );
8742 }
8743
8744 #[tokio::test]
8745 async fn tenant_exceeding_rate_limit_gets_429_opaque() {
8746 use tower::ServiceExt;
8747 let router = app(two_tenant_state(Some(1))).expect("router");
8750
8751 let (r1, r2, r3, r4) = tokio::join!(
8752 router
8753 .clone()
8754 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
8755 router
8756 .clone()
8757 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
8758 router
8759 .clone()
8760 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
8761 router
8762 .clone()
8763 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
8764 );
8765 let responses = [r1.unwrap(), r2.unwrap(), r3.unwrap(), r4.unwrap()];
8766 let ok = responses
8767 .iter()
8768 .filter(|r| r.status() == axum::http::StatusCode::OK)
8769 .count();
8770 assert!(ok >= 1, "the burst's first request must pass");
8771 let limited: Vec<_> = responses
8772 .into_iter()
8773 .filter(|r| r.status() == axum::http::StatusCode::TOO_MANY_REQUESTS)
8774 .collect();
8775 assert!(
8776 !limited.is_empty(),
8777 "a 4-request burst against 1 req/sec must trip the limiter"
8778 );
8779
8780 let json = body_json(limited.into_iter().next().unwrap()).await;
8781 assert_eq!(json["error"]["type"], "rate_limited");
8782 let msg = json["error"]["message"].as_str().unwrap();
8784 assert!(!msg.contains('1'), "no limit value in body: {msg}");
8785 }
8786
8787 #[tokio::test]
8788 async fn rate_limit_buckets_are_independent_per_tenant() {
8789 use tower::ServiceExt;
8790 let router = app(two_tenant_state(Some(1))).expect("router");
8791
8792 let (a1, a2, a3) = tokio::join!(
8794 router
8795 .clone()
8796 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
8797 router
8798 .clone()
8799 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
8800 router
8801 .clone()
8802 .oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
8803 );
8804 let a_limited = [a1.unwrap(), a2.unwrap(), a3.unwrap()]
8805 .iter()
8806 .filter(|r| r.status() == axum::http::StatusCode::TOO_MANY_REQUESTS)
8807 .count();
8808 assert!(a_limited >= 1, "tenant A's burst must trip its limiter");
8809
8810 let b1 = router
8812 .clone()
8813 .oneshot(cap_request(Some("Bearer tenant-b.key-b")))
8814 .await
8815 .unwrap();
8816 assert_eq!(b1.status(), axum::http::StatusCode::OK);
8817 }
8818
8819 #[tokio::test]
8820 async fn key_verify_does_not_stall_the_async_worker() {
8821 use std::sync::atomic::{AtomicUsize, Ordering};
8822 use tower::ServiceExt;
8823 let router = app(two_tenant_state(None)).expect("router");
8826 let ticks = Arc::new(AtomicUsize::new(0));
8827 let ticker = tokio::spawn({
8828 let ticks = ticks.clone();
8829 async move {
8830 loop {
8831 tokio::time::sleep(std::time::Duration::from_millis(1)).await;
8832 ticks.fetch_add(1, Ordering::Relaxed);
8833 }
8834 }
8835 });
8836 let resp = router
8837 .oneshot(cap_request(Some("Bearer tenant-a.key-a")))
8838 .await
8839 .unwrap();
8840 let during = ticks.load(Ordering::Relaxed);
8841 ticker.abort();
8842 assert_eq!(resp.status(), axum::http::StatusCode::OK);
8843 assert!(
8844 during >= 2,
8845 "worker was blocked during the verify ({during} ticks)"
8846 );
8847 }
8848
8849 #[tokio::test]
8850 async fn rate_limit_unset_never_429s() {
8851 use tower::ServiceExt;
8852 let router = app(two_tenant_state(None)).expect("router");
8855 for _ in 0..20 {
8856 let resp = router
8857 .clone()
8858 .oneshot(cap_request(Some("Bearer tenant-a.key-a")))
8859 .await
8860 .unwrap();
8861 assert_eq!(resp.status(), axum::http::StatusCode::OK);
8862 }
8863 }
8864
8865 #[test]
8868 fn u01_is_deterministic_and_in_range() {
8869 let s1 = u01(0xDEAD_BEEF_CAFE_1234_u128);
8870 let s2 = u01(0xDEAD_BEEF_CAFE_1234_u128);
8871 assert_eq!(s1, s2, "u01 must be deterministic for the same seed");
8872 assert!((0.0..1.0).contains(&s1), "u01 must return [0, 1), got {s1}");
8873
8874 let s3 = u01(0x1234_5678_9ABC_DEF0_u128);
8876 assert_ne!(s1, s3, "different seeds should give different values");
8877
8878 for i in 0u64..256 {
8880 let v = u01(i as u128);
8881 assert!((0.0..1.0).contains(&v), "seed {i}: u01={v} out of [0,1)");
8882 }
8883 }
8884
8885 #[test]
8886 fn epsilon_propensity_formula() {
8887 let epsilon = 0.2_f64;
8888 let k = 3_usize;
8889 let greedy = 1_u32;
8890
8891 let p_greedy = epsilon_propensity(greedy, greedy, epsilon, k);
8893 let expected_greedy = (1.0 - epsilon) + epsilon / k as f64;
8894 assert!(
8895 (p_greedy - expected_greedy).abs() < 1e-12,
8896 "{p_greedy} != {expected_greedy}"
8897 );
8898
8899 let p_other = epsilon_propensity(0, greedy, epsilon, k);
8901 let expected_other = epsilon / k as f64;
8902 assert!(
8903 (p_other - expected_other).abs() < 1e-12,
8904 "{p_other} != {expected_other}"
8905 );
8906
8907 for chosen in 0..k as u32 {
8909 let p = epsilon_propensity(chosen, greedy, epsilon, k);
8910 assert!(
8911 p > 0.0 && p <= 1.0,
8912 "propensity {p} out of (0,1] for chosen={chosen}"
8913 );
8914 }
8915 }
8916
8917 #[test]
8918 fn epsilon_branch_and_greedy_branch_both_occur_over_many_seeds() {
8919 let epsilon = 0.3_f64;
8921 let mut saw_explore = false;
8922 let mut saw_greedy = false;
8923 for i in 0u64..200 {
8924 let u = u01(i as u128);
8925 if u < epsilon {
8926 saw_explore = true;
8927 } else {
8928 saw_greedy = true;
8929 }
8930 if saw_explore && saw_greedy {
8931 break;
8932 }
8933 }
8934 assert!(
8935 saw_explore,
8936 "epsilon branch must fire with epsilon=0.3 over 200 seeds"
8937 );
8938 assert!(
8939 saw_greedy,
8940 "greedy branch must occur with epsilon=0.3 over 200 seeds"
8941 );
8942 }
8943
8944 #[test]
8945 fn epsilon_propensity_sums_to_one_over_all_rungs() {
8946 let epsilon = 0.15_f64;
8948 let k = 4_usize;
8949 let greedy = 2_u32;
8950 let total: f64 = (0..k as u32)
8951 .map(|r| epsilon_propensity(r, greedy, epsilon, k))
8952 .sum();
8953 assert!(
8954 (total - 1.0).abs() < 1e-12,
8955 "propensities must sum to 1, got {total}"
8956 );
8957 }
8958
8959 #[tokio::test(start_paused = true)]
8962 async fn keepalive_stream_ticks_then_emits_final_frame() {
8963 let (tx, rx) = tokio::sync::oneshot::channel::<Result<Value, ProxyError>>();
8964 let mut ticks = tokio::time::interval(SSE_KEEPALIVE_EVERY);
8965 ticks.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
8966 ticks.reset();
8967 let mut stream = KeepaliveStream {
8968 rx: Some(rx),
8969 ticks,
8970 format_message: Box::new(anthropic_sse_from_message),
8971 format_error: sse_error_event,
8972 };
8973 async fn next(
8975 stream: &mut KeepaliveStream,
8976 ) -> Option<Option<Result<Bytes, std::convert::Infallible>>> {
8977 std::future::poll_fn(|cx| {
8978 std::task::Poll::Ready(
8979 match futures_core::Stream::poll_next(std::pin::Pin::new(&mut *stream), cx) {
8980 std::task::Poll::Ready(item) => Some(item),
8981 std::task::Poll::Pending => None,
8982 },
8983 )
8984 })
8985 .await
8986 }
8987
8988 assert!(
8990 next(&mut stream).await.is_none(),
8991 "no frame before an interval"
8992 );
8993
8994 tokio::time::advance(SSE_KEEPALIVE_EVERY + Duration::from_millis(1)).await;
8996 let frame = next(&mut stream)
8997 .await
8998 .expect("keepalive due")
8999 .unwrap()
9000 .unwrap();
9001 assert!(
9002 frame.starts_with(b": "),
9003 "keepalive must be an SSE comment (ignored by every conforming parser)"
9004 );
9005
9006 let message = serde_json::json!({
9008 "id": "msg_1", "type": "message", "role": "assistant", "model": "m",
9009 "content": [{ "type": "text", "text": "done" }],
9010 "usage": { "input_tokens": 1, "output_tokens": 1 }
9011 });
9012 tx.send(Ok(message)).unwrap();
9013 let frame = next(&mut stream)
9014 .await
9015 .expect("final frame")
9016 .unwrap()
9017 .unwrap();
9018 let text = String::from_utf8(frame.to_vec()).unwrap();
9019 assert!(text.contains("event: message_start"));
9020 assert!(text.contains("event: message_stop"));
9021 let eos = next(&mut stream)
9022 .await
9023 .expect("stream must end after the final frame");
9024 assert!(eos.is_none(), "end-of-stream after the final frame");
9025 }
9026
9027 #[tokio::test]
9030 async fn streaming_enforce_serves_full_sse_sequence() {
9031 let toml = "[[price]]\nmodel = \"anthropic/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/m\"]\ngates = [\"non-empty\"]\n";
9032 let config = ProxyConfig::from_lookup(|k| match k {
9033 "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
9034 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
9035 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
9036 _ => None,
9037 })
9038 .unwrap();
9039 let mut outs = HashMap::new();
9040 outs.insert(
9041 "anthropic/m".to_owned(),
9042 Ok(model_resp("anthropic/m", "gated answer")),
9043 );
9044 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
9045 map.insert(
9046 "anthropic".to_owned(),
9047 Arc::new(MockProvider::new("anthropic", outs)),
9048 );
9049 let (traces, _rx) = mpsc::channel(64);
9050 let state = AppState {
9051 config: Arc::new(config),
9052 http: reqwest::Client::new(),
9053 providers: ProviderRegistry::from_map(map),
9054 gate_health: Arc::new(GateHealthRegistry::new()),
9055 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
9056 paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
9057 experiments: Arc::default(),
9058 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
9059 traces,
9060 adaptive: None,
9061 eprocess: None,
9062 bandit: None,
9063 promoter: None,
9064 verified_cache: None,
9065 predictor: None,
9066 prior: None,
9067 tenant_rate_limiter: None,
9068 spill: None,
9069 };
9070 let body = Bytes::from_static(
9071 br#"{"model":"m","stream":true,"messages":[{"role":"user","content":"hi"}]}"#,
9072 );
9073 let resp = messages(
9074 State(state),
9075 Extension(TenantId("default".to_owned())),
9076 HeaderMap::new(),
9077 body,
9078 )
9079 .await;
9080 assert_eq!(resp.status(), axum::http::StatusCode::OK);
9081 assert!(
9082 resp.headers()
9083 .get(axum::http::header::CONTENT_TYPE)
9084 .and_then(|v| v.to_str().ok())
9085 .is_some_and(|ct| ct.starts_with("text/event-stream")),
9086 "streaming client must get SSE"
9087 );
9088 let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
9089 .await
9090 .unwrap();
9091 let text = String::from_utf8(bytes.to_vec()).unwrap();
9092 assert!(text.contains("event: message_start"));
9093 assert!(text.contains("gated answer"));
9094 assert!(text.contains("event: message_stop"));
9095 }
9096
9097 #[test]
9102 fn parse_openai_request_plain_text() {
9103 let body = br#"{"model":"gpt-4o","max_tokens":256,"messages":[{"role":"user","content":"hello"}]}"#;
9105 let req = parse_openai_request(body, false).expect("must parse");
9106 assert_eq!(req.model, "gpt-4o");
9107 assert_eq!(req.max_tokens, 256);
9108 assert_eq!(req.messages.len(), 1);
9109 assert_eq!(req.messages[0].role, "user");
9110 assert_eq!(req.messages[0].content, Value::String("hello".to_owned()));
9111 assert!(req.system.is_none());
9112 assert_eq!(req.raw, Value::Null);
9114 }
9115
9116 #[test]
9117 fn parse_openai_request_system_message() {
9118 let body = br#"{"model":"gpt-4o","messages":[{"role":"system","content":"be concise"},{"role":"user","content":"hi"}]}"#;
9119 let req = parse_openai_request(body, false).expect("must parse");
9120 assert_eq!(req.system.as_deref(), Some("be concise"));
9121 assert_eq!(req.messages.len(), 1);
9122 assert_eq!(req.messages[0].role, "user");
9123 }
9124
9125 #[test]
9126 fn parse_openai_request_tool_calls_translate_to_tool_use() {
9127 let body = br#"{
9128 "model":"gpt-4o",
9129 "messages":[
9130 {"role":"user","content":"what's the weather?"},
9131 {"role":"assistant","content":null,"tool_calls":[
9132 {"id":"call_1","type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Paris\"}"}}
9133 ]},
9134 {"role":"tool","tool_call_id":"call_1","content":"15C, cloudy"}
9135 ]
9136 }"#;
9137 let req = parse_openai_request(body, false).expect("must parse");
9138 assert_eq!(req.messages.len(), 3);
9140 let asst = &req.messages[1];
9142 assert_eq!(asst.role, "assistant");
9143 let blocks = asst.content.as_array().expect("content array");
9144 assert_eq!(blocks[0]["type"], "tool_use");
9145 assert_eq!(blocks[0]["name"], "get_weather");
9146 assert_eq!(blocks[0]["id"], "call_1");
9147 assert_eq!(blocks[0]["input"]["city"], "Paris");
9148 let tool_msg = &req.messages[2];
9150 assert_eq!(tool_msg.role, "user");
9151 let result_blocks = tool_msg.content.as_array().expect("result blocks");
9152 assert_eq!(result_blocks[0]["type"], "tool_result");
9153 assert_eq!(result_blocks[0]["tool_use_id"], "call_1");
9154 }
9155
9156 #[test]
9157 fn parse_openai_request_tools_translate_to_anthropic_format() {
9158 let body = br#"{
9159 "model":"gpt-4o",
9160 "messages":[{"role":"user","content":"use a tool"}],
9161 "tools":[{"type":"function","function":{"name":"search","description":"web search","parameters":{"type":"object","properties":{"q":{"type":"string"}}}}}]
9162 }"#;
9163 let req = parse_openai_request(body, false).expect("must parse");
9164 let tools = req.tools.as_array().expect("tools array");
9165 assert_eq!(tools.len(), 1);
9166 assert_eq!(tools[0]["name"], "search");
9167 assert_eq!(tools[0]["description"], "web search");
9168 assert_eq!(tools[0]["input_schema"]["type"], "object");
9169 }
9170
9171 #[test]
9172 fn parse_openai_request_raw_carry_preserves_full_body() {
9173 let body =
9175 br#"{"model":"gpt-4o","max_tokens":100,"messages":[{"role":"user","content":"hi"}]}"#;
9176 let req = parse_openai_request(body, true).expect("must parse");
9177 assert!(req.raw.is_object(), "raw must be the full JSON object");
9178 assert_eq!(req.raw["model"], "gpt-4o");
9179 assert_eq!(req.raw["max_tokens"], 100);
9180 assert!(req.tools.is_null(), "no tools in this request");
9182 }
9183
9184 #[test]
9185 fn parse_openai_request_http_image_returns_none() {
9186 let body = br#"{"model":"gpt-4o","messages":[{"role":"user","content":[
9188 {"type":"text","text":"describe this"},
9189 {"type":"image_url","image_url":{"url":"https://example.com/cat.png"}}
9190 ]}]}"#;
9191 let result = parse_openai_request(body, false);
9192 assert!(result.is_none(), "http image URL must fail translation");
9193 }
9194
9195 #[test]
9196 fn parse_openai_request_data_url_image_translates_to_anthropic_base64() {
9197 let body = br#"{"model":"gpt-4o","messages":[{"role":"user","content":[
9198 {"type":"text","text":"describe"},
9199 {"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgo="}}
9200 ]}]}"#;
9201 let req = parse_openai_request(body, false).expect("data URL must parse");
9202 let blocks = req.messages[0].content.as_array().expect("blocks");
9203 let img = blocks
9204 .iter()
9205 .find(|b| b["type"] == "image")
9206 .expect("image block");
9207 assert_eq!(img["source"]["type"], "base64");
9208 assert_eq!(img["source"]["media_type"], "image/png");
9209 assert_eq!(img["source"]["data"], "iVBORw0KGgo=");
9210 }
9211
9212 #[test]
9215 fn openai_response_json_renders_text_response() {
9216 let resp = ModelResponse {
9217 model: "gpt-4o".to_owned(),
9218 text: "Hello!".to_owned(),
9219 in_tokens: 10,
9220 cache_write_tokens: 0,
9221 cache_read_tokens: 0,
9222 out_tokens: 5,
9223 raw: serde_json::json!({
9224 "content": [{ "type": "text", "text": "Hello!" }]
9225 }),
9226 };
9227 let json = openai_response_json(&resp);
9228 assert_eq!(json["object"], "chat.completion");
9229 assert_eq!(json["model"], "gpt-4o");
9230 assert_eq!(json["choices"][0]["message"]["role"], "assistant");
9231 assert_eq!(json["choices"][0]["message"]["content"], "Hello!");
9232 assert_eq!(json["choices"][0]["finish_reason"], "stop");
9233 assert_eq!(json["usage"]["prompt_tokens"], 10);
9234 assert_eq!(json["usage"]["completion_tokens"], 5);
9235 }
9236
9237 #[test]
9238 fn openai_response_json_renders_tool_call() {
9239 let resp = ModelResponse {
9240 model: "gpt-4o".to_owned(),
9241 text: String::new(),
9242 in_tokens: 20,
9243 cache_write_tokens: 0,
9244 cache_read_tokens: 0,
9245 out_tokens: 15,
9246 raw: serde_json::json!({
9247 "content": [{
9248 "type": "tool_use",
9249 "id": "call_abc",
9250 "name": "search",
9251 "input": {"q": "Rust async"}
9252 }]
9253 }),
9254 };
9255 let json = openai_response_json(&resp);
9256 assert_eq!(json["choices"][0]["finish_reason"], "tool_calls");
9257 let tc = &json["choices"][0]["message"]["tool_calls"][0];
9258 assert_eq!(tc["id"], "call_abc");
9259 assert_eq!(tc["type"], "function");
9260 assert_eq!(tc["function"]["name"], "search");
9261 assert_eq!(json["choices"][0]["message"]["content"], Value::Null);
9263 }
9264
9265 #[test]
9266 fn openai_sse_from_message_plain_text_has_role_then_content_then_stop() {
9267 let resp = ModelResponse {
9268 model: "gpt-4o".to_owned(),
9269 text: "Hi there!".to_owned(),
9270 in_tokens: 5,
9271 cache_write_tokens: 0,
9272 cache_read_tokens: 0,
9273 out_tokens: 3,
9274 raw: serde_json::json!({
9275 "content": [{ "type": "text", "text": "Hi there!" }]
9276 }),
9277 };
9278 let sse = openai_sse_from_message(&openai_response_json(&resp), false);
9279 for line in sse.lines() {
9281 assert!(
9282 line.is_empty() || line.starts_with("data: "),
9283 "bad SSE line: {line:?}"
9284 );
9285 }
9286 let frames: Vec<&str> = sse
9287 .lines()
9288 .filter_map(|l| l.strip_prefix("data: "))
9289 .collect();
9290 assert_eq!(*frames.last().unwrap(), "[DONE]");
9292 let role_frame: Value = serde_json::from_str(frames[0]).unwrap();
9294 assert_eq!(role_frame["choices"][0]["delta"]["role"], "assistant");
9295 assert!(frames.iter().any(|f| {
9297 if *f == "[DONE]" {
9298 return false;
9299 }
9300 serde_json::from_str::<Value>(f)
9301 .ok()
9302 .is_some_and(|v| v["choices"][0]["delta"]["content"] == "Hi there!")
9303 }));
9304 assert!(frames.iter().any(|f| {
9306 if *f == "[DONE]" {
9307 return false;
9308 }
9309 serde_json::from_str::<Value>(f)
9310 .ok()
9311 .is_some_and(|v| v["choices"][0]["finish_reason"] == "stop")
9312 }));
9313 }
9314
9315 #[test]
9318 fn detects_openai_tool_calls() {
9319 let with_tool_calls = Bytes::from_static(br#"{"messages":[
9320 {"role":"user","content":"hi"},
9321 {"role":"assistant","content":null,"tool_calls":[{"id":"c1","type":"function","function":{"name":"f","arguments":"{}"}}]}
9322 ]}"#);
9323 let without = Bytes::from_static(br#"{"messages":[{"role":"user","content":"hi"}]}"#);
9324 let with_tool_msg = Bytes::from_static(
9325 br#"{"messages":[
9326 {"role":"tool","tool_call_id":"c1","content":"result"}
9327 ]}"#,
9328 );
9329 assert!(openai_messages_have_tool_calls(&with_tool_calls));
9330 assert!(!openai_messages_have_tool_calls(&without));
9331 assert!(openai_messages_have_tool_calls(&with_tool_msg));
9332 }
9333
9334 #[test]
9335 fn detects_openai_http_images() {
9336 let http_img = Bytes::from_static(
9337 br#"{"messages":[{"role":"user","content":[
9338 {"type":"image_url","image_url":{"url":"https://example.com/img.png"}}
9339 ]}]}"#,
9340 );
9341 let data_img = Bytes::from_static(
9342 br#"{"messages":[{"role":"user","content":[
9343 {"type":"image_url","image_url":{"url":"data:image/png;base64,abc"}}
9344 ]}]}"#,
9345 );
9346 let no_img = Bytes::from_static(br#"{"messages":[{"role":"user","content":"hi"}]}"#);
9347 assert!(openai_has_http_images(&http_img));
9348 assert!(!openai_has_http_images(&data_img));
9349 assert!(!openai_has_http_images(&no_img));
9350 }
9351
9352 #[test]
9353 fn enforce_can_handle_openai_inbound_all_openai_ladder() {
9354 let tools_body = Bytes::from_static(br#"{"model":"gpt-4o","messages":[{"role":"assistant","content":null,"tool_calls":[{"id":"c","type":"function","function":{"name":"f","arguments":"{}"}}]}]}"#);
9356 let f = extract_openai_features(&HeaderMap::new(), &tools_body);
9357 let ladder = vec!["openai/gpt-4o-mini".to_owned(), "openai/gpt-4o".to_owned()];
9358 let providers = crate::provider::ProviderRegistry::new("http://x", "http://x");
9359 assert!(enforce_can_handle(
9360 &f,
9361 &tools_body,
9362 true,
9363 &ladder,
9364 &providers,
9365 Dialect::Openai
9366 ));
9367 }
9368
9369 #[test]
9370 fn enforce_can_handle_openai_inbound_all_anthropic_ladder_no_http_image() {
9371 let tools_body = Bytes::from_static(br#"{"model":"gpt-4o","messages":[{"role":"assistant","content":null,"tool_calls":[{"id":"c","type":"function","function":{"name":"f","arguments":"{}"}}]}]}"#);
9373 let f = extract_openai_features(&HeaderMap::new(), &tools_body);
9374 let ladder = vec!["anthropic/claude-haiku-4-5".to_owned()];
9375 let providers = crate::provider::ProviderRegistry::new("http://x", "http://x");
9376 assert!(enforce_can_handle(
9377 &f,
9378 &tools_body,
9379 true,
9380 &ladder,
9381 &providers,
9382 Dialect::Openai,
9383 ));
9384 }
9385
9386 #[test]
9387 fn enforce_can_handle_openai_inbound_http_image_falls_back() {
9388 let img_body = Bytes::from_static(
9390 br#"{"model":"gpt-4o","messages":[{"role":"user","content":[
9391 {"type":"image_url","image_url":{"url":"https://example.com/img.png"}}
9392 ]}]}"#,
9393 );
9394 let f = extract_openai_features(&HeaderMap::new(), &img_body);
9395 let ladder = vec!["anthropic/claude-haiku-4-5".to_owned()];
9396 let providers = crate::provider::ProviderRegistry::new("http://x", "http://x");
9397 assert!(!enforce_can_handle(
9398 &f,
9399 &img_body,
9400 true,
9401 &ladder,
9402 &providers,
9403 Dialect::Openai,
9404 ));
9405 }
9406
9407 fn openai_enforce_state(mock_resp: ModelResponse) -> AppState {
9411 let toml = "[[price]]\nmodel = \"mock/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"mock/m\"]\ngates = [\"non-empty\"]\n";
9412 let config = ProxyConfig::from_lookup(|k| match k {
9413 "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
9414 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
9415 _ => None,
9416 })
9417 .unwrap();
9418 let mut outs = HashMap::new();
9419 outs.insert("mock/m".to_owned(), Ok(mock_resp));
9420 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
9421 map.insert("mock".to_owned(), Arc::new(MockProvider::new("mock", outs)));
9422 let (traces, _rx) = mpsc::channel(64);
9423 std::mem::forget(_rx);
9424 let tenant_rate_limiter = build_tenant_rate_limiter(&config);
9425 AppState {
9426 config: Arc::new(config),
9427 http: reqwest::Client::new(),
9428 providers: ProviderRegistry::from_map(map),
9429 gate_health: Arc::new(GateHealthRegistry::new()),
9430 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
9431 paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
9432 experiments: Arc::default(),
9433 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
9434 traces,
9435 adaptive: None,
9436 eprocess: None,
9437 bandit: None,
9438 promoter: None,
9439 verified_cache: None,
9440 predictor: None,
9441 prior: None,
9442 tenant_rate_limiter,
9443 spill: None,
9444 }
9445 }
9446
9447 #[tokio::test]
9448 async fn chat_completions_plain_text_enforce_returns_openai_shape() {
9449 let mock = model_resp("mock/m", "gated answer");
9450 let state = openai_enforce_state(mock);
9451 let body = Bytes::from_static(
9452 br#"{"model":"gpt-4o","messages":[{"role":"user","content":"hello"}]}"#,
9453 );
9454 let resp = chat_completions(
9455 State(state),
9456 Extension(TenantId("default".to_owned())),
9457 HeaderMap::new(),
9458 body,
9459 )
9460 .await;
9461 assert_eq!(resp.status(), axum::http::StatusCode::OK);
9462 let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
9463 .await
9464 .unwrap();
9465 let json: Value = serde_json::from_slice(&bytes).expect("must be JSON");
9466 assert_eq!(json["object"], "chat.completion");
9467 assert_eq!(json["choices"][0]["message"]["role"], "assistant");
9468 assert_eq!(json["choices"][0]["message"]["content"], "gated answer");
9469 assert_eq!(json["choices"][0]["finish_reason"], "stop");
9470 assert!(
9471 json["id"]
9472 .as_str()
9473 .is_some_and(|id| id.starts_with("chatcmpl-"))
9474 );
9475 }
9476
9477 #[tokio::test]
9478 async fn chat_completions_stream_true_returns_sse_with_openai_chunks() {
9479 let mock = model_resp("mock/m", "gated answer");
9480 let state = openai_enforce_state(mock);
9481 let body = Bytes::from_static(
9482 br#"{"model":"gpt-4o","stream":true,"messages":[{"role":"user","content":"hello"}]}"#,
9483 );
9484 let resp = chat_completions(
9485 State(state),
9486 Extension(TenantId("default".to_owned())),
9487 HeaderMap::new(),
9488 body,
9489 )
9490 .await;
9491 assert_eq!(resp.status(), axum::http::StatusCode::OK);
9492 assert!(
9493 resp.headers()
9494 .get(axum::http::header::CONTENT_TYPE)
9495 .and_then(|v| v.to_str().ok())
9496 .is_some_and(|ct| ct.starts_with("text/event-stream")),
9497 "stream:true must return SSE"
9498 );
9499 let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
9500 .await
9501 .unwrap();
9502 let text = String::from_utf8(bytes.to_vec()).unwrap();
9503 assert!(
9505 text.contains("chat.completion.chunk"),
9506 "must have OpenAI chunk frames"
9507 );
9508 assert!(text.contains("[DONE]"), "must end with [DONE]");
9509 assert!(
9510 text.contains("gated answer"),
9511 "content must be in the stream"
9512 );
9513 assert!(
9515 !text.contains("message_start"),
9516 "must not have Anthropic event types"
9517 );
9518 }
9519
9520 fn probe_state(
9524 sample_rate: f64,
9525 k: u32,
9526 outcomes: Vec<(&str, Result<ModelResponse, ProviderError>)>,
9527 ) -> (AppState, mpsc::Receiver<Trace>) {
9528 let toml = format!(
9529 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\nladder = [\"anthropic/claude-haiku-4-5\"]\ngates = [\"non-empty\"]\n\
9530 [escalation.probe]\nk = {k}\nsample_rate = {sample_rate}\n"
9531 );
9532 let config = ProxyConfig::from_lookup(|k_| match k_ {
9533 "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
9534 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
9535 _ => None,
9536 })
9537 .unwrap();
9538 let mut outs = HashMap::new();
9539 for (model, out) in outcomes {
9540 outs.insert(model.to_owned(), out);
9541 }
9542 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
9543 map.insert(
9544 "anthropic".to_owned(),
9545 Arc::new(MockProvider::new("anthropic", outs)),
9546 );
9547 let (traces, rx) = mpsc::channel(64);
9548 let state = AppState {
9549 config: Arc::new(config),
9550 http: reqwest::Client::new(),
9551 providers: ProviderRegistry::from_map(map),
9552 gate_health: Arc::new(GateHealthRegistry::new()),
9553 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
9554 paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
9555 experiments: Arc::default(),
9556 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
9557 traces,
9558 adaptive: None,
9559 eprocess: None,
9560 bandit: None,
9561 promoter: None,
9562 verified_cache: None,
9563 predictor: None,
9564 prior: None,
9565 tenant_rate_limiter: None,
9566 spill: None,
9567 };
9568 (state, rx)
9569 }
9570
9571 async fn run_enforce_get_trace(state: AppState, mut rx: mpsc::Receiver<Trace>) -> Trace {
9573 let resp = messages(
9574 State(state),
9575 Extension(TenantId("default".to_owned())),
9576 HeaderMap::new(),
9577 user_body(),
9578 )
9579 .await;
9580 assert_eq!(resp.status(), axum::http::StatusCode::OK);
9581 rx.try_recv().expect("trace must be enqueued")
9582 }
9583
9584 #[tokio::test]
9586 async fn probe_off_trace_has_no_probe_field() {
9587 let (state, rx) = enforce_state(
9588 &["anthropic/claude-haiku-4-5"],
9589 &["non-empty"],
9590 vec![(
9591 "anthropic/claude-haiku-4-5",
9592 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
9593 )],
9594 );
9595 assert!(
9596 state
9597 .config
9598 .routing
9599 .as_ref()
9600 .unwrap()
9601 .escalation
9602 .probe
9603 .is_none(),
9604 "probe must default to None"
9605 );
9606 let trace = run_enforce_get_trace(state, rx).await;
9607 assert!(
9608 trace.probe.is_none(),
9609 "probe=None config must not set trace.probe"
9610 );
9611 }
9612
9613 #[tokio::test]
9615 async fn probe_sample_rate_zero_never_fires() {
9616 let (state, rx) = probe_state(
9618 0.0,
9619 5,
9620 vec![(
9621 "anthropic/claude-haiku-4-5",
9622 Ok(model_resp("anthropic/claude-haiku-4-5", "hi")),
9623 )],
9624 );
9625 let trace = run_enforce_get_trace(state, rx).await;
9626 assert!(
9627 trace.probe.is_none(),
9628 "sample_rate=0.0 must never set trace.probe"
9629 );
9630 }
9631
9632 #[tokio::test]
9635 async fn probe_on_all_pass_sets_confident_pass() {
9636 let model = "anthropic/claude-haiku-4-5";
9638 let (state, mut rx) = probe_state(1.0, 3, vec![(model, Ok(model_resp(model, "hello")))]);
9639 let resp = messages(
9640 State(state),
9641 Extension(TenantId("default".to_owned())),
9642 HeaderMap::new(),
9643 user_body(),
9644 )
9645 .await;
9646 assert_eq!(resp.status(), axum::http::StatusCode::OK);
9647 let json = body_json(resp).await;
9649 assert_eq!(
9650 json["content"][0]["text"], "hello",
9651 "served content unchanged"
9652 );
9653
9654 let trace = rx.try_recv().expect("trace enqueued");
9655 let sig = trace.probe.expect("probe must be set when sample_rate=1.0");
9656 assert_eq!(sig.k, 3);
9657 assert_eq!(sig.gate_pass_count, 3, "all 3 samples must pass non-empty");
9658 assert_eq!(
9659 sig.regime,
9660 firstpass_core::ProbeRegime::ConfidentPass,
9661 "all-pass → ConfidentPass"
9662 );
9663 assert!(
9664 sig.probe_cost_usd > 0.0,
9665 "k model calls must cost something"
9666 );
9667 }
9668
9669 #[tokio::test]
9672 async fn probe_on_all_fail_sets_confident_fail() {
9673 let model = "anthropic/claude-haiku-4-5";
9674 let (state, mut rx) = probe_state(1.0, 3, vec![(model, Ok(model_resp(model, "")))]);
9676 let resp = messages(
9678 State(state),
9679 Extension(TenantId("default".to_owned())),
9680 HeaderMap::new(),
9681 user_body(),
9682 )
9683 .await;
9684 assert_eq!(resp.status(), axum::http::StatusCode::OK);
9685
9686 let trace = rx.try_recv().expect("trace enqueued");
9687 let sig = trace.probe.expect("probe must be set when sample_rate=1.0");
9688 assert_eq!(
9689 sig.gate_pass_count, 0,
9690 "empty response fails non-empty: all 0 pass"
9691 );
9692 assert_eq!(
9693 sig.regime,
9694 firstpass_core::ProbeRegime::ConfidentFail,
9695 "0 passes → ConfidentFail"
9696 );
9697 }
9698
9699 #[tokio::test]
9702 async fn probe_on_served_output_identical_to_probe_off() {
9703 let model = "anthropic/claude-haiku-4-5";
9704 let mk = |sample_rate: f64| {
9705 probe_state(
9706 sample_rate,
9707 2,
9708 vec![(model, Ok(model_resp(model, "gated answer")))],
9709 )
9710 };
9711
9712 let (state_off, mut rx_off) = mk(0.0);
9714 let resp_off = messages(
9715 State(state_off),
9716 Extension(TenantId("default".to_owned())),
9717 HeaderMap::new(),
9718 user_body(),
9719 )
9720 .await;
9721 let json_off = body_json(resp_off).await;
9722 let trace_off = rx_off.try_recv().unwrap();
9723
9724 let (state_on, mut rx_on) = mk(1.0);
9726 let resp_on = messages(
9727 State(state_on),
9728 Extension(TenantId("default".to_owned())),
9729 HeaderMap::new(),
9730 user_body(),
9731 )
9732 .await;
9733 let json_on = body_json(resp_on).await;
9734 let trace_on = rx_on.try_recv().unwrap();
9735
9736 assert_eq!(
9738 json_off["content"][0]["text"], json_on["content"][0]["text"],
9739 "served text must be identical regardless of probe"
9740 );
9741 assert!(
9743 (trace_off.final_.total_cost_usd - trace_on.final_.total_cost_usd).abs() < 1e-12,
9744 "total_cost_usd must not include probe cost: off={} on={}",
9745 trace_off.final_.total_cost_usd,
9746 trace_on.final_.total_cost_usd
9747 );
9748 assert!(trace_off.probe.is_none());
9750 assert!(trace_on.probe.is_some());
9751 assert!(
9753 trace_on.probe.as_ref().unwrap().probe_cost_usd > 0.0,
9754 "probe cost must be positive"
9755 );
9756 }
9757
9758 #[tokio::test]
9768 async fn probe_does_not_mutate_gate_health() {
9769 let model = "anthropic/claude-haiku-4-5";
9770 let (mut state, rx) = probe_state(1.0, 2, vec![(model, Ok(model_resp(model, "answer")))]);
9771
9772 let registry = GateHealthRegistry::new().with_budget("non-empty", 2, 0.4);
9775 registry.record("default", "non-empty", true);
9778 assert!(
9779 registry.enabled("default", "non-empty"),
9780 "gate must start enabled (window not full yet)"
9781 );
9782 state.gate_health = Arc::new(registry);
9783
9784 let trace = run_enforce_get_trace(state, rx).await;
9796 let sig = trace.probe.expect("probe must fire with sample_rate=1.0");
9797 assert_eq!(sig.k, 2);
9798 assert!(
9799 sig.gate_pass_count <= 2,
9800 "gate_pass_count must be in [0, k]"
9801 );
9802 }
9803
9804 fn predictor_state(
9806 enabled: bool,
9807 outcome: Result<ModelResponse, ProviderError>,
9808 ) -> (AppState, mpsc::Receiver<Trace>) {
9809 let toml = "[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/claude-haiku-4-5\"]\ngates = [\"non-empty\"]\n";
9810 let config = ProxyConfig::from_lookup(|k_| match k_ {
9811 "FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
9812 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
9813 _ => None,
9814 })
9815 .unwrap();
9816 let mut outs = HashMap::new();
9817 outs.insert("anthropic/claude-haiku-4-5".to_owned(), outcome);
9818 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
9819 map.insert(
9820 "anthropic".to_owned(),
9821 Arc::new(MockProvider::new("anthropic", outs)),
9822 );
9823 let (traces, rx) = mpsc::channel(64);
9824 let predictor = enabled.then(|| {
9825 Arc::new(std::sync::Mutex::new(firstpass_core::PassPredictor::new(
9826 0.05, 1e-4,
9827 )))
9828 });
9829 let state = AppState {
9830 config: Arc::new(config),
9831 http: reqwest::Client::new(),
9832 providers: ProviderRegistry::from_map(map),
9833 gate_health: Arc::new(GateHealthRegistry::new()),
9834 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
9835 paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
9836 experiments: Arc::default(),
9837 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
9838 traces,
9839 adaptive: None,
9840 eprocess: None,
9841 bandit: None,
9842 promoter: None,
9843 verified_cache: None,
9844 predictor,
9845 prior: None,
9846 tenant_rate_limiter: None,
9847 spill: None,
9848 };
9849 (state, rx)
9850 }
9851
9852 #[tokio::test]
9853 async fn predictor_off_leaves_predicted_pass_none() {
9854 let (state, rx) =
9855 predictor_state(false, Ok(model_resp("anthropic/claude-haiku-4-5", "ok")));
9856 let trace = run_enforce_get_trace(state, rx).await;
9857 assert!(
9858 trace.predicted_pass.is_none(),
9859 "predictor off => no field (byte-identical)"
9860 );
9861 let j = serde_json::to_string(&trace).unwrap();
9863 assert!(!j.contains("predicted_pass"), "None must be omitted: {j}");
9864 }
9865
9866 #[tokio::test]
9867 async fn predictor_on_records_shadow_prediction_and_serves_identically() {
9868 let (state_off, rx_off) = predictor_state(
9871 false,
9872 Ok(model_resp("anthropic/claude-haiku-4-5", "served answer")),
9873 );
9874 let off = run_enforce_get_trace(state_off, rx_off).await;
9875
9876 let (state_on, rx_on) = predictor_state(
9877 true,
9878 Ok(model_resp("anthropic/claude-haiku-4-5", "served answer")),
9879 );
9880 let on = run_enforce_get_trace(state_on, rx_on).await;
9881
9882 assert_eq!(
9883 on.final_.served_rung, off.final_.served_rung,
9884 "served rung identical"
9885 );
9886 assert_eq!(on.attempts.len(), off.attempts.len(), "same attempts");
9887 assert_eq!(
9888 on.final_.total_cost_usd, off.final_.total_cost_usd,
9889 "predictor never adds served cost"
9890 );
9891 let p = on
9892 .predicted_pass
9893 .expect("predictor on => predicted_pass recorded");
9894 assert!(p > 0.0 && p < 1.0, "shadow prediction in (0,1): {p}");
9895 }
9896
9897 fn rollout_state(
9899 percent: f64,
9900 key: &str,
9901 outcomes: Vec<(&str, Result<ModelResponse, ProviderError>)>,
9902 ) -> (AppState, mpsc::Receiver<Trace>) {
9903 let toml = format!(
9904 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
9905 ladder = [\"anthropic/claude-haiku-4-5\"]\ngates = [\"non-empty\"]\n\
9906 [route.rollout]\npercent = {percent}\nkey = \"{key}\"\n"
9907 );
9908 let config = ProxyConfig::from_lookup(|k| match k {
9909 "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
9910 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
9911 _ => None,
9912 })
9913 .unwrap();
9914 let mut outs = HashMap::new();
9915 for (model, out) in outcomes {
9916 outs.insert(model.to_owned(), out);
9917 }
9918 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
9919 map.insert(
9920 "anthropic".to_owned(),
9921 Arc::new(MockProvider::new("anthropic", outs)),
9922 );
9923 let (traces, rx) = mpsc::channel(64);
9924 let state = AppState {
9925 config: Arc::new(config),
9926 http: reqwest::Client::new(),
9927 providers: ProviderRegistry::from_map(map),
9928 gate_health: Arc::new(GateHealthRegistry::new()),
9929 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
9930 paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
9931 experiments: Arc::default(),
9932 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
9933 traces,
9934 adaptive: None,
9935 eprocess: None,
9936 bandit: None,
9937 promoter: None,
9938 verified_cache: None,
9939 predictor: None,
9940 prior: None,
9941 tenant_rate_limiter: None,
9942 spill: None,
9943 };
9944 (state, rx)
9945 }
9946
9947 #[tokio::test]
9951 async fn rollout_at_zero_percent_never_enforces() {
9952 for i in 0..25 {
9953 let (state, mut rx) = rollout_state(
9954 0.0,
9955 "session",
9956 vec![(
9957 "anthropic/claude-haiku-4-5",
9958 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
9959 )],
9960 );
9961 let mut headers = HeaderMap::new();
9962 headers.insert("x-firstpass-session", format!("s{i}").parse().unwrap());
9963 let resp = messages(
9964 State(state),
9965 Extension(TenantId("default".to_owned())),
9966 headers,
9967 user_body(),
9968 )
9969 .await;
9970 let _ = resp;
9973 if let Ok(t) = rx.try_recv() {
9974 assert!(
9975 t.attempts.is_empty(),
9976 "0% rollout produced an enforced decision for session s{i}"
9977 );
9978 }
9979 }
9980 }
9981
9982 #[tokio::test]
9985 async fn rollout_at_hundred_percent_always_enforces() {
9986 for i in 0..25 {
9987 let (state, mut rx) = rollout_state(
9988 100.0,
9989 "session",
9990 vec![(
9991 "anthropic/claude-haiku-4-5",
9992 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
9993 )],
9994 );
9995 let mut headers = HeaderMap::new();
9996 headers.insert("x-firstpass-session", format!("s{i}").parse().unwrap());
9997 let resp = messages(
9998 State(state),
9999 Extension(TenantId("default".to_owned())),
10000 headers,
10001 user_body(),
10002 )
10003 .await;
10004 assert_eq!(resp.status(), axum::http::StatusCode::OK);
10005 let t = rx.try_recv().expect("trace enqueued");
10006 assert!(
10007 !t.attempts.is_empty(),
10008 "100% rollout skipped enforcement for session s{i}"
10009 );
10010 }
10011 }
10012
10013 #[tokio::test]
10021 async fn dispatch_arm_matches_the_predicted_arm_per_session() {
10022 let mut checked_enforced = 0;
10023 let mut checked_control = 0;
10024 for i in 0..24 {
10025 let session = format!("session-{i}");
10026 let (state, mut rx) = rollout_state(
10027 50.0,
10028 "session",
10029 vec![(
10030 "anthropic/claude-haiku-4-5",
10031 Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
10032 )],
10033 );
10034 let expected = firstpass_core::rollout::decide(
10037 &state.config.prompt_salt,
10038 &firstpass_core::Rollout {
10039 percent: 50.0,
10040 key: firstpass_core::RolloutKey::Session,
10041 },
10042 &session,
10043 )
10044 .enforced;
10045
10046 let mut headers = HeaderMap::new();
10047 headers.insert("x-firstpass-session", session.parse().unwrap());
10048 let _ = messages(
10049 State(state),
10050 Extension(TenantId("default".to_owned())),
10051 headers,
10052 user_body(),
10053 )
10054 .await;
10055 let observed = rx
10056 .try_recv()
10057 .map(|t| !t.attempts.is_empty())
10058 .unwrap_or(false);
10059
10060 assert_eq!(
10061 observed,
10062 expected,
10063 "{session}: dispatch put it in the {} arm, bucketing predicted the {} arm",
10064 if observed { "enforced" } else { "control" },
10065 if expected { "enforced" } else { "control" }
10066 );
10067 if expected {
10068 checked_enforced += 1;
10069 } else {
10070 checked_control += 1;
10071 }
10072 }
10073 assert!(
10075 checked_enforced > 0 && checked_control > 0,
10076 "test saw only one arm ({checked_enforced} enforced, {checked_control} control)"
10077 );
10078 }
10079
10080 const INCUMBENT: &str = "anthropic/claude-sonnet-5";
10083 const CHALLENGER: &str = "anthropic/claude-haiku-4-5";
10084
10085 fn challenger_state(percent: f64, db: &str) -> (AppState, mpsc::Receiver<Trace>) {
10088 let toml = format!(
10089 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
10090 ladder = [\"{INCUMBENT}\"]\ngates = [\"non-empty\"]\n\
10091 [route.challenger]\nladder = [\"{CHALLENGER}\"]\npercent = {percent}\n"
10092 );
10093 let db = db.to_owned();
10094 let config = ProxyConfig::from_lookup(|k| match k {
10095 "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
10096 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
10097 "FIRSTPASS_DB" => Some(db.clone()),
10098 _ => None,
10099 })
10100 .unwrap();
10101 let mut outs = HashMap::new();
10102 for m in [INCUMBENT, CHALLENGER] {
10103 outs.insert(m.to_owned(), Ok(model_resp(m, "answer")));
10104 }
10105 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
10106 map.insert(
10107 "anthropic".to_owned(),
10108 Arc::new(MockProvider::new("anthropic", outs)),
10109 );
10110 let experiments = Arc::new(crate::experiment::ExperimentMonitor::from_config(
10111 config.routing.as_ref().unwrap(),
10112 ));
10113 let (traces, rx) = mpsc::channel(64);
10114 let state = AppState {
10115 config: Arc::new(config),
10116 http: reqwest::Client::new(),
10117 providers: ProviderRegistry::from_map(map),
10118 gate_health: Arc::new(GateHealthRegistry::new()),
10119 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
10120 paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
10121 experiments,
10122 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
10123 traces,
10124 adaptive: None,
10125 eprocess: None,
10126 bandit: None,
10127 promoter: None,
10128 verified_cache: None,
10129 predictor: None,
10130 prior: None,
10131 tenant_rate_limiter: None,
10132 spill: None,
10133 };
10134 (state, rx)
10135 }
10136
10137 async fn serve_session(
10138 state: &AppState,
10139 rx: &mut mpsc::Receiver<Trace>,
10140 session: &str,
10141 ) -> Trace {
10142 let mut headers = HeaderMap::new();
10143 headers.insert("x-firstpass-session", session.parse().unwrap());
10144 let resp = messages(
10145 State(state.clone()),
10146 Extension(TenantId("default".to_owned())),
10147 headers,
10148 user_body(),
10149 )
10150 .await;
10151 assert_eq!(resp.status(), axum::http::StatusCode::OK);
10152 rx.try_recv().expect("trace enqueued")
10153 }
10154
10155 fn session_in_arm(state: &AppState, want: firstpass_core::Arm, nth: usize) -> String {
10157 let ch = state.config.routing.as_ref().unwrap().routes[0]
10158 .challenger
10159 .as_ref()
10160 .unwrap();
10161 (0..)
10162 .map(|i| format!("sess-{i}"))
10163 .filter(|k| ch.arm(&state.config.prompt_salt, k) == want)
10164 .nth(nth)
10165 .unwrap()
10166 }
10167
10168 #[tokio::test]
10171 async fn challenger_arm_serves_its_ladder_and_both_arms_are_recorded() {
10172 use firstpass_core::Arm;
10173 let (state, mut rx) = challenger_state(50.0, "unused.db");
10174 for want in [Arm::Challenger, Arm::Incumbent] {
10175 for n in 0..5 {
10176 let session = session_in_arm(&state, want, n);
10177 let t = serve_session(&state, &mut rx, &session).await;
10178 let rec = t.experiment.as_ref().expect("experiment recorded");
10179 assert_eq!(rec.arm, want, "{session}");
10180 assert!((rec.percent - 50.0).abs() < f64::EPSILON);
10181 let expect_model = if want == Arm::Challenger {
10182 CHALLENGER
10183 } else {
10184 INCUMBENT
10185 };
10186 assert_eq!(t.attempts[0].model, expect_model, "{session}");
10187 assert_eq!(t.route_ix, Some(0));
10188 assert!(
10189 t.policy.id == "challenger-ladder@v0" || want == Arm::Incumbent,
10190 "challenger arm must not claim the incumbent's start-rung policy: {}",
10191 t.policy.id
10192 );
10193 }
10194 }
10195 let reports = state.experiments.reports();
10196 assert_eq!(reports[0].report.challenger.requests, 5);
10197 assert_eq!(reports[0].report.incumbent.requests, 5);
10198 }
10199
10200 #[tokio::test]
10203 async fn halted_experiment_stops_serving_the_challenger() {
10204 use firstpass_core::{Arm, ExperimentStatus, Verdict};
10205 let (state, mut rx) = challenger_state(50.0, "unused.db");
10206 for i in 0..500 {
10207 state.experiments.observe_outcome(
10208 "default",
10209 0,
10210 state.experiments.fingerprint(0).unwrap(),
10211 Arm::Incumbent,
10212 &format!("i{i}"),
10213 Verdict::Pass,
10214 );
10215 state.experiments.observe_outcome(
10216 "default",
10217 0,
10218 state.experiments.fingerprint(0).unwrap(),
10219 Arm::Challenger,
10220 &format!("c{i}"),
10221 Verdict::Fail,
10222 );
10223 }
10224 assert_eq!(
10225 state.experiments.status("default", 0),
10226 ExperimentStatus::Halted
10227 );
10228 let session = session_in_arm(&state, Arm::Challenger, 0);
10229 let t = serve_session(&state, &mut rx, &session).await;
10230 assert_eq!(
10231 t.attempts[0].model, INCUMBENT,
10232 "halted challenger was still served"
10233 );
10234 assert_eq!(t.experiment.unwrap().arm, Arm::Incumbent);
10235 }
10236
10237 #[tokio::test]
10247 async fn shared_log_wiring_halt_propagates_to_replica_b() {
10248 use firstpass_core::{Arm, ExperimentStatus, Verdict};
10249
10250 let shared_log = Arc::new(crate::experiment::InMemoryExperimentLog::new());
10251
10252 let (mut state_a, _rx_a) = challenger_state(50.0, "unused-a.db");
10255 {
10256 let monitor = crate::experiment::ExperimentMonitor::from_config(
10257 state_a.config.routing.as_ref().unwrap(),
10258 )
10259 .with_log(Arc::clone(&shared_log) as Arc<dyn crate::experiment::ExperimentLogBackend>);
10260 state_a.experiments = Arc::new(monitor);
10261 }
10262
10263 let (mut state_b, mut rx_b) = challenger_state(50.0, "unused-b.db");
10266 {
10267 let monitor = crate::experiment::ExperimentMonitor::from_config(
10268 state_b.config.routing.as_ref().unwrap(),
10269 )
10270 .with_log(Arc::clone(&shared_log) as Arc<dyn crate::experiment::ExperimentLogBackend>);
10271 state_b.experiments = Arc::new(monitor);
10272 }
10273
10274 let fp = state_a.experiments.fingerprint(0).unwrap().to_owned();
10275
10276 let mut halted = false;
10279 for i in 0..500 {
10280 state_a
10281 .experiments
10282 .record_outcome(
10283 "default",
10284 0,
10285 &fp,
10286 Arm::Incumbent,
10287 &format!("i{i}"),
10288 Verdict::Pass,
10289 0.10,
10290 )
10291 .await;
10292 state_a
10293 .experiments
10294 .record_outcome(
10295 "default",
10296 0,
10297 &fp,
10298 Arm::Challenger,
10299 &format!("c{i}"),
10300 Verdict::Fail,
10301 0.02,
10302 )
10303 .await;
10304 if state_a.experiments.status("default", 0) == ExperimentStatus::Halted {
10305 halted = true;
10306 break;
10307 }
10308 }
10309 assert!(halted, "experiment never halted on replica A");
10310
10311 assert!(
10313 !state_b.experiments.challenger_blocked("default", 0),
10314 "replica B must not block before syncing"
10315 );
10316
10317 let _ = state_b
10319 .experiments
10320 .sync_from_backend("default", 0, &fp)
10321 .await;
10322 assert!(
10323 state_b.experiments.challenger_blocked("default", 0),
10324 "replica B must block the challenger after syncing the halt from the shared log"
10325 );
10326
10327 let ch_session = session_in_arm(&state_b, Arm::Challenger, 999);
10330 let t = serve_session(&state_b, &mut rx_b, &ch_session).await;
10331 assert_eq!(
10332 t.experiment.as_ref().unwrap().arm,
10333 Arm::Incumbent,
10334 "a halted challenger-arm session must be rerouted to the incumbent on replica B"
10335 );
10336 assert_eq!(
10337 t.attempts[0].model, INCUMBENT,
10338 "the incumbent model must have been called, not the challenger"
10339 );
10340 }
10341
10342 #[tokio::test]
10345 async fn unavailable_experiment_history_serves_the_incumbent_only() {
10346 use firstpass_core::Arm;
10347 let (mut state, mut rx) = challenger_state(50.0, "unused.db");
10348 state.experiments = Arc::new(crate::experiment::ExperimentMonitor::unavailable(
10349 state.config.routing.as_ref().unwrap(),
10350 ));
10351 for n in 0..3 {
10352 let session = session_in_arm(&state, Arm::Challenger, n);
10353 let t = serve_session(&state, &mut rx, &session).await;
10354 assert_eq!(t.attempts[0].model, INCUMBENT);
10355 assert_eq!(t.experiment.unwrap().arm, Arm::Incumbent);
10356 }
10357 }
10358
10359 #[tokio::test]
10362 async fn challenger_arm_cannot_share_the_incumbents_cache_entry() {
10363 use firstpass_core::Arm;
10364 let (mut state, mut rx) = challenger_state(50.0, "unused.db");
10365 state.verified_cache = Some(Arc::new(crate::verified_cache::VerifiedCache::new(
10366 Duration::from_secs(60),
10367 100,
10368 )));
10369 let a = serve_session(&state, &mut rx, &session_in_arm(&state, Arm::Incumbent, 0)).await;
10371 assert!(!a.attempts.is_empty());
10372 let b = serve_session(&state, &mut rx, &session_in_arm(&state, Arm::Challenger, 0)).await;
10374 assert!(
10375 !b.attempts.is_empty(),
10376 "challenger replayed the incumbent's cached answer"
10377 );
10378 assert_eq!(b.attempts[0].model, CHALLENGER);
10379 let c = serve_session(&state, &mut rx, &session_in_arm(&state, Arm::Incumbent, 1)).await;
10382 assert!(
10383 c.attempts.is_empty(),
10384 "incumbent should have been a cache hit"
10385 );
10386 assert_eq!(c.experiment.as_ref().unwrap().arm, Arm::Incumbent);
10387 assert_eq!(c.route_ix, Some(0));
10388 let d = serve_session(&state, &mut rx, &session_in_arm(&state, Arm::Challenger, 1)).await;
10390 assert!(d.attempts.is_empty(), "challenger should hit its own entry");
10391 assert_eq!(d.experiment.as_ref().unwrap().arm, Arm::Challenger);
10392 }
10393
10394 #[tokio::test]
10397 async fn feedback_feeds_the_experiment_once_per_session() {
10398 use firstpass_core::Arm;
10399 let db = std::env::temp_dir().join(format!("firstpass-exp-{}.db", Uuid::now_v7()));
10400 let (state, mut rx) = challenger_state(50.0, &db.to_string_lossy());
10401 let session = session_in_arm(&state, Arm::Challenger, 0);
10402 let t = serve_session(&state, &mut rx, &session).await;
10403 let trace_id = t.trace_id.to_string();
10404
10405 let (tx, handle) = crate::store::open(&db).unwrap();
10407 tx.try_send(t).unwrap();
10408 drop(tx);
10409 handle.await.unwrap();
10410
10411 let send = |verdict: &'static str| {
10412 let state = state.clone();
10413 let body = Bytes::from(
10414 serde_json::json!({
10415 "trace_id": trace_id, "gate_id": "tests", "verdict": verdict, "reporter": "ci"
10416 })
10417 .to_string(),
10418 );
10419 async move {
10420 feedback(
10421 State(state),
10422 Extension(TenantId("default".to_owned())),
10423 body,
10424 )
10425 .await
10426 }
10427 };
10428 assert_eq!(
10429 send("pass").await.status(),
10430 axum::http::StatusCode::ACCEPTED
10431 );
10432 assert_eq!(
10434 send("fail").await.status(),
10435 axum::http::StatusCode::ACCEPTED
10436 );
10437 assert_eq!(
10439 send("abstain").await.status(),
10440 axum::http::StatusCode::ACCEPTED
10441 );
10442
10443 let r = &state.experiments.reports()[0].report;
10444 assert_eq!(r.challenger.outcomes, 1);
10445 assert!(
10446 (r.challenger.outcome_sum - 1.0).abs() < f64::EPSILON,
10447 "first outcome (pass) counts"
10448 );
10449 assert_eq!(r.incumbent.outcomes, 0);
10450 let _ = std::fs::remove_file(&db);
10451 }
10452
10453 async fn spawn_fake_prior_server(body: serde_json::Value) -> String {
10459 use axum::routing::post;
10460 let app = axum::Router::new().route(
10461 "/v1/systemone",
10462 post(move || {
10463 let body = body.clone();
10464 async move { axum::Json(body) }
10465 }),
10466 );
10467 let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
10468 .await
10469 .expect("bind");
10470 let addr = listener.local_addr().expect("addr");
10471 tokio::spawn(async move {
10472 let _ = axum::serve(listener, app).await;
10473 });
10474 format!("http://{addr}")
10475 }
10476
10477 fn prior_client(base_url: String, rungs: &[&str]) -> crate::prior::PriorClient {
10478 let cfg = firstpass_core::PriorConfig {
10479 provider: "typesafe".to_owned(),
10480 model: "jev-latest".to_owned(),
10481 api_key_env: "TYPESAFE_API_KEY".to_owned(),
10482 base_url,
10483 timeout_ms: 2000,
10484 strength: 10.0,
10485 rungs: rungs.iter().map(|s| (*s).to_owned()).collect(),
10486 };
10487 crate::prior::PriorClient::new(reqwest::Client::new(), cfg, "test-key".to_owned())
10488 }
10489
10490 const OPUS: &str = "anthropic/claude-opus-4-8";
10491
10492 #[tokio::test]
10493 async fn confident_prior_starts_above_zero_and_records_decision_prior() {
10494 let base_url = spawn_fake_prior_server(serde_json::json!({
10495 "answers": { "rung": { "choice": "r2", "confidence": 0.9,
10496 "probabilities": { "r0": 0.05, "r1": 0.05, "r2": 0.9 } } }
10497 }))
10498 .await;
10499 let (mut state, rx) = enforce_state(
10500 &[
10501 "anthropic/claude-haiku-4-5",
10502 "anthropic/claude-sonnet-5",
10503 OPUS,
10504 ],
10505 &["non-empty"],
10506 vec![
10507 (
10508 "anthropic/claude-haiku-4-5",
10509 Ok(model_resp("anthropic/claude-haiku-4-5", "ok")),
10510 ),
10511 (
10512 "anthropic/claude-sonnet-5",
10513 Ok(model_resp("anthropic/claude-sonnet-5", "ok")),
10514 ),
10515 (OPUS, Ok(model_resp(OPUS, "ok"))),
10516 ],
10517 );
10518 state.prior = Some(Arc::new(prior_client(
10519 base_url,
10520 &[
10521 "anthropic/claude-haiku-4-5",
10522 "anthropic/claude-sonnet-5",
10523 OPUS,
10524 ],
10525 )));
10526 let trace = run_enforce_get_trace(state, rx).await;
10527
10528 assert!(
10529 !trace.attempts.is_empty(),
10530 "at least one attempt must be recorded"
10531 );
10532 assert!(
10533 trace.attempts[0].rung >= 1,
10534 "a confident hard prior (rung 0/1 nearly hopeless) must start above rung 0, started \
10535 at {}",
10536 trace.attempts[0].rung
10537 );
10538 let decision_prior = trace
10539 .decision_prior
10540 .as_ref()
10541 .expect("a successful prior fetch must record decision_prior");
10542 assert_eq!(decision_prior.len(), 3);
10543 assert!(
10544 decision_prior[2] > decision_prior[0],
10545 "cumulative prior must be monotone non-decreasing: {decision_prior:?}"
10546 );
10547 }
10548
10549 #[tokio::test]
10550 async fn prior_server_error_fails_open_start_rung_zero_no_decision_prior() {
10551 let unreachable = "http://127.0.0.1:1".to_owned();
10554 let (mut state, rx) = enforce_state(
10555 &["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"],
10556 &["non-empty"],
10557 vec![(
10558 "anthropic/claude-haiku-4-5",
10559 Ok(model_resp("anthropic/claude-haiku-4-5", "ok")),
10560 )],
10561 );
10562 state.prior = Some(Arc::new(prior_client(
10563 unreachable,
10564 &["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"],
10565 )));
10566 let trace = run_enforce_get_trace(state, rx).await;
10567
10568 assert_eq!(
10569 trace.attempts[0].rung, 0,
10570 "a failed prior fetch must fail open to the static start rung 0"
10571 );
10572 assert!(
10573 trace.decision_prior.is_none(),
10574 "a failed prior fetch must leave decision_prior unset"
10575 );
10576 }
10577
10578 #[tokio::test]
10581 async fn gate_failed_start_rung_is_never_served_with_prior_on() {
10582 let base_url = spawn_fake_prior_server(serde_json::json!({
10583 "answers": { "rung": { "choice": "r2", "confidence": 0.9,
10584 "probabilities": { "r0": 0.05, "r1": 0.05, "r2": 0.9 } } }
10585 }))
10586 .await;
10587 let (mut state, rx) = enforce_state(
10591 &[
10592 "anthropic/claude-haiku-4-5",
10593 "anthropic/claude-sonnet-5",
10594 OPUS,
10595 ],
10596 &["non-empty"],
10597 vec![
10598 (
10599 "anthropic/claude-haiku-4-5",
10600 Ok(model_resp("anthropic/claude-haiku-4-5", "")),
10601 ),
10602 (
10603 "anthropic/claude-sonnet-5",
10604 Ok(model_resp("anthropic/claude-sonnet-5", "")),
10605 ),
10606 (OPUS, Ok(model_resp(OPUS, "ok"))),
10607 ],
10608 );
10609 state.prior = Some(Arc::new(prior_client(
10610 base_url,
10611 &[
10612 "anthropic/claude-haiku-4-5",
10613 "anthropic/claude-sonnet-5",
10614 OPUS,
10615 ],
10616 )));
10617 let trace = run_enforce_get_trace(state, rx).await;
10618
10619 assert_eq!(
10620 trace.final_.served_rung,
10621 Some(2),
10622 "only the top rung passes the gate, so it must be what's served, regardless of \
10623 which rung the prior started at"
10624 );
10625 for attempt in &trace.attempts[..trace.attempts.len() - 1] {
10626 assert_eq!(
10627 attempt.verdict,
10628 firstpass_core::Verdict::Fail,
10629 "every attempt before the served one must be a recorded gate failure, not a \
10630 silently-served output"
10631 );
10632 }
10633 assert_eq!(
10634 trace.attempts.last().unwrap().verdict,
10635 firstpass_core::Verdict::Pass
10636 );
10637 }
10638
10639 fn ux_state(db: &str) -> AppState {
10643 let (mut state, _rx) = challenger_state(50.0, db);
10644 let hash_a = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
10647 let hash_b = crate::tenant_auth::TenantKeys::hash_key("key-b").unwrap();
10648 let keys = format!("{{\"tenant-a\": {hash_a:?}, \"tenant-b\": {hash_b:?}}}");
10649 let toml = format!(
10650 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
10651 ladder = [\"{INCUMBENT}\"]\ngates = [\"non-empty\"]\n\
10652 [route.challenger]\nladder = [\"{CHALLENGER}\"]\npercent = 50\n"
10653 );
10654 let db = db.to_owned();
10655 state.config = Arc::new(
10656 ProxyConfig::from_lookup(|k| match k {
10657 "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
10658 "FIRSTPASS_MODE" => Some("enforce".to_owned()),
10659 "FIRSTPASS_DB" => Some(db.clone()),
10660 "FIRSTPASS_REQUIRE_AUTH" => Some("true".to_owned()),
10661 "FIRSTPASS_TENANT_KEYS_JSON" => Some(keys.clone()),
10662 _ => None,
10663 })
10664 .unwrap(),
10665 );
10666 let (traces, rx) = mpsc::channel(256);
10668 std::mem::forget(rx);
10669 state.traces = traces;
10670 state
10671 }
10672
10673 #[tokio::test]
10676 async fn paired_ledger_is_seeded_from_todays_pairs_at_startup() {
10677 let db = std::env::temp_dir().join(format!("firstpass-ledger-{}.db", Uuid::now_v7()));
10678 let (tx, handle) = crate::store::open(&db).unwrap();
10679 drop(tx);
10680 handle.await.unwrap();
10681 let now = jiff::Timestamp::now();
10682 let row = |tenant: &str, usd: f64, ts: jiff::Timestamp| crate::store::ExperimentPairRow {
10683 tenant: tenant.to_owned(),
10684 route_ix: 0,
10685 fingerprint: "fp".to_owned(),
10686 trace_id: Uuid::now_v7(),
10687 x_inc: 1.0,
10688 x_ch: 1.0,
10689 challenger_cost_usd: usd,
10690 ch_error: false,
10691 ch_truncated: 0,
10692 ts,
10693 in_log: crate::store::InLogStatus::InLog,
10694 };
10695 let two_days_ago = now - jiff::SignedDuration::from_hours(48);
10696 for r in [
10697 row("tenant-a", 0.40, now),
10698 row("tenant-a", 0.70, now),
10699 row("tenant-a", 9.00, two_days_ago),
10700 row("tenant-b", 0.10, now),
10701 ] {
10702 crate::store::append_experiment_pair(&db, &r).unwrap();
10703 }
10704 let toml = format!(
10705 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
10706 ladder = [\"{INCUMBENT}\"]\ngates = [\"json-valid\"]\n\
10707 [route.challenger]\nladder = [\"{CHALLENGER}\"]\nmode = \"paired\"\n\
10708 max_usd_per_day = 1.0\n"
10709 );
10710 let db_s = db.to_string_lossy().into_owned();
10711 let cfg = ProxyConfig::from_lookup(|k| match k {
10712 "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
10713 "FIRSTPASS_DB" => Some(db_s.clone()),
10714 _ => None,
10715 })
10716 .unwrap();
10717 let ledger = build_paired_ledger(&cfg, false);
10718 assert!((ledger.spent_today("tenant-a", 0, now) - 1.10).abs() < 1e-9);
10719 assert!((ledger.spent_today("tenant-b", 0, now) - 0.10).abs() < 1e-9);
10720 assert!(
10721 !ledger.may_spend("tenant-a", 0, 1.0, now),
10722 "cap survives restart"
10723 );
10724 assert!(ledger.may_spend("tenant-b", 0, 1.0, now));
10725 let _ = std::fs::remove_file(&db);
10726 }
10727
10728 #[tokio::test]
10731 async fn build_paired_ledger_seeds_offline_pairs_when_shared_log_is_true() {
10732 let db =
10733 std::env::temp_dir().join(format!("firstpass-offline-ledger-{}.db", Uuid::now_v7()));
10734 let (tx, handle) = crate::store::open(&db).unwrap();
10735 drop(tx);
10736 handle.await.unwrap();
10737 let now = jiff::Timestamp::now();
10738 let toml = format!(
10739 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
10740 ladder = [\"{INCUMBENT}\"]\ngates = [\"json-valid\"]\n\
10741 [route.challenger]\nladder = [\"{CHALLENGER}\"]\nmode = \"paired\"\n\
10742 max_usd_per_day = 2.0\n"
10743 );
10744 let db_s = db.to_string_lossy().into_owned();
10745 let cfg = ProxyConfig::from_lookup(|k| match k {
10746 "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
10747 "FIRSTPASS_DB" => Some(db_s.clone()),
10748 _ => None,
10749 })
10750 .unwrap();
10751
10752 let offline_row = crate::store::ExperimentPairRow {
10754 tenant: "default".to_owned(),
10755 route_ix: 0,
10756 fingerprint: "fp".to_owned(),
10757 trace_id: Uuid::now_v7(),
10758 x_inc: 1.0,
10759 x_ch: 0.0,
10760 challenger_cost_usd: 0.7,
10761 ch_error: false,
10762 ch_truncated: 0,
10763 ts: now,
10764 in_log: crate::store::InLogStatus::NotInLog,
10765 };
10766 let log_row = crate::store::ExperimentPairRow {
10767 in_log: crate::store::InLogStatus::InLog,
10768 challenger_cost_usd: 0.4,
10769 trace_id: Uuid::now_v7(),
10770 ..offline_row.clone()
10771 };
10772 crate::store::append_experiment_pair(&db, &offline_row).unwrap();
10773 crate::store::append_experiment_pair(&db, &log_row).unwrap();
10774
10775 let ledger = build_paired_ledger(&cfg, true);
10777 let spent = ledger.spent_today("default", 0, now);
10778 assert!(
10779 (spent - 0.7).abs() < 1e-9,
10780 "only offline row seeded; got {spent}"
10781 );
10782
10783 let ledger2 = build_paired_ledger(&cfg, false);
10785 let spent2 = ledger2.spent_today("default", 0, now);
10786 assert!(
10787 (spent2 - 1.1).abs() < 1e-9,
10788 "both rows seeded with shared_log=false; got {spent2}"
10789 );
10790
10791 let _ = std::fs::remove_file(&db);
10792 }
10793
10794 #[tokio::test]
10797 async fn build_paired_ledger_unknown_rows_count_toward_spend_with_shared_log() {
10798 let db =
10799 std::env::temp_dir().join(format!("firstpass-unknown-ledger-{}.db", Uuid::now_v7()));
10800 let (tx, handle) = crate::store::open(&db).unwrap();
10801 drop(tx);
10802 handle.await.unwrap();
10803 let now = jiff::Timestamp::now();
10804 let toml = format!(
10805 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
10806 ladder = [\"{INCUMBENT}\"]\ngates = [\"json-valid\"]\n\
10807 [route.challenger]\nladder = [\"{CHALLENGER}\"]\nmode = \"paired\"\n\
10808 max_usd_per_day = 2.0\n"
10809 );
10810 let db_s = db.to_string_lossy().into_owned();
10811 let cfg = ProxyConfig::from_lookup(|k| match k {
10812 "FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
10813 "FIRSTPASS_DB" => Some(db_s.clone()),
10814 _ => None,
10815 })
10816 .unwrap();
10817
10818 let unknown_row = crate::store::ExperimentPairRow {
10819 tenant: "default".to_owned(),
10820 route_ix: 0,
10821 fingerprint: "fp".to_owned(),
10822 trace_id: Uuid::now_v7(),
10823 x_inc: 1.0,
10824 x_ch: 0.0,
10825 challenger_cost_usd: 0.6,
10826 ch_error: false,
10827 ch_truncated: 0,
10828 ts: now,
10829 in_log: crate::store::InLogStatus::Unknown,
10830 };
10831 let log_row = crate::store::ExperimentPairRow {
10832 in_log: crate::store::InLogStatus::InLog,
10833 challenger_cost_usd: 0.3,
10834 trace_id: Uuid::now_v7(),
10835 ..unknown_row.clone()
10836 };
10837 crate::store::append_experiment_pair(&db, &unknown_row).unwrap();
10838 crate::store::append_experiment_pair(&db, &log_row).unwrap();
10839
10840 let ledger = build_paired_ledger(&cfg, true);
10842 let spent = ledger.spent_today("default", 0, now);
10843 assert!(
10844 (spent - 0.6).abs() < 1e-9,
10845 "Unknown row seeded (conservative); InLog row skipped; got {spent}"
10846 );
10847
10848 let _ = std::fs::remove_file(&db);
10849 }
10850
10851 #[tokio::test]
10854 async fn experiments_endpoint_reports_paired_mode_and_tenant_spend() {
10855 use tower::ServiceExt;
10856 let db = std::env::temp_dir().join(format!("firstpass-ux-paired-{}.db", Uuid::now_v7()));
10857 let mut state = ux_state(&db.to_string_lossy());
10858 let toml = format!(
10859 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
10860 ladder = [\"{INCUMBENT}\"]\ngates = [\"json-valid\"]\n\
10861 [route.challenger]\nladder = [\"{CHALLENGER}\"]\nmode = \"paired\"\n\
10862 sample_rate = 0.25\nmax_usd_per_day = 2.0\n"
10863 );
10864 let mut cfg = (*state.config).clone();
10865 cfg.routing = Some(firstpass_core::Config::parse(&toml).expect("paired config parses"));
10866 state.config = Arc::new(cfg);
10867 let now = jiff::Timestamp::now();
10868 state.paired_ledger.debit("tenant-a", 0, 0.5, now);
10869 state.paired_ledger.debit("tenant-b", 0, 1.25, now);
10870 let router = app(state).expect("router");
10871
10872 let resp = router
10873 .clone()
10874 .oneshot(ux_get("/v1/experiments", Some("tenant-a.key-a")))
10875 .await
10876 .unwrap();
10877 assert_eq!(resp.status(), axum::http::StatusCode::OK);
10878 let a = body_json(resp).await;
10879 let c = &a["configured"][0];
10880 assert_eq!(c["mode"], "paired", "{a}");
10881 assert_eq!(c["paired"]["sample_rate"], 0.25);
10882 assert_eq!(c["paired"]["max_usd_per_day"], 2.0);
10883 assert_eq!(c["paired"]["spent_today_usd"], 0.5, "tenant-a's spend only");
10884
10885 let resp = router
10886 .oneshot(ux_get("/v1/experiments", Some("tenant-b.key-b")))
10887 .await
10888 .unwrap();
10889 let b = body_json(resp).await;
10890 assert_eq!(b["configured"][0]["paired"]["spent_today_usd"], 1.25);
10891 let _ = std::fs::remove_file(&db);
10892 }
10893
10894 #[tokio::test]
10898 async fn paired_challenger_that_fails_its_gate_is_recorded_as_a_failure() {
10899 use tower::ServiceExt;
10900 let db = std::env::temp_dir().join(format!("firstpass-paired-fail-{}.db", Uuid::now_v7()));
10901 let (tx, handle) = crate::store::open(&db).unwrap();
10903 drop(tx);
10904 handle.await.unwrap();
10905 let (mut state, _rx) = challenger_state(50.0, &db.to_string_lossy());
10906 let toml = format!(
10907 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
10908 ladder = [\"{INCUMBENT}\"]\ngates = [\"json-valid\"]\n\
10909 [route.challenger]\nladder = [\"{CHALLENGER}\"]\nmode = \"paired\"\n\
10910 max_usd_per_day = 5.0\n"
10911 );
10912 let mut cfg = (*state.config).clone();
10913 cfg.routing = Some(firstpass_core::Config::parse(&toml).expect("paired config parses"));
10914 state.experiments = Arc::new(crate::experiment::ExperimentMonitor::from_config(
10915 cfg.routing.as_ref().unwrap(),
10916 ));
10917 state.config = Arc::new(cfg);
10918 let mut outs = HashMap::new();
10919 outs.insert(
10920 INCUMBENT.to_owned(),
10921 Ok(model_resp(INCUMBENT, "{\"ok\": 1}")),
10922 );
10923 outs.insert(
10924 CHALLENGER.to_owned(),
10925 Ok(model_resp(CHALLENGER, "not json")),
10926 );
10927 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
10928 map.insert(
10929 "anthropic".to_owned(),
10930 Arc::new(MockProvider::new("anthropic", outs)),
10931 );
10932 state.providers = ProviderRegistry::from_map(map);
10933 let router = app(state).expect("router");
10934
10935 let req = axum::http::Request::builder()
10936 .method("POST")
10937 .uri("/v1/messages")
10938 .body(Body::from(user_body()))
10939 .unwrap();
10940 let resp = router.oneshot(req).await.unwrap();
10941 assert_eq!(resp.status(), axum::http::StatusCode::OK);
10942
10943 let mut pairs = Vec::new();
10945 for _ in 0..200 {
10946 pairs = crate::store::load_all_experiment_pairs(&db).unwrap_or_default();
10947 if !pairs.is_empty() {
10948 break;
10949 }
10950 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
10951 }
10952 assert_eq!(pairs.len(), 1, "one pair recorded");
10953 assert_eq!(pairs[0].x_inc, 1.0, "incumbent passed json-valid");
10954 assert_eq!(
10955 pairs[0].x_ch, 0.0,
10956 "challenger failed json-valid on every rung"
10957 );
10958 assert_eq!(
10959 pairs[0].in_log,
10960 crate::store::InLogStatus::NotInLog,
10961 "no shared log is configured, so the pair cannot be in one"
10962 );
10963 let _ = std::fs::remove_file(&db);
10964 }
10965
10966 #[tokio::test]
10969 async fn paired_challenger_error_is_a_failure_and_arm_health_is_reported() {
10970 use tower::ServiceExt;
10971 let db = std::env::temp_dir().join(format!("firstpass-paired-err-{}.db", Uuid::now_v7()));
10972 let (tx, handle) = crate::store::open(&db).unwrap();
10973 drop(tx);
10974 handle.await.unwrap();
10975 let (mut state, _rx) = challenger_state(50.0, &db.to_string_lossy());
10976 let toml = format!(
10977 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
10978 ladder = [\"{INCUMBENT}\"]\ngates = [\"json-valid\"]\n\
10979 [route.challenger]\nladder = [\"{CHALLENGER}\"]\nmode = \"paired\"\n\
10980 max_usd_per_day = 5.0\n"
10981 );
10982 let mut cfg = (*state.config).clone();
10983 cfg.routing = Some(firstpass_core::Config::parse(&toml).expect("paired config parses"));
10984 state.experiments = Arc::new(crate::experiment::ExperimentMonitor::from_config(
10985 cfg.routing.as_ref().unwrap(),
10986 ));
10987 state.config = Arc::new(cfg);
10988 let mut cut = model_resp(INCUMBENT, "{\"ok\": 1}");
10990 cut.raw = serde_json::json!({"stop_reason": "max_tokens"});
10991 let mut outs = HashMap::new();
10992 outs.insert(INCUMBENT.to_owned(), Ok(cut));
10993 outs.insert(
10994 CHALLENGER.to_owned(),
10995 Err(ProviderError::Transport("timed out".into())),
10996 );
10997 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
10998 map.insert(
10999 "anthropic".to_owned(),
11000 Arc::new(MockProvider::new("anthropic", outs)),
11001 );
11002 state.providers = ProviderRegistry::from_map(map);
11003 let monitor = state.experiments.clone();
11004 let router = app(state).expect("router");
11005
11006 let req = axum::http::Request::builder()
11007 .method("POST")
11008 .uri("/v1/messages")
11009 .body(Body::from(user_body()))
11010 .unwrap();
11011 let resp = router.oneshot(req).await.unwrap();
11012 assert_eq!(resp.status(), axum::http::StatusCode::OK);
11013
11014 let mut pairs = Vec::new();
11015 for _ in 0..200 {
11016 pairs = crate::store::load_all_experiment_pairs(&db).unwrap_or_default();
11017 if !pairs.is_empty() {
11018 break;
11019 }
11020 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
11021 }
11022 assert_eq!(
11023 pairs.len(),
11024 1,
11025 "the challenger's error still records a pair"
11026 );
11027 assert_eq!(pairs[0].x_inc, 1.0);
11028 assert_eq!(pairs[0].x_ch, 0.0, "an error is a failure");
11029 let health = monitor.reports()[0].health;
11030 assert_eq!(health.incumbent.truncated, 1, "{health:?}");
11031 assert_eq!(health.incumbent.errors, 0, "{health:?}");
11032 assert_eq!(health.challenger.errors, 1, "{health:?}");
11033 let _ = std::fs::remove_file(&db);
11034 }
11035
11036 #[derive(Debug)]
11038 struct FlakyReadLog {
11039 inner: crate::experiment::InMemoryExperimentLog,
11040 reads: std::sync::atomic::AtomicUsize,
11041 always_fail: bool,
11042 }
11043
11044 #[async_trait::async_trait]
11045 impl crate::experiment::ExperimentLogBackend for FlakyReadLog {
11046 async fn append(
11047 &self,
11048 tenant: &str,
11049 route_ix: usize,
11050 fingerprint: &str,
11051 record: crate::experiment::LogRecord,
11052 ) -> Result<u64, crate::experiment::LogError> {
11053 if self.always_fail {
11054 return Err(crate::experiment::LogError::Backend("down".into()));
11055 }
11056 self.inner
11057 .append(tenant, route_ix, fingerprint, record)
11058 .await
11059 }
11060 async fn load_since(
11061 &self,
11062 tenant: &str,
11063 route_ix: usize,
11064 fingerprint: &str,
11065 offset: u64,
11066 ) -> Result<Vec<Option<crate::experiment::LogRecord>>, crate::experiment::LogError>
11067 {
11068 let n = self.reads.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
11069 if self.always_fail || n > 0 {
11070 return Err(crate::experiment::LogError::Backend("down".into()));
11071 }
11072 self.inner
11073 .load_since(tenant, route_ix, fingerprint, offset)
11074 .await
11075 }
11076 async fn claim_session(&self, t: &str, r: usize, f: &str, s: &str) -> bool {
11077 self.inner.claim_session(t, r, f, s).await
11078 }
11079 }
11080
11081 fn flaky_log(always_fail: bool) -> Arc<dyn crate::experiment::ExperimentLogBackend> {
11082 Arc::new(FlakyReadLog {
11083 inner: crate::experiment::InMemoryExperimentLog::new(),
11084 reads: std::sync::atomic::AtomicUsize::new(0),
11085 always_fail,
11086 })
11087 }
11088
11089 #[tokio::test]
11093 async fn paired_spend_is_debited_locally_when_the_log_read_back_fails() {
11094 use tower::ServiceExt;
11095 let db = std::env::temp_dir().join(format!("firstpass-paired-flaky-{}.db", Uuid::now_v7()));
11096 let (tx, handle) = crate::store::open(&db).unwrap();
11097 drop(tx);
11098 handle.await.unwrap();
11099 let (mut state, _rx) = challenger_state(50.0, &db.to_string_lossy());
11100 let toml = format!(
11101 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
11102 ladder = [\"{INCUMBENT}\"]\ngates = [\"json-valid\"]\n\
11103 [route.challenger]\nladder = [\"{CHALLENGER}\"]\nmode = \"paired\"\n\
11104 max_usd_per_day = 5.0\n"
11105 );
11106 let mut cfg = (*state.config).clone();
11107 cfg.routing = Some(firstpass_core::Config::parse(&toml).expect("paired config parses"));
11108 state.experiments = Arc::new(
11109 crate::experiment::ExperimentMonitor::from_config(cfg.routing.as_ref().unwrap())
11110 .with_log(flaky_log(false)),
11111 );
11112 state.config = Arc::new(cfg);
11113 let mut outs = HashMap::new();
11114 outs.insert(
11115 INCUMBENT.to_owned(),
11116 Ok(model_resp(INCUMBENT, "{\"ok\": 1}")),
11117 );
11118 outs.insert(
11119 CHALLENGER.to_owned(),
11120 Ok(model_resp(CHALLENGER, "{\"ok\": 1}")),
11121 );
11122 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
11123 map.insert(
11124 "anthropic".to_owned(),
11125 Arc::new(MockProvider::new("anthropic", outs)),
11126 );
11127 state.providers = ProviderRegistry::from_map(map);
11128 let ledger = state.paired_ledger.clone();
11129 let router = app(state).expect("router");
11130 let req = axum::http::Request::builder()
11131 .method("POST")
11132 .uri("/v1/messages")
11133 .body(Body::from(user_body()))
11134 .unwrap();
11135 assert_eq!(
11136 router.oneshot(req).await.unwrap().status(),
11137 axum::http::StatusCode::OK
11138 );
11139 let mut pairs = Vec::new();
11140 for _ in 0..200 {
11141 pairs = crate::store::load_all_experiment_pairs(&db).unwrap_or_default();
11142 if !pairs.is_empty() {
11143 break;
11144 }
11145 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
11146 }
11147 assert_eq!(pairs.len(), 1, "the challenger ran (first sync succeeded)");
11148 let spent = ledger.spent_today("default", 0, jiff::Timestamp::now());
11149 assert!(
11150 spent > 0.0,
11151 "challenger spend debited locally despite the failed read-back: {spent}"
11152 );
11153 let _ = std::fs::remove_file(&db);
11154 }
11155
11156 #[derive(Debug)]
11158 struct AppendFailLog;
11159
11160 #[async_trait::async_trait]
11161 impl crate::experiment::ExperimentLogBackend for AppendFailLog {
11162 async fn append(
11163 &self,
11164 _: &str,
11165 _: usize,
11166 _: &str,
11167 _: crate::experiment::LogRecord,
11168 ) -> Result<u64, crate::experiment::LogError> {
11169 Err(crate::experiment::LogError::Backend("rpush failed".into()))
11170 }
11171 async fn load_since(
11172 &self,
11173 _: &str,
11174 _: usize,
11175 _: &str,
11176 _: u64,
11177 ) -> Result<Vec<Option<crate::experiment::LogRecord>>, crate::experiment::LogError>
11178 {
11179 Ok(vec![])
11180 }
11181 async fn claim_session(&self, _: &str, _: usize, _: &str, _: &str) -> bool {
11182 false
11183 }
11184 }
11185
11186 #[tokio::test]
11189 async fn paired_pair_that_missed_the_log_is_persisted_as_offline() {
11190 use tower::ServiceExt;
11191 let db =
11192 std::env::temp_dir().join(format!("firstpass-paired-offline-{}.db", Uuid::now_v7()));
11193 let (tx, handle) = crate::store::open(&db).unwrap();
11194 drop(tx);
11195 handle.await.unwrap();
11196 let (mut state, _rx) = challenger_state(50.0, &db.to_string_lossy());
11197 let toml = format!(
11198 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
11199 ladder = [\"{INCUMBENT}\"]\ngates = [\"json-valid\"]\n\
11200 [route.challenger]\nladder = [\"{CHALLENGER}\"]\nmode = \"paired\"\n\
11201 max_usd_per_day = 5.0\n"
11202 );
11203 let mut cfg = (*state.config).clone();
11204 cfg.routing = Some(firstpass_core::Config::parse(&toml).expect("paired config parses"));
11205 state.experiments = Arc::new(
11206 crate::experiment::ExperimentMonitor::from_config(cfg.routing.as_ref().unwrap())
11207 .with_log(Arc::new(AppendFailLog)),
11208 );
11209 state.config = Arc::new(cfg);
11210 let mut outs = HashMap::new();
11211 outs.insert(
11212 INCUMBENT.to_owned(),
11213 Ok(model_resp(INCUMBENT, "{\"ok\": 1}")),
11214 );
11215 outs.insert(
11216 CHALLENGER.to_owned(),
11217 Ok(model_resp(CHALLENGER, "{\"ok\": 1}")),
11218 );
11219 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
11220 map.insert(
11221 "anthropic".to_owned(),
11222 Arc::new(MockProvider::new("anthropic", outs)),
11223 );
11224 state.providers = ProviderRegistry::from_map(map);
11225 let router = app(state).expect("router");
11226 let req = axum::http::Request::builder()
11227 .method("POST")
11228 .uri("/v1/messages")
11229 .body(Body::from(user_body()))
11230 .unwrap();
11231 assert_eq!(
11232 router.oneshot(req).await.unwrap().status(),
11233 axum::http::StatusCode::OK
11234 );
11235 let mut pairs = Vec::new();
11236 for _ in 0..200 {
11237 pairs = crate::store::load_all_experiment_pairs(&db).unwrap_or_default();
11238 if !pairs.is_empty() {
11239 break;
11240 }
11241 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
11242 }
11243 assert_eq!(pairs.len(), 1);
11244 assert_eq!(
11245 pairs[0].in_log,
11246 crate::store::InLogStatus::NotInLog,
11247 "the append failed, so the row must be marked offline"
11248 );
11249 let _ = std::fs::remove_file(&db);
11250 }
11251
11252 #[tokio::test]
11256 async fn sqlite_write_failure_withholds_local_evidence() {
11257 use tower::ServiceExt;
11258 let bad_db = "/nonexistent-firstpass-dir/firstpass-sqlite-fail.db";
11259 let (mut state, _rx) = challenger_state(50.0, bad_db);
11260 let toml = format!(
11261 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
11262 ladder = [\"{INCUMBENT}\"]\ngates = [\"json-valid\"]\n\
11263 [route.challenger]\nladder = [\"{CHALLENGER}\"]\nmode = \"paired\"\n\
11264 max_usd_per_day = 5.0\n"
11265 );
11266 let mut cfg = (*state.config).clone();
11267 cfg.routing = Some(firstpass_core::Config::parse(&toml).expect("paired config parses"));
11268 state.experiments = Arc::new(crate::experiment::ExperimentMonitor::from_config(
11269 cfg.routing.as_ref().unwrap(),
11270 ));
11271 state.config = Arc::new(cfg);
11272 let mut outs = HashMap::new();
11273 outs.insert(
11274 INCUMBENT.to_owned(),
11275 Ok(model_resp(INCUMBENT, "{\"ok\": 1}")),
11276 );
11277 outs.insert(
11278 CHALLENGER.to_owned(),
11279 Ok(model_resp(CHALLENGER, "{\"ok\": 1}")),
11280 );
11281 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
11282 map.insert(
11283 "anthropic".to_owned(),
11284 Arc::new(MockProvider::new("anthropic", outs)),
11285 );
11286 state.providers = ProviderRegistry::from_map(map);
11287 let (ledger, monitor) = (state.paired_ledger.clone(), state.experiments.clone());
11288 let router = app(state).expect("router");
11289 let req = axum::http::Request::builder()
11290 .method("POST")
11291 .uri("/v1/messages")
11292 .body(Body::from(user_body()))
11293 .unwrap();
11294 assert_eq!(
11295 router.oneshot(req).await.unwrap().status(),
11296 axum::http::StatusCode::OK
11297 );
11298 for _ in 0..200 {
11300 if ledger.spent_today("default", 0, jiff::Timestamp::now()) > 0.0 {
11301 break;
11302 }
11303 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
11304 }
11305 assert!(
11306 ledger.spent_today("default", 0, jiff::Timestamp::now()) > 0.0,
11307 "challenger ran"
11308 );
11309 tokio::time::sleep(std::time::Duration::from_millis(100)).await;
11310 let pairs: u64 = monitor.reports().iter().map(|r| r.report.pairs).sum();
11311 assert_eq!(
11312 pairs, 0,
11313 "the durable write failed, so no evidence was applied"
11314 );
11315 }
11316
11317 #[tokio::test]
11320 async fn live_challenger_is_not_served_when_the_cold_start_sync_fails() {
11321 use firstpass_core::Arm;
11322 let (mut state, mut rx) = challenger_state(50.0, ":memory:");
11323 let cfg = state.config.routing.clone().unwrap();
11324 state.experiments = Arc::new(
11325 crate::experiment::ExperimentMonitor::from_config(&cfg).with_log(flaky_log(true)),
11326 );
11327 let session = session_in_arm(&state, Arm::Challenger, 0);
11328 let trace = serve_session(&state, &mut rx, &session).await;
11329 let rec = trace
11330 .experiment
11331 .expect("experiment record on a challenger route");
11332 assert_eq!(rec.arm, Arm::Incumbent, "fail closed: served the incumbent");
11333 assert_eq!(trace.attempts[0].model, INCUMBENT);
11334 }
11335
11336 #[tokio::test]
11338 async fn halted_paired_challenger_stops_spending() {
11339 use tower::ServiceExt;
11340 let db = std::env::temp_dir().join(format!("firstpass-paired-halt-{}.db", Uuid::now_v7()));
11341 let (tx, handle) = crate::store::open(&db).unwrap();
11342 drop(tx);
11343 handle.await.unwrap();
11344 let (mut state, _rx) = challenger_state(50.0, &db.to_string_lossy());
11345 let toml = format!(
11346 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
11347 ladder = [\"{INCUMBENT}\"]\ngates = [\"json-valid\"]\n\
11348 [route.challenger]\nladder = [\"{CHALLENGER}\"]\nmode = \"paired\"\n\
11349 max_usd_per_day = 5.0\n"
11350 );
11351 let mut cfg = (*state.config).clone();
11352 cfg.routing = Some(firstpass_core::Config::parse(&toml).expect("paired config parses"));
11353 state.experiments = Arc::new(crate::experiment::ExperimentMonitor::from_config(
11354 cfg.routing.as_ref().unwrap(),
11355 ));
11356 state.config = Arc::new(cfg);
11357 let fingerprint = state.experiments.fingerprint(0).unwrap().to_owned();
11358 for _ in 0..500 {
11362 if state.experiments.challenger_blocked("default", 0) {
11363 break;
11364 }
11365 state.experiments.apply_pair_locally(
11366 "default",
11367 0,
11368 &fingerprint,
11369 1.0,
11370 0.0,
11371 0.001,
11372 uuid::Uuid::nil(),
11373 );
11374 }
11375 assert!(
11376 state.experiments.challenger_blocked("default", 0),
11377 "a challenger that always loses halts"
11378 );
11379 let mut outs = HashMap::new();
11380 outs.insert(
11381 INCUMBENT.to_owned(),
11382 Ok(model_resp(INCUMBENT, "{\"ok\": 1}")),
11383 );
11384 outs.insert(
11385 CHALLENGER.to_owned(),
11386 Ok(model_resp(CHALLENGER, "{\"ok\": 1}")),
11387 );
11388 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
11389 map.insert(
11390 "anthropic".to_owned(),
11391 Arc::new(MockProvider::new("anthropic", outs)),
11392 );
11393 state.providers = ProviderRegistry::from_map(map);
11394 let ledger = state.paired_ledger.clone();
11395 let router = app(state).expect("router");
11396
11397 let req = axum::http::Request::builder()
11398 .method("POST")
11399 .uri("/v1/messages")
11400 .body(Body::from(user_body()))
11401 .unwrap();
11402 let resp = router.oneshot(req).await.unwrap();
11403 assert_eq!(
11404 resp.status(),
11405 axum::http::StatusCode::OK,
11406 "incumbent still serves"
11407 );
11408 tokio::time::sleep(std::time::Duration::from_millis(300)).await;
11409 assert!(
11410 crate::store::load_all_experiment_pairs(&db)
11411 .unwrap()
11412 .is_empty(),
11413 "no pair after the halt"
11414 );
11415 assert!(
11416 ledger.may_spend("default", 0, 1e-9, jiff::Timestamp::now()),
11417 "nothing debited after the halt"
11418 );
11419 let _ = std::fs::remove_file(&db);
11420 }
11421
11422 #[tokio::test]
11425 async fn shadow_reports_no_pass_when_every_rung_fails_its_gate() {
11426 let (mut state, _rx) = challenger_state(50.0, ":memory:");
11427 let mut outs = HashMap::new();
11428 outs.insert(INCUMBENT.to_owned(), Ok(model_resp(INCUMBENT, "not json")));
11429 let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
11430 map.insert(
11431 "anthropic".to_owned(),
11432 Arc::new(MockProvider::new("anthropic", outs)),
11433 );
11434 state.providers = ProviderRegistry::from_map(map);
11435 let cfg = firstpass_core::Config::parse(&format!(
11436 "[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
11437 ladder = [\"{INCUMBENT}\"]\ngates = [\"json-valid\"]\n"
11438 ))
11439 .unwrap();
11440 let shadow = firstpass_core::rollout::Shadow {
11441 sample_rate: 1.0,
11442 max_usd_per_day: 5.0,
11443 };
11444 let signal = evaluate_shadow(
11445 &state,
11446 &cfg.routes[0],
11447 shadow,
11448 0,
11449 &user_body(),
11450 Auth::from_headers(&HeaderMap::new(), firstpass_core::Dialect::Anthropic),
11451 &Features::new(TaskKind::Other),
11452 "t".to_owned(),
11453 "s".to_owned(),
11454 "anthropic.messages",
11455 0.0,
11456 )
11457 .await;
11458 assert_eq!(signal.skipped, None);
11459 assert!(!signal.would_pass, "a best attempt is not a pass");
11460 assert_eq!(signal.would_serve_rung, None);
11461 }
11462
11463 fn ux_get(uri: &str, bearer: Option<&str>) -> axum::http::Request<Body> {
11464 let mut b = axum::http::Request::builder().method("GET").uri(uri);
11465 if let Some(k) = bearer {
11466 b = b.header("authorization", format!("Bearer {k}"));
11467 }
11468 b.body(Body::empty()).unwrap()
11469 }
11470
11471 #[tokio::test]
11475 async fn experiments_endpoint_reports_per_arm_counts_scoped_to_the_tenant() {
11476 use firstpass_core::Arm;
11477 use tower::ServiceExt;
11478 let db = std::env::temp_dir().join(format!("firstpass-ux-{}.db", Uuid::now_v7()));
11479 let mut state = ux_state(&db.to_string_lossy());
11480 let (traces_tx, mut traces_rx) = mpsc::channel(256);
11482 state.traces = traces_tx;
11483 let router = app(state.clone()).expect("router");
11484
11485 let mut sessions = Vec::new();
11487 for arm in [Arm::Incumbent, Arm::Challenger] {
11488 for n in 0..6 {
11489 sessions.push((arm, session_in_arm(&state, arm, n)));
11490 }
11491 }
11492 for (_, session) in &sessions {
11493 let req = axum::http::Request::builder()
11494 .method("POST")
11495 .uri("/v1/messages")
11496 .header("authorization", "Bearer tenant-a.key-a")
11497 .header("x-firstpass-session", session.as_str())
11498 .body(Body::from(user_body()))
11499 .unwrap();
11500 let resp = router.clone().oneshot(req).await.unwrap();
11501 assert_eq!(resp.status(), axum::http::StatusCode::OK);
11502 }
11503 let mut traces = Vec::new();
11504 while let Ok(t) = traces_rx.try_recv() {
11505 traces.push(t);
11506 }
11507 assert_eq!(traces.len(), 12);
11508
11509 let (tx, handle) = crate::store::open(&db).unwrap();
11511 for t in &traces {
11512 tx.try_send(t.clone()).unwrap();
11513 }
11514 drop(tx);
11515 handle.await.unwrap();
11516
11517 let mut challenger_seen = 0;
11519 for t in &traces {
11520 let arm = t.experiment.as_ref().unwrap().arm;
11521 let verdict = if arm == Arm::Challenger {
11522 challenger_seen += 1;
11523 if challenger_seen <= 2 { "fail" } else { "pass" }
11524 } else {
11525 "pass"
11526 };
11527 let req = axum::http::Request::builder()
11528 .method("POST")
11529 .uri("/v1/feedback")
11530 .header("authorization", "Bearer tenant-a.key-a")
11531 .body(Body::from(
11532 serde_json::json!({
11533 "trace_id": t.trace_id, "gate_id": "tests",
11534 "verdict": verdict, "reporter": "ci"
11535 })
11536 .to_string(),
11537 ))
11538 .unwrap();
11539 let resp = router.clone().oneshot(req).await.unwrap();
11540 assert_eq!(resp.status(), axum::http::StatusCode::ACCEPTED);
11541 }
11542
11543 let resp = router
11545 .clone()
11546 .oneshot(ux_get("/v1/experiments", Some("tenant-a.key-a")))
11547 .await
11548 .unwrap();
11549 assert_eq!(resp.status(), axum::http::StatusCode::OK);
11550 let a = body_json(resp).await;
11551 let exps = a["experiments"].as_array().unwrap();
11552 assert_eq!(exps.len(), 1, "{a}");
11553 let r = &exps[0]["report"];
11554 assert_eq!(r["status"], "undecided");
11555 assert_eq!(r["pairs"], 6);
11556 assert_eq!(r["incumbent"]["requests"], 6);
11557 assert_eq!(r["incumbent"]["outcomes"], 6);
11558 assert_eq!(r["incumbent"]["outcome_sum"], 6.0);
11559 assert_eq!(r["challenger"]["requests"], 6);
11560 assert_eq!(r["challenger"]["outcomes"], 6);
11561 assert_eq!(r["challenger"]["outcome_sum"], 4.0);
11562 assert_eq!(exps[0]["challenger_ladder"][0], CHALLENGER);
11563 assert_eq!(exps[0]["recommendation"], "keep_running");
11564 assert_eq!(a["configured"][0]["unavailable"], false);
11565 assert_eq!(a["configured"][0]["mode"], "live");
11566 assert!(a["configured"][0]["paired"].is_null());
11567
11568 let resp = router
11570 .clone()
11571 .oneshot(ux_get("/v1/experiments", Some("tenant-b.key-b")))
11572 .await
11573 .unwrap();
11574 assert_eq!(resp.status(), axum::http::StatusCode::OK);
11575 let b = body_json(resp).await;
11576 assert_eq!(b["experiments"].as_array().unwrap().len(), 0, "{b}");
11577 assert!(!b.to_string().contains("tenant-a"));
11578
11579 let resp = router
11581 .clone()
11582 .oneshot(ux_get("/v1/receipts?limit=3", Some("tenant-a.key-a")))
11583 .await
11584 .unwrap();
11585 let ra = body_json(resp).await;
11586 assert_eq!(ra["receipts"].as_array().unwrap().len(), 3);
11587 assert_eq!(ra["summary"]["requests"], 12, "summary spans all receipts");
11588 let first = &ra["receipts"][0];
11589 assert!(first["experiment"]["arm"].is_string());
11590 assert_eq!(first["hash"].as_str().unwrap().len(), 64);
11591 assert!(first["prev_hash"].is_string());
11592 assert!(first["attempts"][0]["gates"][0]["gate_id"].is_string());
11593 let resp = router
11594 .clone()
11595 .oneshot(ux_get("/v1/receipts", Some("tenant-b.key-b")))
11596 .await
11597 .unwrap();
11598 let rb = body_json(resp).await;
11599 assert_eq!(rb["receipts"].as_array().unwrap().len(), 0);
11600 assert_eq!(rb["summary"]["requests"], 0);
11601
11602 for bearer in [None, Some("tenant-a.wrong")] {
11604 let resp = router
11605 .clone()
11606 .oneshot(ux_get("/v1/experiments", bearer))
11607 .await
11608 .unwrap();
11609 assert_eq!(resp.status(), axum::http::StatusCode::UNAUTHORIZED);
11610 }
11611 let _ = std::fs::remove_file(&db);
11612 }
11613
11614 #[test]
11617 fn receipts_summary_json_keeps_losses_signed() {
11618 let s = summary_json(&crate::store::ReceiptsSummary {
11619 requests: 2,
11620 spent_usd: 6.0,
11621 baseline_usd: 6.0,
11622 first_pass: 1,
11623 escalated: 1,
11624 gates: [3, 1, 2],
11625 truncated: 4,
11626 });
11627 assert_eq!(s["requests"], 2);
11628 assert_eq!(s["truncated"], 4);
11629 assert_eq!(s["first_pass"], 1);
11630 assert_eq!(s["escalated"], 1);
11631 assert_eq!(s["gates"]["pass"], 3);
11632 assert_eq!(s["gates"]["fail"], 1);
11633 assert_eq!(s["gates"]["abstain"], 2);
11634 assert!((s["saved_usd"].as_f64().unwrap()).abs() < 1e-9);
11635 let loss = summary_json(&crate::store::ReceiptsSummary {
11636 spent_usd: 5.0,
11637 baseline_usd: 3.0,
11638 ..Default::default()
11639 });
11640 assert!(
11641 loss["saved_usd"].as_f64().unwrap() < 0.0,
11642 "loss stays negative"
11643 );
11644 }
11645
11646 #[tokio::test]
11649 async fn panel_is_served_with_a_csp_pinning_its_own_script() {
11650 use sha2::Digest as _;
11651 let resp = panel().await.into_response();
11652 let csp = resp
11653 .headers()
11654 .get(axum::http::header::CONTENT_SECURITY_POLICY)
11655 .expect("panel sets a CSP")
11656 .to_str()
11657 .unwrap()
11658 .to_owned();
11659 let body = PANEL_HTML.split_once("<script>").unwrap().1;
11660 let script = body.split_once("</script>").unwrap().0;
11661 assert!(script.len() > 1000, "extracted the panel script");
11664 let want = format!(
11665 "script-src 'sha256-{}'",
11666 base64_std(&sha2::Sha256::digest(script.as_bytes()))
11667 );
11668 assert!(csp.contains(&want), "{csp}");
11669 for directive in [
11670 "default-src 'none'",
11671 "connect-src 'self'",
11672 "frame-ancestors 'none'",
11673 ] {
11674 assert!(csp.contains(directive), "{directive} missing: {csp}");
11675 }
11676 assert!(!csp.contains("unsafe-eval") && !csp.contains("script-src 'unsafe-inline'"));
11677 assert_eq!(
11678 PANEL_HTML.matches("<script").count(),
11679 1,
11680 "one inline script"
11681 );
11682 assert_eq!(
11683 resp.headers()[axum::http::header::X_CONTENT_TYPE_OPTIONS],
11684 "nosniff"
11685 );
11686 }
11687
11688 #[test]
11689 fn base64_std_matches_rfc4648_vectors() {
11690 for (input, want) in [
11691 ("", ""),
11692 ("f", "Zg=="),
11693 ("fo", "Zm8="),
11694 ("foo", "Zm9v"),
11695 ("foob", "Zm9vYg=="),
11696 ("fooba", "Zm9vYmE="),
11697 ("foobar", "Zm9vYmFy"),
11698 ] {
11699 assert_eq!(base64_std(input.as_bytes()), want, "{input:?}");
11700 }
11701 assert_eq!(base64_std(&[0xfb, 0xff]), "+/8=");
11702 }
11703
11704 #[test]
11707 fn panel_wires_the_experiments_view_and_has_no_external_references() {
11708 assert!(
11709 PANEL_HTML.contains("/v1/experiments"),
11710 "fetches the endpoint"
11711 );
11712 assert!(PANEL_HTML.contains("id=\"view-experiments\""));
11713 assert!(PANEL_HTML.contains("id=\"exp-list\""));
11714 assert!(
11715 PANEL_HTML.contains("[route.challenger]"),
11716 "empty state teaches the config"
11717 );
11718 assert!(
11719 PANEL_HTML.contains("data-copy"),
11720 "adopt snippet has a copy button"
11721 );
11722 assert!(
11723 PANEL_HTML.contains("document.hidden"),
11724 "polling pauses in a hidden tab"
11725 );
11726 assert!(PANEL_HTML.contains("prefers-reduced-motion"));
11727 assert!(PANEL_HTML.contains("prefers-color-scheme: dark"));
11728 for needle in [
11731 "http://",
11732 "https://",
11733 "src=\"//",
11734 "href=\"//",
11735 "@import",
11736 "<link",
11737 ] {
11738 assert!(!PANEL_HTML.contains(needle), "external reference: {needle}");
11739 }
11740 for tag in ["<script", "<img", "<iframe", "<source"] {
11741 for (i, _) in PANEL_HTML.match_indices(tag) {
11742 let end = PANEL_HTML[i..]
11743 .find('>')
11744 .map_or(PANEL_HTML.len(), |e| i + e);
11745 assert!(
11746 !PANEL_HTML[i..end].contains("src="),
11747 "{tag} loads an external source: {}",
11748 &PANEL_HTML[i..end]
11749 );
11750 }
11751 }
11752 }
11753
11754 #[test]
11758 fn panel_interpolates_only_escaped_or_numeric_values() {
11759 let start = PANEL_HTML.find("<script>").expect("script") + "<script>".len();
11760 let js = &PANEL_HTML[start..PANEL_HTML.find("</script>").expect("end")];
11761 let bytes = js.as_bytes();
11762 let mut i = 0;
11763 let mut count = 0;
11764 while let Some(off) = js[i..].find("${") {
11765 let begin = i + off + 2;
11766 let (mut depth, mut j) = (1, begin);
11767 while depth > 0 && j < bytes.len() {
11768 match bytes[j] {
11769 b'{' => depth += 1,
11770 b'}' => depth -= 1,
11771 _ => {}
11772 }
11773 j += 1;
11774 }
11775 let expr = js[begin..j - 1].trim();
11776 assert!(
11777 ["esc(", "usd(", "num(", "pct(", "cls(", "h_"]
11778 .iter()
11779 .any(|p| expr.starts_with(p)),
11780 "unescaped interpolation in the panel: ${{{expr}}}"
11781 );
11782 count += 1;
11783 i = begin;
11784 }
11785 assert!(
11786 count > 40,
11787 "the scan must actually see the templates ({count})"
11788 );
11789 assert!(js.contains("const esc ="), "the escaper must exist");
11790 }
11791
11792 struct TestStream {
11797 chunks: std::collections::VecDeque<Result<Bytes, reqwest::Error>>,
11798 }
11799
11800 impl TestStream {
11801 fn from_ok(v: Vec<&'static [u8]>) -> Self {
11802 Self {
11803 chunks: v.into_iter().map(|b| Ok(Bytes::from_static(b))).collect(),
11804 }
11805 }
11806 }
11807
11808 impl Stream for TestStream {
11809 type Item = Result<Bytes, reqwest::Error>;
11810 fn poll_next(
11811 mut self: std::pin::Pin<&mut Self>,
11812 _cx: &mut Context<'_>,
11813 ) -> Poll<Option<Self::Item>> {
11814 Poll::Ready(self.chunks.pop_front())
11815 }
11816 }
11817
11818 fn anthropic_sse_bytes() -> (&'static [u8], &'static [u8]) {
11822 let c1: &'static [u8] = b"event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-haiku-4-5\",\"usage\":{\"input_tokens\":100,\"cache_crea";
11824 let c2: &'static [u8] = b"tion_input_tokens\":50,\"cache_read_input_tokens\":25}}}\n\nevent: message_delta\ndata: {\"type\":\"message_delta\",\"usage\":{\"output_tokens\":200},\"delta\":{\"stop_reason\":\"max_tokens\"}}\n\n";
11826 (c1, c2)
11827 }
11828
11829 #[tokio::test]
11838 async fn sse_tap_parses_usage_across_split_chunks_and_relays_bytes_unchanged() {
11839 let (c1, c2) = anthropic_sse_bytes();
11840 let all_bytes: Vec<u8> = [c1, c2].concat();
11841
11842 let stream = TestStream::from_ok(vec![c1, c2]);
11843 let (tx, rx) = tokio::sync::oneshot::channel::<SseUsage>();
11844 let tap = SseTap::new(stream, move |u, _reason| {
11845 let _ = tx.send(u);
11846 });
11847
11848 let collected = axum::body::to_bytes(Body::from_stream(tap), usize::MAX)
11850 .await
11851 .unwrap();
11852
11853 assert_eq!(
11855 collected.as_ref(),
11856 all_bytes.as_slice(),
11857 "relayed bytes must equal input bytes exactly"
11858 );
11859
11860 let usage = rx.await.unwrap();
11861 assert_eq!(usage.in_tokens, 100, "input_tokens from message_start");
11862 assert_eq!(
11863 usage.cache_write_tokens, 50,
11864 "cache_creation from message_start"
11865 );
11866 assert_eq!(usage.cache_read_tokens, 25, "cache_read from message_start");
11867 assert_eq!(usage.out_tokens, 200, "output_tokens from message_delta");
11868 assert_eq!(
11869 usage.model.as_deref(),
11870 Some("claude-haiku-4-5"),
11871 "model from message_start"
11872 );
11873 assert_eq!(
11874 usage.stop_reason.as_deref(),
11875 Some("max_tokens"),
11876 "stop_reason from message_delta"
11877 );
11878
11879 let config = test_config();
11881 let req = Bytes::from_static(br#"{"model":"claude-haiku-4-5","messages":[]}"#);
11882 let trace = build_stream_trace_from_usage(
11883 &config,
11884 &req,
11885 10,
11886 None,
11887 usage,
11888 "anthropic",
11889 StreamOutcome::Served,
11890 "anthropic.messages",
11891 None,
11892 );
11893 assert!(
11894 trace.attempts[0].truncated,
11895 "max_tokens stop_reason must set truncated=true"
11896 );
11897 assert!(
11898 trace.attempts[0].cost_usd > 0.0,
11899 "non-zero tokens must produce a non-zero cost"
11900 );
11901 }
11902
11903 #[tokio::test]
11913 async fn observe_records_an_upstream_429_as_an_error_on_both_paths() {
11914 const ERR: &str =
11915 r#"{"type":"error","error":{"type":"rate_limit_error","message":"slow down"}}"#;
11916 let app = axum::Router::new().route(
11917 "/v1/messages",
11918 axum::routing::post(|| async {
11919 (
11920 axum::http::StatusCode::TOO_MANY_REQUESTS,
11921 [(axum::http::header::CONTENT_TYPE, "application/json")],
11922 ERR,
11923 )
11924 }),
11925 );
11926 let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
11927 .await
11928 .expect("bind");
11929 let addr = listener.local_addr().expect("addr");
11930 tokio::spawn(async move {
11931 let _ = axum::serve(listener, app).await;
11932 });
11933 let base_url = format!("http://{addr}");
11934 let config = ProxyConfig::from_lookup(|k| match k {
11935 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some(base_url.clone()),
11936 _ => None,
11937 })
11938 .unwrap();
11939 let (traces, mut rx) = mpsc::channel(4);
11940 let state = AppState {
11941 config: Arc::new(config),
11942 http: reqwest::Client::new(),
11943 providers: ProviderRegistry::from_map(HashMap::new()),
11944 gate_health: Arc::new(GateHealthRegistry::new()),
11945 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
11946 paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
11947 experiments: Arc::default(),
11948 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
11949 traces,
11950 adaptive: None,
11951 eprocess: None,
11952 bandit: None,
11953 promoter: None,
11954 verified_cache: None,
11955 predictor: None,
11956 prior: None,
11957 tenant_rate_limiter: None,
11958 spill: None,
11959 };
11960 for body in [
11961 &br#"{"model":"claude-haiku-4-5","messages":[{"role":"user","content":"hi"}]}"#[..],
11962 &br#"{"model":"claude-haiku-4-5","stream":true,"messages":[{"role":"user","content":"hi"}]}"#[..],
11963 ] {
11964 let resp = observe_passthrough(
11965 state.clone(),
11966 HeaderMap::new(),
11967 Bytes::from_static(body),
11968 None,
11969 "default".to_owned(),
11970 )
11971 .await;
11972 assert_eq!(resp.status(), axum::http::StatusCode::TOO_MANY_REQUESTS);
11973 let relayed = axum::body::to_bytes(resp.into_body(), 1 << 20).await.unwrap();
11974 assert_eq!(relayed.as_ref(), ERR.as_bytes(), "relayed byte-for-byte");
11975 let trace = tokio::time::timeout(std::time::Duration::from_secs(5), rx.recv())
11976 .await
11977 .expect("trace within 5s")
11978 .expect("a trace");
11979 assert_eq!(
11980 trace.final_.served_from,
11981 ServedFrom::Error,
11982 "stream={}",
11983 String::from_utf8_lossy(body).contains("stream")
11984 );
11985 assert!(trace.attempts.is_empty());
11986 }
11987 }
11988
11989 #[tokio::test]
11990 async fn observe_stream_receipt_has_sse_usage_and_non_zero_cost() {
11991 const SSE: &str = concat!(
11993 "event: message_start\n",
11994 "data: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-haiku-4-5\",",
11995 "\"usage\":{\"input_tokens\":100,\"cache_creation_input_tokens\":0,",
11996 "\"cache_read_input_tokens\":0}}}\n",
11997 "\n",
11998 "event: message_delta\n",
11999 "data: {\"type\":\"message_delta\",\"usage\":{\"output_tokens\":50},",
12000 "\"delta\":{\"stop_reason\":\"max_tokens\"}}\n",
12001 "\n",
12002 );
12003
12004 let app = axum::Router::new().route(
12006 "/v1/messages",
12007 axum::routing::post(|| async {
12008 (
12009 [(
12010 axum::http::header::CONTENT_TYPE,
12011 "text/event-stream; charset=utf-8",
12012 )],
12013 SSE,
12014 )
12015 }),
12016 );
12017 let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
12018 .await
12019 .expect("bind");
12020 let addr = listener.local_addr().expect("addr");
12021 tokio::spawn(async move {
12022 let _ = axum::serve(listener, app).await;
12023 });
12024 let base_url = format!("http://{addr}");
12025
12026 let config = ProxyConfig::from_lookup(|k| match k {
12027 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some(base_url.clone()),
12028 _ => None,
12029 })
12030 .unwrap();
12031 let (traces, mut rx) = mpsc::channel(4);
12032 let state = AppState {
12033 config: Arc::new(config),
12034 http: reqwest::Client::new(),
12035 providers: ProviderRegistry::from_map(HashMap::new()),
12036 gate_health: Arc::new(GateHealthRegistry::new()),
12037 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
12038 paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
12039 experiments: Arc::default(),
12040 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
12041 traces,
12042 adaptive: None,
12043 eprocess: None,
12044 bandit: None,
12045 promoter: None,
12046 verified_cache: None,
12047 predictor: None,
12048 prior: None,
12049 tenant_rate_limiter: None,
12050 spill: None,
12051 };
12052
12053 let body = Bytes::from_static(
12054 br#"{"model":"claude-haiku-4-5","stream":true,"messages":[{"role":"user","content":"hi"}]}"#,
12055 );
12056 let resp = observe_stream(state, HeaderMap::new(), body, None, "default".to_owned()).await;
12057 assert_eq!(resp.status(), axum::http::StatusCode::OK);
12058
12059 let _ = axum::body::to_bytes(resp.into_body(), 1 << 20)
12062 .await
12063 .unwrap();
12064
12065 let trace = rx
12067 .recv()
12068 .await
12069 .expect("trace must be sent after stream ends");
12070 assert_eq!(trace.attempts.len(), 1, "one attempt");
12071 let attempt = &trace.attempts[0];
12072 assert_eq!(
12073 attempt.in_tokens, 100,
12074 "input tokens from SSE message_start"
12075 );
12076 assert_eq!(
12077 attempt.out_tokens, 50,
12078 "output tokens from SSE message_delta"
12079 );
12080 assert!(
12081 attempt.cost_usd > 0.0,
12082 "non-zero tokens must produce a non-zero cost"
12083 );
12084 assert!(
12085 attempt.truncated,
12086 "max_tokens stop_reason must set truncated=true in the receipt"
12087 );
12088 assert_eq!(
12089 attempt.model, "claude-haiku-4-5",
12090 "model from SSE message_start"
12091 );
12092 }
12093
12094 #[test]
12102 fn sse_tap_oversize_line_skipped_and_following_valid_event_still_parsed() {
12103 let mut big_chunk = b"data: ".to_vec();
12105 big_chunk.extend(std::iter::repeat_n(b'x', SSE_LINE_BUF_CAP + 1));
12106 big_chunk.push(b'\n');
12107 big_chunk.extend_from_slice(
12109 b"event: message_start\n\
12110 data: {\"type\":\"message_start\",\"message\":{\"model\":\"m\",\
12111 \"usage\":{\"input_tokens\":42,\"cache_creation_input_tokens\":0,\
12112 \"cache_read_input_tokens\":0}}}\n\n",
12113 );
12114
12115 let mut line_buf: Vec<u8> = Vec::new();
12116 let mut overflow = false;
12117 let mut usage = SseUsage::default();
12118 parse_sse_chunk_into(&big_chunk, &mut line_buf, &mut overflow, &mut usage);
12119
12120 assert!(
12122 line_buf.len() <= SSE_LINE_BUF_CAP,
12123 "line buffer must stay bounded: len={}",
12124 line_buf.len()
12125 );
12126 assert_eq!(
12128 usage.in_tokens, 42,
12129 "valid event after oversize line must still be parsed"
12130 );
12131 assert_eq!(
12132 usage.model.as_deref(),
12133 Some("m"),
12134 "model must be extracted from the valid event"
12135 );
12136 }
12137
12138 #[tokio::test]
12149 async fn sse_tap_upstream_error_fires_done_with_upstream_error_reason() {
12150 let err: reqwest::Error = reqwest::Client::new().get("not-a-url").build().unwrap_err();
12152
12153 let ok_chunk: &'static [u8] =
12154 b"event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"m\",\"usage\":{\"input_tokens\":5,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0}}}\n\n";
12155
12156 let stream = TestStream {
12157 chunks: [Ok(Bytes::from_static(ok_chunk)), Err(err)]
12158 .into_iter()
12159 .collect(),
12160 };
12161
12162 let (tx, rx) = tokio::sync::oneshot::channel::<SseEndReason>();
12163 let tap = SseTap::new(stream, move |_u, reason| {
12164 let _ = tx.send(reason);
12165 });
12166
12167 let _ = axum::body::to_bytes(Body::from_stream(tap), usize::MAX).await;
12169
12170 let reason = rx.await.expect("on_done must fire");
12171 assert_eq!(
12172 reason,
12173 SseEndReason::UpstreamError,
12174 "transport error mid-stream must produce UpstreamError, not Eof"
12175 );
12176 }
12177
12178 #[tokio::test]
12183 async fn sse_tap_normal_end_fires_done_with_eof_reason() {
12184 let (c1, c2) = anthropic_sse_bytes();
12185 let stream = TestStream::from_ok(vec![c1, c2]);
12186 let (tx, rx) = tokio::sync::oneshot::channel::<SseEndReason>();
12187 let tap = SseTap::new(stream, move |_u, reason| {
12188 let _ = tx.send(reason);
12189 });
12190
12191 let _ = axum::body::to_bytes(Body::from_stream(tap), usize::MAX)
12192 .await
12193 .unwrap();
12194
12195 let reason = rx.await.expect("on_done must fire");
12196 assert_eq!(
12197 reason,
12198 SseEndReason::Eof,
12199 "normal stream end must produce Eof, not Dropped or UpstreamError"
12200 );
12201 }
12202
12203 #[test]
12215 fn build_trace_dated_snapshot_falls_back_to_request_model_for_pricing() {
12216 let config = test_config();
12217 let req = Bytes::from_static(br#"{"model":"claude-haiku-4-5","messages":[]}"#);
12218 let resp = Bytes::from_static(
12220 br#"{"model":"claude-haiku-4-5-20251001","usage":{"input_tokens":500,"output_tokens":100}}"#,
12221 );
12222 let trace = build_trace(
12223 &config,
12224 &req,
12225 &resp,
12226 10,
12227 None,
12228 "anthropic",
12229 "anthropic.messages",
12230 None,
12231 );
12232 let attempt = &trace.attempts[0];
12233
12234 assert!(
12236 attempt.cost_usd > 0.0,
12237 "dated snapshot model must fall back to request model for pricing; got $0"
12238 );
12239 assert_eq!(
12241 attempt.model, "claude-haiku-4-5-20251001",
12242 "attempt model must be the dated snapshot name, not the canonical request model"
12243 );
12244 }
12245
12246 #[test]
12251 fn build_stream_trace_dated_snapshot_falls_back_to_request_model_for_pricing() {
12252 let config = test_config();
12253 let req = Bytes::from_static(br#"{"model":"claude-haiku-4-5","messages":[]}"#);
12254 let usage = SseUsage {
12256 model: Some("claude-haiku-4-5-20251001".to_owned()),
12257 in_tokens: 500,
12258 out_tokens: 100,
12259 ..Default::default()
12260 };
12261 let trace = build_stream_trace_from_usage(
12262 &config,
12263 &req,
12264 10,
12265 None,
12266 usage,
12267 "anthropic",
12268 StreamOutcome::Served,
12269 "anthropic.messages",
12270 None,
12271 );
12272 let attempt = &trace.attempts[0];
12273
12274 assert!(
12275 attempt.cost_usd > 0.0,
12276 "dated snapshot model must fall back to request model for pricing; got $0"
12277 );
12278 assert_eq!(
12279 attempt.model, "claude-haiku-4-5-20251001",
12280 "attempt model must be the dated snapshot name, not the canonical request model"
12281 );
12282 }
12283
12284 #[test]
12287 fn gemini_stream_max_tokens_is_truncated() {
12288 let config = test_config();
12289 let req = Bytes::from_static(br#"{"contents":[]}"#);
12290 let usage = SseUsage {
12291 in_tokens: 10,
12292 out_tokens: 64,
12293 stop_reason: Some("MAX_TOKENS".to_owned()),
12294 completed: true,
12295 ..Default::default()
12296 };
12297 let trace = build_stream_trace_from_usage(
12298 &config,
12299 &req,
12300 10,
12301 None,
12302 usage,
12303 "google",
12304 StreamOutcome::Served,
12305 "gemini.generate_content",
12306 Some("gemini-2.5-pro"),
12307 );
12308 assert!(trace.attempts[0].truncated);
12309 }
12310
12311 #[test]
12319 fn price_fallback_does_not_apply_to_unrelated_unpriced_model() {
12320 let config = test_config();
12321 let req = Bytes::from_static(br#"{"model":"claude-haiku-4-5","messages":[]}"#);
12322 let resp = Bytes::from_static(
12324 br#"{"model":"gpt-99-turbo","usage":{"input_tokens":500,"output_tokens":100}}"#,
12325 );
12326 let trace = build_trace(
12327 &config,
12328 &req,
12329 &resp,
12330 10,
12331 None,
12332 "anthropic",
12333 "anthropic.messages",
12334 None,
12335 );
12336 let attempt = &trace.attempts[0];
12337 assert_eq!(
12338 attempt.cost_usd, 0.0,
12339 "an unrelated unpriced response model must NOT fall back to the request model's price"
12340 );
12341 assert_eq!(
12342 attempt.model, "gpt-99-turbo",
12343 "the unpriced response model name must still appear on the attempt"
12344 );
12345 }
12346
12347 #[test]
12354 fn price_fallback_strips_provider_prefix_from_request_model() {
12355 let config = test_config();
12356 let req = Bytes::from_static(br#"{"model":"openai/gpt-4.1-mini","messages":[]}"#);
12358 let usage = SseUsage {
12360 model: Some("gpt-4.1-mini-20250514".to_owned()),
12361 in_tokens: 300,
12362 out_tokens: 50,
12363 ..Default::default()
12364 };
12365 let trace = build_stream_trace_from_usage(
12366 &config,
12367 &req,
12368 10,
12369 None,
12370 usage,
12371 "openai",
12372 StreamOutcome::Served,
12373 "openai.chat_completions",
12374 None,
12375 );
12376 let attempt = &trace.attempts[0];
12377 assert!(
12379 attempt.cost_usd > 0.0,
12380 "snapshot of request model must be priced correctly even with provider-prefix in req body"
12381 );
12382 assert_eq!(
12383 attempt.model, "gpt-4.1-mini-20250514",
12384 "attempt model must be the snapshot name from SSE, not the canonical request model"
12385 );
12386 }
12387
12388 fn test_state_with_channel() -> (AppState, tokio::sync::mpsc::Receiver<firstpass_core::Trace>) {
12392 let config = test_config();
12393 let (traces, rx) = mpsc::channel(4);
12394 let state = AppState {
12395 config: Arc::new(config),
12396 http: reqwest::Client::new(),
12397 providers: ProviderRegistry::from_map(HashMap::new()),
12398 gate_health: Arc::new(GateHealthRegistry::new()),
12399 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
12400 paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
12401 experiments: Arc::default(),
12402 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
12403 traces,
12404 adaptive: None,
12405 eprocess: None,
12406 bandit: None,
12407 promoter: None,
12408 verified_cache: None,
12409 predictor: None,
12410 prior: None,
12411 tenant_rate_limiter: None,
12412 spill: None,
12413 };
12414 (state, rx)
12415 }
12416
12417 #[tokio::test]
12424 async fn stream_callback_upstream_error_records_billed_usage_as_error() {
12425 let (state, mut rx) = test_state_with_channel();
12426 let req = Bytes::from_static(br#"{"model":"claude-haiku-4-5","messages":[]}"#);
12427 let usage = SseUsage {
12428 model: Some("claude-haiku-4-5".to_owned()),
12429 in_tokens: 200,
12430 out_tokens: 40,
12431 ..Default::default()
12432 };
12433 let cb = stream_trace_callback(
12435 &state,
12436 req,
12437 15,
12438 None,
12439 "default".to_owned(),
12440 "anthropic",
12441 true,
12442 "anthropic.messages",
12443 None,
12444 );
12445 cb(usage, SseEndReason::UpstreamError);
12446
12447 let trace = tokio::time::timeout(std::time::Duration::from_secs(5), rx.recv())
12448 .await
12449 .expect("trace within 5s")
12450 .expect("a trace");
12451 assert_eq!(
12452 trace.final_.served_from,
12453 ServedFrom::Error,
12454 "UpstreamError must produce ServedFrom::Error"
12455 );
12456 assert_eq!(trace.attempts.len(), 1, "one attempt with billed usage");
12457 let attempt = &trace.attempts[0];
12458 assert_eq!(
12459 attempt.verdict,
12460 Verdict::Abstain,
12461 "UpstreamError attempt must be Abstain, not Pass"
12462 );
12463 assert!(
12464 attempt.cost_usd > 0.0,
12465 "billed tokens must produce non-zero cost in the error trace"
12466 );
12467 assert_eq!(attempt.in_tokens, 200);
12468 }
12469
12470 async fn dropped_receipt(usage: SseUsage) -> Trace {
12472 let (state, mut rx) = test_state_with_channel();
12473 let req = Bytes::from_static(br#"{"model":"claude-haiku-4-5","messages":[]}"#);
12474 let cb = stream_trace_callback(
12475 &state,
12476 req,
12477 5,
12478 None,
12479 "default".to_owned(),
12480 "anthropic",
12481 true,
12482 "anthropic.messages",
12483 None,
12484 );
12485 cb(usage, SseEndReason::Dropped);
12486 tokio::time::timeout(std::time::Duration::from_secs(5), rx.recv())
12487 .await
12488 .expect("trace within 5s")
12489 .expect("a trace")
12490 }
12491
12492 #[tokio::test]
12496 async fn stream_callback_dropped_before_completion_is_a_partial_not_a_pass() {
12497 for usage in [
12498 SseUsage::default(),
12499 SseUsage {
12500 model: Some("claude-haiku-4-5".to_owned()),
12501 in_tokens: 100,
12502 out_tokens: 30,
12503 ..Default::default()
12504 },
12505 ] {
12506 let had_tokens = usage.in_tokens;
12507 let trace = dropped_receipt(usage).await;
12508 assert_eq!(trace.final_.served_from, ServedFrom::Attempt);
12509 assert_eq!(trace.attempts.len(), 1);
12510 assert_eq!(trace.attempts[0].verdict, Verdict::Abstain, "never a pass");
12511 assert_eq!(trace.attempts[0].in_tokens, had_tokens, "billed usage kept");
12512 }
12513 }
12514
12515 #[tokio::test]
12517 async fn stream_callback_dropped_after_completion_is_a_served_pass() {
12518 let trace = dropped_receipt(SseUsage {
12519 model: Some("claude-haiku-4-5".to_owned()),
12520 in_tokens: 100,
12521 out_tokens: 30,
12522 stop_reason: Some("end_turn".to_owned()),
12523 completed: true,
12524 ..Default::default()
12525 })
12526 .await;
12527 assert_eq!(trace.final_.served_from, ServedFrom::Attempt);
12528 assert_eq!(trace.attempts[0].verdict, Verdict::Pass);
12529 }
12530
12531 #[test]
12533 fn exact_provider_qualified_model_is_priced() {
12534 let config = test_config();
12535 let req = Bytes::from_static(br#"{"model":"openai/gpt-4.1-mini","messages":[]}"#);
12536 let resp = Bytes::from_static(br#"{"usage":{"prompt_tokens":100,"completion_tokens":10}}"#);
12537 let trace = build_trace(
12538 &config,
12539 &req,
12540 &resp,
12541 1,
12542 None,
12543 "openai",
12544 "openai.chat_completions",
12545 None,
12546 );
12547 assert!(trace.attempts[0].cost_usd > 0.0, "{:?}", trace.attempts[0]);
12548 }
12549
12550 #[tokio::test]
12553 async fn stream_callback_eof_before_completion_is_a_partial() {
12554 let (state, mut rx) = test_state_with_channel();
12555 let req = Bytes::from_static(br#"{"model":"claude-haiku-4-5","messages":[]}"#);
12556 for (completed, want) in [(false, Verdict::Abstain), (true, Verdict::Pass)] {
12557 let cb = stream_trace_callback(
12558 &state,
12559 req.clone(),
12560 5,
12561 None,
12562 "default".to_owned(),
12563 "anthropic",
12564 true,
12565 "anthropic.messages",
12566 None,
12567 );
12568 cb(
12569 SseUsage {
12570 model: Some("claude-haiku-4-5".to_owned()),
12571 in_tokens: 50,
12572 completed,
12573 ..Default::default()
12574 },
12575 SseEndReason::Eof,
12576 );
12577 let trace = tokio::time::timeout(std::time::Duration::from_secs(5), rx.recv())
12578 .await
12579 .expect("trace within 5s")
12580 .expect("a trace");
12581 assert_eq!(trace.attempts[0].verdict, want, "completed={completed}");
12582 }
12583 }
12584
12585 #[test]
12587 fn sse_parser_tolerates_crlf() {
12588 let mut u = SseUsage::default();
12589 let (mut buf, mut over) = (Vec::new(), false);
12590 parse_sse_chunk_into(
12591 b"data: {\"type\":\"message_start\",\"message\":{\"model\":\"m\",\"usage\":{\"input_tokens\":7}}}\r\ndata: [DONE]\r\n",
12592 &mut buf,
12593 &mut over,
12594 &mut u,
12595 );
12596 assert_eq!(u.in_tokens, 7);
12597 assert!(u.completed);
12598 }
12599
12600 #[test]
12602 fn sse_completion_is_set_only_by_terminal_events() {
12603 let parse = |lines: &[&str]| {
12604 let mut u = SseUsage::default();
12605 let (mut buf, mut over) = (Vec::new(), false);
12606 for l in lines {
12607 parse_sse_chunk_into(format!("{l}\n").as_bytes(), &mut buf, &mut over, &mut u);
12608 }
12609 u.completed
12610 };
12611 assert!(!parse(&[
12612 r#"data: {"type":"message_start","message":{"model":"m","usage":{"input_tokens":5}}}"#
12613 ]));
12614 assert!(parse(&[
12615 r#"data: {"type":"message_delta","usage":{"output_tokens":3},"delta":{"stop_reason":"end_turn"}}"#
12616 ]));
12617 assert!(parse(&[r#"data: {"type":"message_stop"}"#]));
12618 assert!(!parse(&[
12619 r#"data: {"model":"gpt","choices":[{"delta":{"content":"hi"},"finish_reason":null}]}"#
12620 ]));
12621 assert!(parse(&[
12622 r#"data: {"model":"gpt","choices":[{"finish_reason":"stop"}]}"#
12623 ]));
12624 assert!(parse(&["data: [DONE]"]));
12625 }
12626
12627 #[test]
12630 fn sse_tap_drop_before_exhaustion_fires_dropped() {
12631 use std::sync::{Arc, Mutex};
12632 let (c1, _c2) = anthropic_sse_bytes();
12633 let stream = TestStream::from_ok(vec![c1, b"more-data"]);
12635 let captured = Arc::new(Mutex::new(None::<SseEndReason>));
12636 let cap2 = captured.clone();
12637 let tap = SseTap::new(stream, move |_u, reason| {
12638 *cap2.lock().unwrap() = Some(reason);
12639 });
12640 drop(tap);
12642 let reason = captured.lock().unwrap().expect("on_done must fire on drop");
12643 assert_eq!(
12644 reason,
12645 SseEndReason::Dropped,
12646 "tap dropped before stream ends must fire Dropped"
12647 );
12648 }
12649
12650 #[test]
12654 fn error_trace_carries_upstream_provider() {
12655 let config = test_config();
12656 let req = Bytes::from_static(br#"{"model":"claude-haiku-4-5","messages":[]}"#);
12657 let trace = build_error_trace(&config, &req, 10, None, "anthropic", "anthropic.messages");
12658 assert_eq!(
12659 trace.upstream_provider, "anthropic",
12660 "error trace must record the provider for metrics even without an attempt"
12661 );
12662 }
12663
12664 #[tokio::test]
12672 async fn observe_records_upstream_429_openai_both_paths() {
12673 const ERR: &str = r#"{"error":{"message":"Rate limit reached","type":"rate_limit_error","code":"rate_limit_exceeded"}}"#;
12674 let app = axum::Router::new().route(
12675 "/v1/chat/completions",
12676 axum::routing::post(|| async {
12677 (
12678 axum::http::StatusCode::TOO_MANY_REQUESTS,
12679 [(axum::http::header::CONTENT_TYPE, "application/json")],
12680 ERR,
12681 )
12682 }),
12683 );
12684 let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
12685 .await
12686 .expect("bind");
12687 let addr = listener.local_addr().expect("addr");
12688 tokio::spawn(async move {
12689 let _ = axum::serve(listener, app).await;
12690 });
12691 let base_url = format!("http://{addr}");
12692 let config = ProxyConfig::from_lookup(|k| match k {
12693 "FIRSTPASS_UPSTREAM_OPENAI" => Some(base_url.clone()),
12694 _ => None,
12695 })
12696 .unwrap();
12697 let (traces, mut rx) = mpsc::channel(4);
12698 let state = AppState {
12699 config: Arc::new(config),
12700 http: reqwest::Client::new(),
12701 providers: ProviderRegistry::from_map(HashMap::new()),
12702 gate_health: Arc::new(GateHealthRegistry::new()),
12703 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
12704 paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
12705 experiments: Arc::default(),
12706 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
12707 traces,
12708 adaptive: None,
12709 eprocess: None,
12710 bandit: None,
12711 promoter: None,
12712 verified_cache: None,
12713 predictor: None,
12714 prior: None,
12715 tenant_rate_limiter: None,
12716 spill: None,
12717 };
12718
12719 for body in [
12720 &br#"{"model":"gpt-4.1-mini","messages":[{"role":"user","content":"hi"}]}"#[..],
12722 &br#"{"model":"gpt-4.1-mini","stream":true,"messages":[{"role":"user","content":"hi"}]}"#[..],
12724 ] {
12725 let resp = observe_passthrough_openai_path(
12726 state.clone(),
12727 HeaderMap::new(),
12728 Bytes::from_static(body),
12729 None,
12730 "default".to_owned(),
12731 crate::upstream::OPENAI_CHAT_PATH,
12732 )
12733 .await;
12734 assert_eq!(
12735 resp.status(),
12736 axum::http::StatusCode::TOO_MANY_REQUESTS,
12737 "upstream 429 must be relayed, stream={}",
12738 String::from_utf8_lossy(body).contains("stream")
12739 );
12740 let relayed = axum::body::to_bytes(resp.into_body(), 1 << 20)
12741 .await
12742 .unwrap();
12743 assert_eq!(relayed.as_ref(), ERR.as_bytes(), "body relayed byte-for-byte");
12744 let trace = tokio::time::timeout(std::time::Duration::from_secs(5), rx.recv())
12745 .await
12746 .expect("trace within 5s")
12747 .expect("a trace");
12748 assert_eq!(
12749 trace.final_.served_from,
12750 ServedFrom::Error,
12751 "OpenAI 429 must be ServedFrom::Error, stream={}",
12752 String::from_utf8_lossy(body).contains("stream")
12753 );
12754 assert!(trace.attempts.is_empty(), "no attempts on 429");
12755 }
12756 }
12757
12758 fn thinking_tool_resp() -> ModelResponse {
12761 ModelResponse {
12762 model: "anthropic/claude-3-7-sonnet-20250219".to_owned(),
12763 text: String::new(),
12764 in_tokens: 20,
12765 cache_write_tokens: 100,
12766 cache_read_tokens: 50,
12767 out_tokens: 30,
12768 raw: serde_json::json!({
12769 "stop_reason": "tool_use",
12770 "stop_sequence": null,
12771 "content": [
12772 { "type": "thinking", "thinking": "I need to call the tool.", "signature": "sig_abc123" },
12773 { "type": "tool_use", "id": "tu_1", "name": "search", "input": { "q": "test" } }
12774 ]
12775 }),
12776 }
12777 }
12778
12779 #[test]
12780 fn anthropic_response_json_carries_stop_reason_and_cache_tokens() {
12781 let resp = thinking_tool_resp();
12782 let json = anthropic_response_json(&resp);
12783
12784 assert_eq!(
12785 json["stop_reason"], "tool_use",
12786 "stop_reason must be forwarded"
12787 );
12788 assert_eq!(json["stop_sequence"], Value::Null);
12789
12790 assert_eq!(json["usage"]["cache_creation_input_tokens"], 100);
12792 assert_eq!(json["usage"]["cache_read_input_tokens"], 50);
12793 assert_eq!(json["usage"]["input_tokens"], 20);
12794 assert_eq!(json["usage"]["output_tokens"], 30);
12795
12796 let content = json["content"].as_array().unwrap();
12798 assert_eq!(content[0]["type"], "thinking");
12799 assert_eq!(content[1]["type"], "tool_use");
12800 }
12801
12802 #[test]
12803 fn anthropic_response_json_omits_cache_tokens_when_zero() {
12804 let resp = ModelResponse {
12805 cache_write_tokens: 0,
12806 cache_read_tokens: 0,
12807 ..thinking_tool_resp()
12808 };
12809 let json = anthropic_response_json(&resp);
12810 assert!(
12811 json["usage"]["cache_creation_input_tokens"].is_null(),
12812 "zero cache_write omitted"
12813 );
12814 assert!(
12815 json["usage"]["cache_read_input_tokens"].is_null(),
12816 "zero cache_read omitted"
12817 );
12818 }
12819
12820 #[test]
12821 fn anthropic_response_json_maps_openai_finish_reason_to_anthropic_stop_reason() {
12822 let cases = [
12825 ("length", "max_tokens"),
12826 ("stop", "end_turn"),
12827 ("content_filter", "end_turn"),
12828 ];
12829 for (finish_reason, expected_stop_reason) in cases {
12830 let resp = ModelResponse {
12831 model: "gpt-4o".to_owned(),
12832 text: String::new(),
12833 in_tokens: 5,
12834 cache_write_tokens: 0,
12835 cache_read_tokens: 0,
12836 out_tokens: 3,
12837 raw: serde_json::json!({
12838 "choices": [{ "finish_reason": finish_reason, "message": { "content": null } }]
12839 }),
12840 };
12841 let json = anthropic_response_json(&resp);
12842 assert_eq!(
12843 json["stop_reason"].as_str(),
12844 Some(expected_stop_reason),
12845 "finish_reason={finish_reason} → stop_reason={expected_stop_reason}"
12846 );
12847 }
12848 let resp_with_calls = ModelResponse {
12851 model: "gpt-4o".to_owned(),
12852 text: String::new(),
12853 in_tokens: 5,
12854 cache_write_tokens: 0,
12855 cache_read_tokens: 0,
12856 out_tokens: 3,
12857 raw: serde_json::json!({
12858 "choices": [{ "finish_reason": "tool_calls", "message": {
12859 "content": null,
12860 "tool_calls": [{"id":"c1","type":"function",
12861 "function":{"name":"f","arguments":"{}"}}]
12862 }}]
12863 }),
12864 };
12865 assert_eq!(
12866 anthropic_response_json(&resp_with_calls)["stop_reason"].as_str(),
12867 Some("tool_use"),
12868 "finish_reason=tool_calls with tool_call in message → stop_reason=tool_use"
12869 );
12870 }
12871
12872 #[test]
12873 fn anthropic_sse_from_message_uses_real_stop_reason_and_thinking() {
12874 let resp = thinking_tool_resp();
12875 let message = anthropic_response_json(&resp);
12876 let sse = anthropic_sse_from_message(&message);
12877
12878 let frames: Vec<Value> = sse
12879 .lines()
12880 .filter_map(|l| {
12881 if l.starts_with("event: ") {
12882 return None;
12883 }
12884 l.strip_prefix("data: ")
12885 })
12886 .map(|d| serde_json::from_str::<Value>(d).expect("valid JSON"))
12887 .collect();
12888
12889 let msg_delta = frames
12891 .iter()
12892 .find(|f| f["type"] == "message_delta")
12893 .expect("message_delta frame");
12894 assert_eq!(
12895 msg_delta["delta"]["stop_reason"], "tool_use",
12896 "stop_reason must not be hardcoded to end_turn"
12897 );
12898
12899 assert!(
12901 frames
12902 .iter()
12903 .any(|f| f["content_block"]["type"] == "thinking"),
12904 "thinking block_start must appear"
12905 );
12906 assert!(
12908 frames.iter().any(|f| f["delta"]["type"] == "thinking_delta"
12909 && f["delta"]["thinking"] == "I need to call the tool."),
12910 "thinking_delta must carry the thinking text"
12911 );
12912 assert!(
12914 frames
12915 .iter()
12916 .any(|f| f["delta"]["type"] == "signature_delta"
12917 && f["delta"]["signature"] == "sig_abc123"),
12918 "signature_delta must carry the signature"
12919 );
12920
12921 assert!(
12923 frames
12924 .iter()
12925 .any(|f| f["content_block"]["type"] == "tool_use"
12926 && f["content_block"]["name"] == "search"),
12927 "tool_use block must still appear"
12928 );
12929 }
12930
12931 #[test]
12932 fn anthropic_sse_from_message_handles_redacted_thinking() {
12933 let resp = ModelResponse {
12934 raw: serde_json::json!({
12935 "stop_reason": "end_turn",
12936 "content": [
12937 { "type": "redacted_thinking", "data": "base64encodeddata==" }
12938 ]
12939 }),
12940 ..thinking_tool_resp()
12941 };
12942 let message = anthropic_response_json(&resp);
12943 let sse = anthropic_sse_from_message(&message);
12944 let frames: Vec<Value> = sse
12945 .lines()
12946 .filter_map(|l| {
12947 if l.starts_with("event: ") {
12948 return None;
12949 }
12950 l.strip_prefix("data: ")
12951 })
12952 .map(|d| serde_json::from_str::<Value>(d).expect("valid JSON"))
12953 .collect();
12954
12955 assert!(
12957 frames.iter().any(|f| {
12958 f["type"] == "content_block_start"
12959 && f["content_block"]["type"] == "redacted_thinking"
12960 && f["content_block"]["data"] == "base64encodeddata=="
12961 }),
12962 "content_block_start must carry the full redacted_thinking block with data"
12963 );
12964 assert!(
12966 !frames
12967 .iter()
12968 .any(|f| f["delta"]["type"] == "redacted_thinking_delta"),
12969 "no redacted_thinking_delta must appear"
12970 );
12971 }
12972
12973 #[test]
12976 fn openai_response_json_sets_finish_reason_length_when_truncated() {
12977 let resp = ModelResponse {
12979 model: "gpt-4o".to_owned(),
12980 text: "partial".to_owned(),
12981 in_tokens: 5,
12982 cache_write_tokens: 0,
12983 cache_read_tokens: 0,
12984 out_tokens: 100,
12985 raw: serde_json::json!({ "stop_reason": "max_tokens", "content": [{ "type": "text", "text": "partial" }] }),
12986 };
12987 let json = openai_response_json(&resp);
12988 assert_eq!(json["choices"][0]["finish_reason"], "length");
12989 }
12990
12991 #[test]
12992 fn openai_sse_from_message_tool_calls_have_index() {
12993 let resp = ModelResponse {
12994 model: "gpt-4o".to_owned(),
12995 text: String::new(),
12996 in_tokens: 10,
12997 cache_write_tokens: 0,
12998 cache_read_tokens: 0,
12999 out_tokens: 8,
13000 raw: serde_json::json!({
13001 "content": [
13002 { "type": "tool_use", "id": "c1", "name": "alpha", "input": { "x": 1 } },
13003 { "type": "tool_use", "id": "c2", "name": "beta", "input": { "y": 2 } }
13004 ]
13005 }),
13006 };
13007 let sse = openai_sse_from_message(&openai_response_json(&resp), false);
13008 let frames: Vec<Value> = sse
13010 .lines()
13011 .filter_map(|l| l.strip_prefix("data: "))
13012 .filter(|f| *f != "[DONE]")
13013 .map(|d| serde_json::from_str::<Value>(d).expect("valid JSON"))
13014 .collect();
13015
13016 let tc_frame = frames
13017 .iter()
13018 .find(|f| !f["choices"][0]["delta"]["tool_calls"].is_null())
13019 .expect("tool_calls delta frame");
13020 let tcs = tc_frame["choices"][0]["delta"]["tool_calls"]
13021 .as_array()
13022 .expect("tool_calls array");
13023 assert_eq!(tcs.len(), 2, "both tool calls present");
13024 assert_eq!(tcs[0]["index"], 0, "first call has index 0");
13025 assert_eq!(tcs[1]["index"], 1, "second call has index 1");
13026 }
13027
13028 #[test]
13029 fn openai_sse_from_message_emits_usage_chunk() {
13030 let resp = ModelResponse {
13031 model: "gpt-4o".to_owned(),
13032 text: "hi".to_owned(),
13033 in_tokens: 3,
13034 cache_write_tokens: 0,
13035 cache_read_tokens: 0,
13036 out_tokens: 2,
13037 raw: serde_json::json!({ "content": [{ "type": "text", "text": "hi" }] }),
13038 };
13039 let sse = openai_sse_from_message(&openai_response_json(&resp), true);
13040 let frames: Vec<Value> = sse
13041 .lines()
13042 .filter_map(|l| l.strip_prefix("data: "))
13043 .filter(|f| *f != "[DONE]")
13044 .map(|d| serde_json::from_str::<Value>(d).expect("valid JSON"))
13045 .collect();
13046
13047 let usage_frame = frames
13049 .iter()
13050 .find(|f| f["choices"].as_array().is_some_and(|a| a.is_empty()))
13051 .expect("usage chunk with choices:[] must appear before [DONE]");
13052 assert_eq!(usage_frame["usage"]["prompt_tokens"], 3);
13053 assert_eq!(usage_frame["usage"]["completion_tokens"], 2);
13054
13055 assert_eq!(
13057 sse.lines()
13058 .filter_map(|l| l.strip_prefix("data: "))
13059 .next_back()
13060 .unwrap(),
13061 "[DONE]"
13062 );
13063 }
13064
13065 #[test]
13066 fn openai_sse_from_message_no_usage_chunk_without_include_usage() {
13067 let resp = ModelResponse {
13068 model: "gpt-4o".to_owned(),
13069 text: "hi".to_owned(),
13070 in_tokens: 3,
13071 cache_write_tokens: 0,
13072 cache_read_tokens: 0,
13073 out_tokens: 2,
13074 raw: serde_json::json!({ "content": [{ "type": "text", "text": "hi" }] }),
13075 };
13076 let sse = openai_sse_from_message(&openai_response_json(&resp), false);
13077 let has_empty_choices_chunk = sse
13078 .lines()
13079 .filter_map(|l| l.strip_prefix("data: "))
13080 .filter(|f| *f != "[DONE]")
13081 .any(|d| {
13082 serde_json::from_str::<Value>(d)
13083 .ok()
13084 .and_then(|v| v.get("choices").cloned())
13085 .and_then(|c| c.as_array().map(|a| a.is_empty()))
13086 .unwrap_or(false)
13087 });
13088 assert!(
13089 !has_empty_choices_chunk,
13090 "usage chunk must not appear when include_usage is false"
13091 );
13092 }
13093
13094 #[test]
13095 fn stream_options_include_usage_parses_correctly() {
13096 assert!(stream_options_include_usage(
13097 br#"{"stream": true, "stream_options": {"include_usage": true}}"#
13098 ));
13099 assert!(!stream_options_include_usage(
13100 br#"{"stream": true, "stream_options": {"include_usage": false}}"#
13101 ));
13102 assert!(!stream_options_include_usage(br#"{"stream": true}"#));
13103 assert!(!stream_options_include_usage(b"not json"));
13104 }
13105
13106 #[test]
13109 fn translated_tool_use_input_is_always_an_object() {
13110 for args in ["null", "[1]", "\"s\""] {
13113 let resp = ModelResponse {
13114 model: "gpt-4o".to_owned(),
13115 text: String::new(),
13116 in_tokens: 1,
13117 cache_write_tokens: 0,
13118 cache_read_tokens: 0,
13119 out_tokens: 1,
13120 raw: serde_json::json!({"choices": [{"finish_reason": "tool_calls", "message": {
13121 "role": "assistant", "content": null,
13122 "tool_calls": [{"id": "c", "type": "function",
13123 "function": {"name": "f", "arguments": args}}]}}]}),
13124 };
13125 let json = anthropic_response_json(&resp);
13126 assert!(
13127 json["content"][0]["input"].is_object(),
13128 "args={args}: {json}"
13129 );
13130 }
13131 }
13132
13133 #[test]
13134 fn anthropic_response_json_translates_openai_tool_calls_to_tool_use_blocks() {
13135 let resp = ModelResponse {
13138 model: "gpt-4o".to_owned(),
13139 text: String::new(),
13140 in_tokens: 5,
13141 cache_write_tokens: 0,
13142 cache_read_tokens: 0,
13143 out_tokens: 3,
13144 raw: serde_json::json!({
13145 "choices": [{
13146 "finish_reason": "tool_calls",
13147 "message": {
13148 "role": "assistant",
13149 "content": null,
13150 "tool_calls": [{
13151 "id": "call_abc123",
13152 "type": "function",
13153 "function": {
13154 "name": "get_weather",
13155 "arguments": "{\"location\":\"London\"}"
13156 }
13157 }]
13158 }
13159 }],
13160 "usage": {"prompt_tokens": 5, "completion_tokens": 3}
13161 }),
13162 };
13163 let json = anthropic_response_json(&resp);
13164 let content = json["content"].as_array().expect("content must be array");
13165 assert_eq!(content.len(), 1, "exactly one tool_use block");
13166 assert_eq!(content[0]["type"], "tool_use");
13167 assert_eq!(content[0]["id"], "call_abc123");
13168 assert_eq!(content[0]["name"], "get_weather");
13169 assert_eq!(content[0]["input"]["location"], "London");
13170 assert_eq!(json["stop_reason"], "tool_use");
13172 }
13173
13174 #[test]
13175 fn anthropic_response_json_openai_tool_calls_with_text_emits_both_blocks() {
13176 let resp = ModelResponse {
13177 model: "gpt-4o".to_owned(),
13178 text: "Here is the weather call.".to_owned(),
13179 in_tokens: 5,
13180 cache_write_tokens: 0,
13181 cache_read_tokens: 0,
13182 out_tokens: 10,
13183 raw: serde_json::json!({
13184 "choices": [{
13185 "finish_reason": "tool_calls",
13186 "message": {
13187 "content": "Here is the weather call.",
13188 "tool_calls": [{
13189 "id": "call_xyz",
13190 "type": "function",
13191 "function": { "name": "search", "arguments": "{}" }
13192 }]
13193 }
13194 }]
13195 }),
13196 };
13197 let json = anthropic_response_json(&resp);
13198 let content = json["content"].as_array().expect("content array");
13199 assert!(
13200 content.iter().any(|b| b["type"] == "text"),
13201 "text block must appear when message content is non-empty"
13202 );
13203 assert!(
13204 content.iter().any(|b| b["type"] == "tool_use"),
13205 "tool_use block must appear"
13206 );
13207 }
13208
13209 #[test]
13210 fn anthropic_response_json_no_tool_use_blocks_never_emits_tool_use_stop_reason() {
13211 let resp = ModelResponse {
13215 model: "gpt-4o".to_owned(),
13216 text: "some text".to_owned(),
13217 in_tokens: 3,
13218 cache_write_tokens: 0,
13219 cache_read_tokens: 0,
13220 out_tokens: 2,
13221 raw: serde_json::json!({
13222 "choices": [{
13223 "finish_reason": "tool_calls",
13224 "message": {
13225 "content": "some text",
13226 "tool_calls": [] }
13228 }]
13229 }),
13230 };
13231 let json = anthropic_response_json(&resp);
13232 assert_eq!(
13233 json["stop_reason"], "end_turn",
13234 "stop_reason must be end_turn when content has no tool_use blocks"
13235 );
13236 }
13237
13238 #[test]
13239 fn anthropic_sse_from_message_tool_use_block_produces_input_json_delta() {
13240 let resp = ModelResponse {
13243 model: "gpt-4o".to_owned(),
13244 text: String::new(),
13245 in_tokens: 4,
13246 cache_write_tokens: 0,
13247 cache_read_tokens: 0,
13248 out_tokens: 2,
13249 raw: serde_json::json!({
13250 "choices": [{
13251 "finish_reason": "tool_calls",
13252 "message": {
13253 "content": null,
13254 "tool_calls": [{
13255 "id": "call_1",
13256 "type": "function",
13257 "function": { "name": "lookup", "arguments": "{\"q\":\"test\"}" }
13258 }]
13259 }
13260 }]
13261 }),
13262 };
13263 let message = anthropic_response_json(&resp);
13264 let sse = anthropic_sse_from_message(&message);
13265 let frames: Vec<Value> = sse
13266 .lines()
13267 .filter_map(|l| l.strip_prefix("data: "))
13268 .map(|d| serde_json::from_str::<Value>(d).expect("valid JSON"))
13269 .collect();
13270
13271 assert!(
13272 frames
13273 .iter()
13274 .any(|f| f["content_block"]["type"] == "tool_use"
13275 && f["content_block"]["name"] == "lookup"),
13276 "content_block_start with tool_use block must appear"
13277 );
13278 assert!(
13279 frames
13280 .iter()
13281 .any(|f| f["delta"]["type"] == "input_json_delta"),
13282 "input_json_delta must be emitted for the tool call"
13283 );
13284 }
13285
13286 #[test]
13287 fn anthropic_sse_message_start_has_null_stop_reason() {
13288 let resp = thinking_tool_resp();
13291 let message = anthropic_response_json(&resp);
13292 let sse = anthropic_sse_from_message(&message);
13293 let frames: Vec<Value> = sse
13294 .lines()
13295 .filter_map(|l| l.strip_prefix("data: "))
13296 .map(|d| serde_json::from_str::<Value>(d).expect("valid JSON"))
13297 .collect();
13298
13299 let start = frames
13300 .iter()
13301 .find(|f| f["type"] == "message_start")
13302 .expect("message_start must appear");
13303 assert!(
13304 start["message"]["stop_reason"].is_null(),
13305 "message_start.message.stop_reason must be null, got: {}",
13306 start["message"]["stop_reason"]
13307 );
13308 assert!(
13309 start["message"]["stop_sequence"].is_null(),
13310 "message_start.message.stop_sequence must be null"
13311 );
13312 let delta = frames
13314 .iter()
13315 .find(|f| f["type"] == "message_delta")
13316 .expect("message_delta must appear");
13317 assert!(
13318 !delta["delta"]["stop_reason"].is_null(),
13319 "message_delta.delta.stop_reason must be non-null"
13320 );
13321 }
13322
13323 #[test]
13324 fn anthropic_sse_message_start_has_zero_output_tokens() {
13325 let resp = thinking_tool_resp(); let message = anthropic_response_json(&resp);
13331 assert!(
13333 message["usage"]["output_tokens"].as_u64().unwrap_or(0) > 0,
13334 "precondition: source message must have non-zero output_tokens"
13335 );
13336 let sse = anthropic_sse_from_message(&message);
13337 let frames: Vec<serde_json::Value> = sse
13338 .lines()
13339 .filter_map(|l| l.strip_prefix("data: "))
13340 .map(|d| serde_json::from_str::<serde_json::Value>(d).expect("valid JSON"))
13341 .collect();
13342 let start = frames
13343 .iter()
13344 .find(|f| f["type"] == "message_start")
13345 .expect("message_start must appear");
13346 assert_eq!(
13347 start["message"]["usage"]["output_tokens"].as_u64(),
13348 Some(0),
13349 "message_start.message.usage.output_tokens must be 0, got: {}",
13350 start["message"]["usage"]["output_tokens"]
13351 );
13352 }
13353
13354 #[test]
13355 fn anthropic_response_json_maps_gemini_finish_reason() {
13356 for (gemini_reason, expected) in [("MAX_TOKENS", "max_tokens"), ("STOP", "end_turn")] {
13359 let resp = ModelResponse {
13360 model: "google/gemini-3.5-flash-lite".to_owned(),
13361 text: "some text".to_owned(),
13362 in_tokens: 5,
13363 cache_write_tokens: 0,
13364 cache_read_tokens: 0,
13365 out_tokens: 3,
13366 raw: serde_json::json!({
13367 "candidates": [{"finishReason": gemini_reason,
13368 "content": {"parts": [{"text": "some text"}]}}]
13369 }),
13370 };
13371 let json = anthropic_response_json(&resp);
13372 assert_eq!(
13373 json["stop_reason"].as_str(),
13374 Some(expected),
13375 "Gemini finishReason={gemini_reason} → stop_reason={expected}"
13376 );
13377 }
13378 }
13379
13380 #[tokio::test]
13385 async fn count_tokens_passthrough_relays_upstream_response() {
13386 let mock = axum::Router::new().route(
13387 "/v1/messages/count_tokens",
13388 axum::routing::post(|| async { axum::Json(serde_json::json!({"input_tokens": 42})) }),
13389 );
13390 let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
13391 let addr = listener.local_addr().unwrap();
13392 tokio::spawn(async move {
13393 axum::serve(listener, mock).await.unwrap();
13394 });
13395 let upstream = format!("http://{addr}");
13396
13397 let config = ProxyConfig::from_lookup(move |k| match k {
13398 "FIRSTPASS_UPSTREAM_ANTHROPIC" => Some(upstream.clone()),
13399 _ => None,
13400 })
13401 .unwrap();
13402 let (traces, mut rx) = mpsc::channel(64);
13403 let state = AppState {
13404 config: Arc::new(config),
13405 http: reqwest::Client::new(),
13406 providers: ProviderRegistry::new("http://127.0.0.1:1", "http://127.0.0.1:1"),
13407 gate_health: Arc::new(GateHealthRegistry::new()),
13408 shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
13409 paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
13410 experiments: Arc::default(),
13411 guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
13412 traces,
13413 adaptive: None,
13414 eprocess: None,
13415 bandit: None,
13416 promoter: None,
13417 verified_cache: None,
13418 predictor: None,
13419 prior: None,
13420 tenant_rate_limiter: None,
13421 spill: None,
13422 };
13423
13424 let body = Bytes::from_static(br#"{"model":"claude-3-haiku-20240307","messages":[]}"#);
13425 let resp = count_tokens_passthrough(
13426 State(state),
13427 Extension(TenantId("default".to_owned())),
13428 HeaderMap::new(),
13429 body,
13430 )
13431 .await;
13432
13433 assert_eq!(resp.status(), axum::http::StatusCode::OK);
13434 let json = body_json(resp).await;
13435 assert_eq!(json["input_tokens"], 42, "token count must be relayed");
13436 assert!(
13438 rx.try_recv().is_err(),
13439 "count_tokens must not enqueue a trace"
13440 );
13441 }
13442
13443 #[test]
13446 fn percent_decode_ascii_handles_common_cases() {
13447 assert_eq!(percent_decode_ascii("AIza-key"), "AIza-key");
13448 assert_eq!(percent_decode_ascii("AIza%3Dkey"), "AIza=key"); assert_eq!(percent_decode_ascii("key%2Bplus"), "key+plus"); assert_eq!(percent_decode_ascii("%41%42%43"), "ABC"); assert_eq!(percent_decode_ascii("%"), "%"); assert_eq!(percent_decode_ascii("%XY"), "%XY"); }
13455
13456 #[test]
13457 fn parse_gemini_query_decodes_key() {
13458 let (key, scrubbed) = parse_gemini_query(Some("key=AIza%3Dsecret&alt=sse"));
13459 assert_eq!(key.as_deref(), Some("AIza=secret"), "key must be decoded");
13460 assert_eq!(scrubbed, "alt=sse", "key must be scrubbed from query");
13461 }
13463
13464 #[test]
13465 fn gemini_sse_tokens_and_finish_reason_parsed() {
13466 let mut u = SseUsage::default();
13467 let (mut buf, mut over) = (Vec::new(), false);
13468 parse_sse_chunk_into(
13471 b"data: {\"candidates\":[{\"finishReason\":\"STOP\"}],\"usageMetadata\":{\"promptTokenCount\":10,\"candidatesTokenCount\":5,\"thoughtsTokenCount\":2}}\n",
13472 &mut buf,
13473 &mut over,
13474 &mut u,
13475 );
13476 assert_eq!(u.in_tokens, 10, "promptTokenCount");
13477 assert_eq!(u.out_tokens, 7, "candidatesTokenCount + thoughtsTokenCount");
13478 assert!(u.completed, "STOP sets completed");
13479 }
13481
13482 #[test]
13483 fn gemini_sse_finish_reason_and_usage_in_separate_chunks_still_complete() {
13484 let mut u = SseUsage::default();
13487 let (mut buf, mut over) = (Vec::new(), false);
13488 parse_sse_chunk_into(
13489 b"data: {\"candidates\":[{\"content\":{\"parts\":[{\"text\":\"hi\"}]},\"finishReason\":\"STOP\"}]}\n",
13490 &mut buf,
13491 &mut over,
13492 &mut u,
13493 );
13494 parse_sse_chunk_into(
13495 b"data: {\"usageMetadata\":{\"promptTokenCount\":100,\"cachedContentTokenCount\":60,\"candidatesTokenCount\":5}}\n",
13496 &mut buf,
13497 &mut over,
13498 &mut u,
13499 );
13500 assert!(
13501 u.completed,
13502 "finishReason in its own chunk completes the stream"
13503 );
13504 assert_eq!(u.stop_reason.as_deref(), Some("STOP"));
13505 assert_eq!(
13506 u.in_tokens, 40,
13507 "cached content is split out of promptTokenCount"
13508 );
13509 assert_eq!(u.cache_read_tokens, 60);
13510 assert_eq!(u.out_tokens, 5);
13511 }
13512
13513 #[test]
13514 fn gemini_buffered_usage_splits_cached_content() {
13515 let body = br#"{"candidates":[],"usageMetadata":{"promptTokenCount":100,"cachedContentTokenCount":60,"candidatesTokenCount":5,"thoughtsTokenCount":2}}"#;
13516 let u = response_usage(body);
13517 assert_eq!(
13518 (u.in_tokens, u.cache_read_tokens, u.out_tokens),
13519 (40, 60, 7)
13520 );
13521 }
13522
13523 #[test]
13524 fn gemini_sse_intermediate_chunk_updates_tokens_without_completing() {
13525 let mut u = SseUsage::default();
13527 let (mut buf, mut over) = (Vec::new(), false);
13528 parse_sse_chunk_into(
13529 b"data: {\"candidates\":[{\"content\":{\"parts\":[{\"text\":\"hi\"}]}}],\"usageMetadata\":{\"promptTokenCount\":8,\"candidatesTokenCount\":3}}\n",
13530 &mut buf,
13531 &mut over,
13532 &mut u,
13533 );
13534 assert_eq!(u.in_tokens, 8);
13535 assert_eq!(u.out_tokens, 3);
13536 assert!(!u.completed, "no finishReason → not completed");
13537 }
13539
13540 #[test]
13541 fn response_usage_parses_gemini_usage_metadata() {
13542 let body = serde_json::json!({
13543 "candidates": [{ "content": { "role": "model", "parts": [{ "text": "hi" }] } }],
13544 "usageMetadata": {
13545 "promptTokenCount": 20,
13546 "candidatesTokenCount": 5,
13547 "thoughtsTokenCount": 3,
13548 "totalTokenCount": 28,
13549 },
13550 });
13551 let usage = response_usage(body.to_string().as_bytes());
13552 assert_eq!(usage.in_tokens, 20, "promptTokenCount");
13553 assert_eq!(
13554 usage.out_tokens, 8,
13555 "candidatesTokenCount + thoughtsTokenCount"
13556 );
13557 assert_eq!(
13558 usage.model, None,
13559 "Gemini response has no top-level model field"
13560 );
13561 }
13563}