use std::sync::Arc;
use std::time::Instant;
use axum::body::Body;
use axum::extract::{Request, State};
use axum::http::HeaderMap;
use axum::middleware::Next;
use axum::response::{IntoResponse, Response};
use axum::routing::{get, post};
use axum::{Extension, Json, Router};
use bytes::Bytes;
use firstpass_core::features::{hour_bucket, token_bucket};
use firstpass_core::hashchain::sha256_hex;
use firstpass_core::{
Attempt, DeferredVerdict, Dialect, FEATURE_VERSION, Features, FinalOutcome, GENESIS_HASH, Mode,
ModelRef, PolicyRef, ProbeRegime, ProbeSignal, RequestInfo, RoutingMode, Score, ServedFrom,
TaskKind, Trace, Verdict,
features::{DifficultyHint, TrajectorySignals},
};
use futures_core::Stream;
use serde::Deserialize;
use serde_json::Value;
use std::future::Future;
use std::task::{Context, Poll};
use std::time::Duration;
use tokio::sync::mpsc::error::TrySendError;
use uuid::Uuid;
use crate::config::ProxyConfig;
use crate::error::ProxyError;
use crate::gate::{GateHealthRegistry, aggregate_with_policy, resolve_gates};
use crate::provider::{Auth, ChatMessage, ModelRequest, ModelResponse, ProviderRegistry};
use crate::router::{EnforceCtx, EngineOutcome, route_enforce};
use crate::store;
use crate::tenant_auth::{TenantId, auth_middleware};
use crate::upstream::{forward_anthropic, forward_anthropic_streaming};
use firstpass_core::Route;
use firstpass_core::trace::{RiskCertificate, ShadowSignal, ThresholdSource};
#[derive(Clone)]
pub struct AppState {
pub config: Arc<ProxyConfig>,
pub http: reqwest::Client,
pub providers: ProviderRegistry,
pub gate_health: Arc<GateHealthRegistry>,
pub shadow_ledger: Arc<crate::shadow::ShadowLedger>,
pub paired_ledger: Arc<crate::shadow::ShadowLedger>,
pub experiments: Arc<crate::experiment::ExperimentMonitor>,
pub guardrails: Arc<crate::guard::GuardrailRegistry>,
pub traces: store::TraceSender,
pub adaptive: Option<Arc<std::sync::Mutex<firstpass_core::conformal::AdaptiveConformal>>>,
pub eprocess: Option<Arc<std::sync::Mutex<firstpass_core::eprocess::EProcessRiskControl>>>,
pub bandit: Option<Arc<std::sync::Mutex<crate::bandit::StartRungBandit>>>,
pub promoter: Option<Arc<crate::affinity::SessionPromoter>>,
pub verified_cache: Option<Arc<dyn crate::verified_cache::CacheStore>>,
pub predictor: Option<Arc<std::sync::Mutex<firstpass_core::PassPredictor>>>,
pub prior: Option<Arc<crate::prior::PriorClient>>,
pub tenant_rate_limiter: Option<Arc<governor::DefaultKeyedRateLimiter<String>>>,
pub spill: Option<store::SpillHandle>,
}
#[must_use]
fn routing_cfg_condense(state: &AppState) -> Option<&firstpass_core::config::CondenseConfig> {
state
.config
.routing
.as_ref()
.and_then(|r| r.escalation.condense.as_ref())
}
#[must_use]
fn mode_now() -> jiff::Timestamp {
jiff::Timestamp::now()
}
fn cache_hit_trace(
ctx: &crate::router::EnforceCtx<'_>,
entry: &crate::verified_cache::Entry,
ts: jiff::Timestamp,
) -> Trace {
Trace {
trace_id: uuid::Uuid::now_v7(),
prev_hash: firstpass_core::GENESIS_HASH.to_owned(),
tenant_id: ctx.tenant_id.clone(),
session_id: ctx.session_id.clone(),
ts,
mode: Mode::Enforce,
policy: PolicyRef {
id: format!("{}+cache", ctx.policy_id),
explore: false,
propensity: None,
mode_profile: None,
},
request: RequestInfo {
api: ctx.api.clone(),
prompt_hash: ctx.prompt_hash.clone(),
features: ctx.features.clone(),
},
attempts: Vec::new(),
final_: FinalOutcome {
served_rung: None,
served_from: ServedFrom::Cache,
total_cost_usd: 0.0,
gate_cost_usd: 0.0,
total_latency_ms: 0,
escalations: 0,
counterfactual_baseline_usd: entry.original_cost_usd,
savings_usd: entry.original_cost_usd,
cache_source: Some(entry.source),
},
deferred: Vec::new(),
predicted_pass: None,
decision_prior: None,
probe: None,
elastic: None,
risk: None,
rollout: None,
shadow: None,
route_ix: None,
experiment: None,
upstream_provider: String::new(),
}
}
pub async fn build_verified_cache(
config: &ProxyConfig,
) -> Result<Option<Arc<dyn crate::verified_cache::CacheStore>>, String> {
let Some(c) = config
.routing
.as_ref()
.and_then(|r| r.escalation.verified_cache.as_ref())
else {
return Ok(None);
};
if let Some(url) = c.redis_url.as_deref() {
#[cfg(feature = "redis-cache")]
{
let store = crate::verified_cache::RedisStore::connect(url, c.ttl_secs).await?;
tracing::info!("verified cache: shared via redis");
return Ok(Some(Arc::new(store)));
}
#[cfg(not(feature = "redis-cache"))]
{
let _ = url;
return Err(
"[escalation.verified_cache] redis_url is set, but this binary was built without \
the `redis-cache` feature. Rebuild with `--features redis-cache`, or remove \
redis_url to use the in-process cache."
.to_owned(),
);
}
}
Ok(Some(Arc::new(crate::verified_cache::VerifiedCache::new(
std::time::Duration::from_secs(c.ttl_secs),
c.max_entries,
))))
}
pub async fn build_promoter_async(
config: &ProxyConfig,
) -> Result<Option<Arc<crate::affinity::SessionPromoter>>, String> {
let Some(sp) = config
.routing
.as_ref()
.and_then(|r| r.escalation.session_promotion.as_ref())
else {
return Ok(None);
};
let promoter = crate::affinity::SessionPromoter::new(sp.clone()).map_err(|e| e.to_string())?;
if let Some(url) = sp.redis_url.as_deref() {
#[cfg(feature = "redis-cache")]
{
let ttl = sp.window_duration().map_err(|e| e.to_string())?.as_secs();
let store = crate::affinity::RedisPromotionStore::connect(url, ttl).await?;
tracing::info!("session promotion: shared via redis");
return Ok(Some(Arc::new(promoter.with_shared(Arc::new(store)))));
}
#[cfg(not(feature = "redis-cache"))]
{
let _ = url;
return Err(
"[escalation.session_promotion] redis_url is set, but this binary was \
built without the `redis-cache` feature. Rebuild with \
`--features redis-cache`, or remove redis_url to keep promotion \
in-process."
.to_owned(),
);
}
}
Ok(Some(Arc::new(promoter)))
}
#[must_use]
pub fn build_paired_ledger(config: &ProxyConfig, shared_log: bool) -> crate::shadow::ShadowLedger {
let ledger = crate::shadow::ShadowLedger::new();
let any_paired = config.routing.as_ref().is_some_and(|r| {
r.routes
.iter()
.any(|route| route.challenger.as_ref().is_some_and(|c| c.is_paired()))
});
if !any_paired {
return ledger;
}
match crate::store::load_all_experiment_pairs(&config.db_path) {
Ok(rows) => {
let now = jiff::Timestamp::now();
let mut seen = std::collections::HashSet::new();
for row in rows
.iter()
.filter(|r| !shared_log || r.in_log != crate::store::InLogStatus::InLog)
.filter(|r| r.trace_id.is_nil() || seen.insert(r.trace_id))
{
ledger.seed(
&row.tenant,
row.route_ix,
row.challenger_cost_usd,
row.ts,
now,
);
}
}
Err(e) => tracing::warn!(%e, "paired ledger: could not read today's spend from the store"),
}
ledger
}
pub async fn build_experiment_log(
config: &ProxyConfig,
) -> Result<Option<Arc<dyn crate::experiment::ExperimentLogBackend>>, String> {
use std::collections::HashSet;
let urls: HashSet<&str> = config
.routing
.iter()
.flat_map(|r| &r.routes)
.filter_map(|r| r.challenger.as_ref()?.redis_url.as_deref())
.collect();
let mut url_iter = urls.into_iter();
let Some(url) = url_iter.next() else {
return Ok(None); };
if let Some(other) = url_iter.next() {
let mut found = vec![url, other];
found.extend(url_iter);
found.sort_unstable();
return Err(format!(
"[route.challenger] multiple distinct redis_url values across routes are not \
supported — all experiment logs must share one Redis instance (keys are namespaced \
by tenant + route + fingerprint). URLs found: {found:?}"
));
}
#[cfg(feature = "redis-cache")]
{
let log = crate::experiment::RedisExperimentLog::connect(url).await?;
tracing::info!("experiment log: shared via redis");
Ok(Some(
Arc::new(log) as Arc<dyn crate::experiment::ExperimentLogBackend>
))
}
#[cfg(not(feature = "redis-cache"))]
{
let _ = url;
Err(
"[route.challenger] redis_url is set, but this binary was built without the \
`redis-cache` feature. Rebuild with `--features redis-cache`, or remove redis_url \
from all [route.challenger] sections to keep experiment state in-process."
.to_owned(),
)
}
}
#[must_use]
pub fn build_tenant_rate_limiter(
config: &ProxyConfig,
) -> Option<Arc<governor::DefaultKeyedRateLimiter<String>>> {
let per_sec = config.tenant_rate_per_sec?;
Some(Arc::new(governor::RateLimiter::keyed(
governor::Quota::per_second(per_sec),
)))
}
pub async fn tenant_rate_limit_middleware(
State(state): State<AppState>,
Extension(tenant): Extension<TenantId>,
req: Request,
next: Next,
) -> Response {
if let Some(limiter) = &state.tenant_rate_limiter
&& limiter.check_key(&tenant.0).is_err()
{
return ProxyError::RateLimited.into_response();
}
next.run(req).await
}
impl std::fmt::Debug for AppState {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("AppState")
.field("config", &self.config)
.finish_non_exhaustive()
}
}
fn offer_trace(traces: &store::TraceSender, spill: Option<&store::SpillHandle>, trace: Trace) {
record_trace_metrics(&trace);
match traces.try_send(trace) {
Ok(()) => {}
Err(TrySendError::Full(t)) => {
if let Some(handle) = spill {
match store::append_to_spill(handle, &t) {
Ok(()) => {
metrics::counter!("firstpass_receipts_spilled_total").increment(1);
}
Err(e) => {
tracing::error!(%e, "durable mode: spill write failed; trace lost");
metrics::counter!("firstpass_traces_dropped_total").increment(1);
}
}
} else {
tracing::warn!("trace channel full; dropping trace (writer behind under load)");
metrics::counter!("firstpass_traces_dropped_total").increment(1);
}
}
Err(TrySendError::Closed(_)) => {
tracing::warn!("trace writer is gone; dropping trace");
}
}
}
fn risk_certificate(
eprocess: Option<(firstpass_core::eprocess::Certification, f64, f64)>,
aci: Option<(f64, f64)>,
fixed: Option<f64>,
) -> Option<RiskCertificate> {
let cert = |source, threshold, alpha| RiskCertificate {
source,
threshold,
alpha,
delta: None,
e_value: None,
certified_at_round: None,
};
if let Some((c, alpha, delta)) = eprocess {
return Some(RiskCertificate {
delta: Some(delta),
e_value: Some(c.e_value),
certified_at_round: Some(c.certified_at_round),
..cert(ThresholdSource::Eprocess, c.threshold, Some(alpha))
});
}
aci.map(|(t, alpha)| cert(ThresholdSource::Aci, t, Some(alpha)))
.or_else(|| fixed.map(|t| cert(ThresholdSource::Fixed, t, None)))
}
fn record_trace_metrics(trace: &Trace) {
if trace.mode == Mode::Enforce {
metrics::histogram!("firstpass_enforce_latency_ms")
.record(trace.final_.total_latency_ms as f64);
if trace.final_.escalations > 0 {
metrics::counter!("firstpass_escalations_total")
.increment(u64::from(trace.final_.escalations));
}
}
let served_from = match trace.final_.served_from {
ServedFrom::Attempt => "attempt",
ServedFrom::BestAttempt => "best_attempt",
ServedFrom::Error => "error",
ServedFrom::Cache => "cache",
};
metrics::counter!("firstpass_served_total", "served_from" => served_from).increment(1);
if trace.final_.served_from == ServedFrom::Error {
let provider = trace
.attempts
.last()
.map(|a| a.provider.clone())
.or_else(|| {
if trace.upstream_provider.is_empty() {
None
} else {
Some(trace.upstream_provider.clone())
}
})
.unwrap_or_else(|| "unknown".to_owned());
metrics::counter!("firstpass_upstream_failures_total", "provider" => provider).increment(1);
}
for a in &trace.attempts {
metrics::histogram!(
"firstpass_attempt_latency_ms",
"provider" => a.provider.clone(),
"rung" => a.rung.to_string()
)
.record(a.latency_ms as f64);
metrics::counter!(
"firstpass_attempt_total",
"provider" => a.provider.clone(),
"rung" => a.rung.to_string()
)
.increment(1);
metrics::gauge!(
"firstpass_attempt_cost_usd_total",
"provider" => a.provider.clone(),
"rung" => a.rung.to_string()
)
.increment(a.cost_usd);
for g in &a.gates {
let verdict = match g.verdict {
Verdict::Pass => "pass",
Verdict::Fail => "fail",
Verdict::Abstain => "abstain",
};
metrics::counter!(
"firstpass_gate_verdict_total",
"gate_id" => g.gate_id.clone(),
"verdict" => verdict
)
.increment(1);
metrics::histogram!("firstpass_gate_latency_ms", "gate_id" => g.gate_id.clone())
.record(g.ms as f64);
metrics::gauge!("firstpass_gate_cost_by_id_usd_total", "gate_id" => g.gate_id.clone())
.increment(g.cost_usd);
}
}
metrics::gauge!("firstpass_cost_usd_total").increment(trace.final_.total_cost_usd);
metrics::gauge!("firstpass_gate_cost_usd_total").increment(trace.final_.gate_cost_usd);
metrics::gauge!("firstpass_baseline_usd_total")
.increment(trace.final_.counterfactual_baseline_usd);
metrics::gauge!("firstpass_savings_usd_total").increment(trace.final_.savings_usd);
if let Some(rung) = trace.final_.served_rung {
let model = trace
.attempts
.iter()
.find(|a| a.rung == rung)
.map(|a| a.model.clone())
.unwrap_or_else(|| "unknown".to_owned());
metrics::counter!(
"firstpass_served_rung_total",
"rung" => rung.to_string(),
"model" => model
)
.increment(1);
}
}
const MAX_BODY_BYTES: usize = 16 * 1024 * 1024;
pub(crate) fn u01(seed: u128) -> f64 {
let lo = splitmix64_finalise(seed as u64);
let hi = splitmix64_finalise((seed >> 64) as u64);
((lo ^ hi) >> 11) as f64 * (1.0_f64 / (1u64 << 53) as f64)
}
fn splitmix64_finalise(mut z: u64) -> u64 {
z = z.wrapping_add(0x9E37_79B9_7F4A_7C15);
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
z ^ (z >> 31)
}
#[must_use]
pub(crate) fn epsilon_propensity(chosen: u32, greedy: u32, epsilon: f64, k: usize) -> f64 {
let greedy_term = if chosen == greedy { 1.0 - epsilon } else { 0.0 };
greedy_term + epsilon / k as f64
}
pub fn app(state: AppState) -> Result<Router, ProxyError> {
crate::metrics::install()?;
let max_concurrency = state.config.max_concurrency;
let business = Router::new()
.route("/v1/messages", post(messages))
.route("/v1/messages/count_tokens", post(count_tokens_passthrough))
.route("/v1/chat/completions", post(chat_completions))
.route("/v1/responses", post(responses))
.route(
"/v1beta/models/{model_action}",
post(gemini_generate_content),
)
.route("/v1/feedback", post(feedback))
.route("/v1/capabilities", get(capabilities))
.route("/v1/models", get(models))
.route("/v1/receipts", get(receipts))
.route("/v1/experiments", get(experiments))
.layer(axum::middleware::from_fn_with_state(
state.clone(),
tenant_rate_limit_middleware,
))
.layer(axum::middleware::from_fn_with_state(
state.clone(),
auth_middleware,
));
Ok(Router::new()
.merge(business)
.route("/healthz", get(healthz))
.route("/metrics", get(crate::metrics::handler))
.route("/panel", get(panel))
.layer(axum::extract::DefaultBodyLimit::max(MAX_BODY_BYTES))
.layer(tower::limit::GlobalConcurrencyLimitLayer::new(
max_concurrency,
))
.with_state(state))
}
async fn healthz() -> impl IntoResponse {
Json(serde_json::json!({
"status": "ok",
"service": "firstpass",
"version": env!("CARGO_PKG_VERSION"),
}))
}
const PANEL_HTML: &str = include_str!("panel.html");
async fn receipts(
State(state): State<AppState>,
Extension(TenantId(tenant)): Extension<TenantId>,
axum::extract::Query(q): axum::extract::Query<ReceiptsQuery>,
) -> Response {
let limit = q.limit.unwrap_or(20).clamp(1, 200);
let db = state.config.db_path.clone();
let (traces, summary) = match tokio::task::spawn_blocking(move || {
Ok::<_, crate::store::StoreError>((
crate::store::load_recent_tenant_traces(&db, &tenant, limit)?,
crate::store::tenant_receipts_summary(&db, &tenant)?,
))
})
.await
{
Ok(Ok(t)) => t,
_ => (Vec::new(), crate::store::ReceiptsSummary::default()),
};
let summary = summary_json(&summary);
let items: Vec<serde_json::Value> = traces
.iter()
.map(|t| {
let f = &t.final_;
serde_json::json!({
"trace_id": t.trace_id,
"ts": t.ts,
"session_id": t.session_id,
"served_rung": f.served_rung,
"served_from": f.served_from,
"served_model": f.served_rung
.and_then(|r| t.attempts.iter().find(|a| a.rung == r))
.map(|a| a.model.clone()),
"attempts": t.attempts.iter().map(|a| serde_json::json!({
"rung": a.rung,
"model": a.model,
"verdict": a.verdict,
"cost_usd": a.cost_usd,
"latency_ms": a.latency_ms,
"gates": a.gates.iter().map(|g| serde_json::json!({
"gate_id": g.gate_id,
"verdict": g.verdict,
"score": g.score,
"reason": g.reason,
})).collect::<Vec<_>>(),
})).collect::<Vec<_>>(),
"escalations": f.escalations,
"total_latency_ms": f.total_latency_ms,
"total_cost_usd": f.total_cost_usd,
"baseline_usd": f.counterfactual_baseline_usd,
"savings_usd": f.savings_usd,
"prev_hash": t.prev_hash,
"hash": t.hash().ok(),
"experiment": t.experiment.as_ref().map(|e| serde_json::json!({
"arm": e.arm,
"percent": e.percent,
"bucket": e.bucket,
})),
"deferred": t.deferred.iter().map(|d| serde_json::json!({
"gate_id": d.gate_id,
"verdict": d.verdict,
"reporter": d.reporter,
})).collect::<Vec<_>>(),
})
})
.collect();
axum::Json(serde_json::json!({ "receipts": items, "summary": summary })).into_response()
}
fn summary_json(s: &crate::store::ReceiptsSummary) -> serde_json::Value {
let [pass, fail, abstain] = s.gates;
serde_json::json!({
"requests": s.requests,
"spent_usd": s.spent_usd,
"baseline_usd": s.baseline_usd,
"saved_usd": s.baseline_usd - s.spent_usd,
"first_pass": s.first_pass,
"escalated": s.escalated,
"gates": { "pass": pass, "fail": fail, "abstain": abstain },
"truncated": s.truncated,
})
}
async fn experiments(
State(state): State<AppState>,
Extension(TenantId(tenant)): Extension<TenantId>,
) -> Response {
let Some(routing) = state.config.routing.as_ref() else {
return axum::Json(serde_json::json!({ "experiments": [], "configured": [] }))
.into_response();
};
let snaps: Vec<_> = state
.experiments
.reports()
.into_iter()
.filter(|s| s.tenant == tenant)
.collect();
let views = crate::experiment::views(snaps, routing);
let configured: Vec<serde_json::Value> = routing
.routes
.iter()
.enumerate()
.filter_map(|(ix, r)| r.challenger.as_ref().map(|c| (ix, r, c)))
.map(|(ix, r, c)| {
let status = state.experiments.status(&tenant, ix);
let paired = c.is_paired().then(|| {
serde_json::json!({
"sample_rate": c.sample_rate,
"max_usd_per_day": c.max_usd_per_day,
"spent_today_usd": state.paired_ledger.spent_today(
&tenant,
ix,
jiff::Timestamp::now(),
),
})
});
serde_json::json!({
"route_ix": ix,
"incumbent_ladder": r.ladder,
"challenger_ladder": c.ladder,
"mode": if c.is_paired() { "paired" } else { "live" },
"paired": paired,
"percent": c.percent,
"margin": c.margin,
"alpha": c.alpha,
"unavailable": state.experiments.challenger_blocked(&tenant, ix)
&& status != firstpass_core::ExperimentStatus::Halted,
})
})
.collect();
axum::Json(serde_json::json!({ "experiments": views, "configured": configured }))
.into_response()
}
#[derive(Debug, serde::Deserialize)]
struct ReceiptsQuery {
limit: Option<usize>,
}
async fn panel() -> impl IntoResponse {
(
[
(
axum::http::header::CONTENT_TYPE,
"text/html; charset=utf-8".to_owned(),
),
(
axum::http::header::CONTENT_SECURITY_POLICY,
PANEL_CSP.clone(),
),
(
axum::http::header::X_CONTENT_TYPE_OPTIONS,
"nosniff".to_owned(),
),
(
axum::http::header::REFERRER_POLICY,
"no-referrer".to_owned(),
),
],
PANEL_HTML,
)
}
static PANEL_CSP: std::sync::LazyLock<String> = std::sync::LazyLock::new(|| {
use sha2::Digest as _;
let script = PANEL_HTML
.split_once("<script>")
.and_then(|(_, rest)| rest.split_once("</script>"))
.map_or("", |(body, _)| body);
let digest = sha2::Sha256::digest(script.as_bytes());
format!(
"default-src 'none'; script-src 'sha256-{}'; style-src 'unsafe-inline'; \
connect-src 'self'; img-src 'self' data:; base-uri 'none'; form-action 'none'; \
frame-ancestors 'none'",
base64_std(&digest)
)
});
fn base64_std(bytes: &[u8]) -> String {
const ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
let mut out = String::with_capacity(bytes.len().div_ceil(3) * 4);
for chunk in bytes.chunks(3) {
let b = [
chunk[0],
chunk.get(1).copied().unwrap_or(0),
chunk.get(2).copied().unwrap_or(0),
];
let n = (u32::from(b[0]) << 16) | (u32::from(b[1]) << 8) | u32::from(b[2]);
for (i, shift) in [18u32, 12, 6, 0].into_iter().enumerate() {
if i <= chunk.len() {
out.push(char::from(ALPHABET[((n >> shift) & 63) as usize]));
} else {
out.push('=');
}
}
}
out
}
async fn capabilities(State(state): State<AppState>) -> impl IntoResponse {
let (ladder, gates) = state
.config
.routing
.as_ref()
.and_then(|c| c.routes.iter().find(|r| r.mode == Mode::Enforce))
.map(|r| (r.ladder.clone(), r.gates.clone()))
.unwrap_or_default();
let routing_modes: Vec<serde_json::Value> = RoutingMode::ALL
.iter()
.map(|m| {
let p = m.preset();
serde_json::json!({
"name": m.as_str(),
"description": p.description,
"tradeoff": p.tradeoff,
})
})
.collect();
Json(serde_json::json!({
"service": "firstpass",
"version": env!("CARGO_PKG_VERSION"),
"feature_version": FEATURE_VERSION,
"modes": ["observe", "enforce"],
"routing_modes": routing_modes,
"wire_apis": ["anthropic.messages", "openai.chat_completions", "openai.responses", "gemini.generate_content"],
"ladder": ladder,
"gates": gates,
"feedback_api": "POST /v1/feedback",
"offboarding": "unset ANTHROPIC_BASE_URL (or OPENAI_BASE_URL for OpenAI clients, GOOGLE_GEMINI_BASE_URL for Gemini CLI)",
}))
}
async fn models(State(state): State<AppState>) -> impl IntoResponse {
Json(model_list(state.config.routing.as_ref()))
}
fn model_list(routing: Option<&firstpass_core::config::Config>) -> serde_json::Value {
let prices = routing.map(|c| c.price_table()).unwrap_or_default();
let mut seen = std::collections::HashSet::new();
let data: Vec<serde_json::Value> = routing
.map(|c| c.routes.as_slice())
.unwrap_or_default()
.iter()
.flat_map(|r| r.ladder.iter())
.filter(|id| seen.insert(id.as_str()))
.map(|id| {
let price = prices.get(id);
serde_json::json!({
"id": id,
"object": "model",
"created": 0,
"owned_by": id.split_once('/').map_or("firstpass", |(p, _)| p),
"firstpass": {
"input_per_mtok": price.map(|p| p.input_per_mtok),
"output_per_mtok": price.map(|p| p.output_per_mtok),
},
})
})
.collect();
serde_json::json!({ "object": "list", "data": data })
}
#[derive(Debug, Deserialize)]
struct FeedbackRequest {
trace_id: String,
gate_id: String,
verdict: String,
#[serde(default)]
score: Option<f64>,
reporter: String,
}
async fn feedback(
State(state): State<AppState>,
Extension(TenantId(tenant)): Extension<TenantId>,
body: Bytes,
) -> Response {
let req: FeedbackRequest = match serde_json::from_slice(&body) {
Ok(r) => r,
Err(e) => {
return ProxyError::BadRequest(format!("invalid feedback body: {e}")).into_response();
}
};
let verdict = match req.verdict.as_str() {
"pass" => Verdict::Pass,
"fail" => Verdict::Fail,
"abstain" => Verdict::Abstain,
other => {
return ProxyError::BadRequest(format!("unknown verdict {other:?}")).into_response();
}
};
let score = match req.score {
Some(s) => match Score::new(s) {
Ok(sc) => Some(sc),
Err(_) => {
return ProxyError::BadRequest(format!("score {s} out of range [0,1]"))
.into_response();
}
},
None => None,
};
let db = state.config.db_path.clone();
let (db_check, tenant_check, tid_check) = (db.clone(), tenant.clone(), req.trace_id.clone());
match tokio::task::spawn_blocking(move || {
store::trace_exists(&db_check, &tenant_check, &tid_check)
})
.await
{
Ok(Ok(true)) => {}
Ok(Ok(false)) => {
return ProxyError::NotFound(format!("unknown trace_id {:?}", req.trace_id))
.into_response();
}
Ok(Err(e)) => {
tracing::error!(%e, "feedback: trace_exists check failed");
return ProxyError::Internal(e.to_string()).into_response();
}
Err(e) => {
tracing::error!(%e, "feedback: trace_exists task panicked");
return ProxyError::Internal(e.to_string()).into_response();
}
}
if verdict == Verdict::Fail
&& let Some(cache) = state.verified_cache.as_ref()
{
let reported = req.trace_id.parse::<uuid::Uuid>().ok();
let origin = {
let (d, t, i) = (db.clone(), tenant.clone(), req.trace_id.clone());
tokio::task::spawn_blocking(move || store::trace_cache_source(&d, &t, &i))
.await
.ok()
.and_then(Result::ok)
.flatten()
.flatten()
};
let mut dropped = 0usize;
for id in reported.into_iter().chain(origin) {
dropped += cache.retract(id).await;
}
if dropped > 0 {
tracing::info!(
trace = %req.trace_id,
?origin,
dropped,
"verified cache: retracted a disproven answer"
);
metrics::counter!("firstpass_cache_retractions_total").increment(dropped as u64);
}
}
let feedback_signal = match verdict {
Verdict::Pass => Some(true),
Verdict::Fail => Some(false),
Verdict::Abstain => None,
};
let dv = DeferredVerdict {
gate_id: req.gate_id,
verdict,
score,
reported_at: jiff::Timestamp::now(),
reporter: req.reporter,
};
let trace_id = req.trace_id.clone();
match tokio::task::spawn_blocking(move || store::append_deferred(&db, &req.trace_id, &dv)).await
{
Ok(Ok(())) => {
if let (Some(a), Some(correct)) = (state.adaptive.as_ref(), feedback_signal)
&& let Ok(mut g) = a.lock()
{
g.observe_served(correct);
metrics::gauge!("firstpass_serve_threshold").set(g.threshold());
metrics::gauge!("firstpass_realized_served_failure")
.set(g.realized_served_failure());
}
if let (Some(e), Some(correct)) = (state.eprocess.as_ref(), feedback_signal) {
let db_s = state.config.db_path.clone();
let (tenant_s, tid_s) = (tenant.clone(), trace_id.clone());
let served = tokio::task::spawn_blocking(move || {
store::load_trace_view(&db_s, &tenant_s, &tid_s)
.ok()
.flatten()
.and_then(|t| {
let rung = t.final_.served_rung?;
let a = t.attempts.iter().find(|a| a.rung == rung)?;
Some(crate::calibrate::gate_score(&a.gates, a.verdict))
})
})
.await
.ok()
.flatten();
if let (Some(sc), Ok(mut g)) = (served, e.lock()) {
g.observe_served(sc, correct);
metrics::counter!("firstpass_eprocess_rounds_total").increment(1);
}
}
let attributed_route = tokio::task::spawn_blocking({
let (db, ten, tid) = (
state.config.db_path.clone(),
tenant.clone(),
trace_id.clone(),
);
move || store::trace_route(&db, &ten, &tid)
})
.await
.ok()
.and_then(Result::ok)
.flatten()
.flatten()
.unwrap_or(0) as usize;
if let (Some(cfg), Some(correct)) = (
state.config.routing.as_ref().and_then(|r| r.guardrail),
feedback_signal,
) {
let cooldown = state
.config
.routing
.as_ref()
.map_or(3_600, |r| r.guardrail_cooldown_secs);
let reaction = state.guardrails.record(
&tenant,
attributed_route,
&cfg,
correct,
jiff::Timestamp::now().as_second(),
cooldown,
);
match &reaction {
crate::guard::Reaction::Demoted(v) => {
metrics::counter!("firstpass_guardrail_demotions_total").increment(1);
tracing::error!(
tenant = %tenant,
n = v.n,
rate = v.rate,
bound = v.bound,
alpha = cfg.alpha,
"GUARDRAIL: served-failure bound exceeded target — route demoted to observe; traffic now serves as it would without Firstpass"
);
}
crate::guard::Reaction::Alarmed(v) => {
metrics::counter!("firstpass_guardrail_breaches_total").increment(1);
tracing::error!(
tenant = %tenant,
n = v.n,
rate = v.rate,
bound = v.bound,
alpha = cfg.alpha,
"GUARDRAIL: served-failure bound exceeded target (alarm only — routing unchanged)"
);
}
crate::guard::Reaction::None => {}
}
}
if state.experiments.is_active() && verdict != Verdict::Abstain {
let (db_x, ten_x, tid_x) = (
state.config.db_path.clone(),
tenant.clone(),
trace_id.clone(),
);
match tokio::task::spawn_blocking(move || {
store::trace_experiment(&db_x, &ten_x, &tid_x)
})
.await
{
Ok(Ok(Some((session, route_ix, rec, cost_usd)))) => {
state
.experiments
.record_outcome(
&tenant,
route_ix,
&rec.fingerprint,
rec.arm,
&session,
verdict,
cost_usd,
)
.await;
}
Ok(Ok(None)) => {}
Ok(Err(e)) => {
tracing::warn!(%e, "feedback: experiment attribution lookup failed");
}
Err(e) => {
tracing::warn!(%e, "feedback: experiment attribution task failed");
}
}
}
(
axum::http::StatusCode::ACCEPTED,
Json(serde_json::json!({ "status": "recorded", "trace_id": trace_id })),
)
.into_response()
}
Ok(Err(e)) => {
tracing::error!(%e, "feedback: append_deferred failed");
ProxyError::Internal(e.to_string()).into_response()
}
Err(e) => {
tracing::error!(%e, "feedback: append_deferred task panicked");
ProxyError::Internal(e.to_string()).into_response()
}
}
}
const SESSION_HEADER: &str = "x-firstpass-session";
const AGENT_HEADER: &str = "x-firstpass-agent";
const SUBAGENT_HEADER: &str = "x-firstpass-subagent";
#[expect(
clippy::too_many_arguments,
reason = "mirrors the enforce context it evaluates; a wrapper struct would only move the list"
)]
fn spawn_shadow(
state: &AppState,
route: &firstpass_core::Route,
route_ix: usize,
body: Bytes,
auth: Auth,
features: Features,
tenant: String,
session_id: String,
api: &str,
) {
let Some(shadow) = route.shadow else {
return;
};
if !shadow.sampled(&state.config.prompt_salt, &session_id) {
return;
}
let (state, route, api) = (state.clone(), route.clone(), api.to_owned());
tokio::spawn(async move {
let signal = evaluate_shadow(
&state, &route, shadow, route_ix, &body, auth, &features, tenant, session_id, &api, 0.0,
)
.await;
tracing::debug!(
would_pass = signal.would_pass,
projected_usd = signal.projected_cost_usd,
skipped = ?signal.skipped,
"shadow evaluation complete"
);
});
}
#[expect(
clippy::too_many_arguments,
reason = "mirrors the enforce context; grouping them into a struct would only move the list"
)]
async fn evaluate_shadow(
state: &AppState,
route: &firstpass_core::Route,
shadow: firstpass_core::rollout::Shadow,
route_ix: usize,
body: &Bytes,
auth: Auth,
features: &Features,
tenant: String,
session_id: String,
api: &str,
actual_cost_usd: f64,
) -> ShadowSignal {
let skipped = |why: &str| ShadowSignal {
would_serve_rung: None,
would_pass: false,
projected_cost_usd: 0.0,
actual_cost_usd,
skipped: Some(why.to_owned()),
};
let now = jiff::Timestamp::now();
if !state
.shadow_ledger
.may_spend(&tenant, route_ix, shadow.max_usd_per_day, now)
{
return skipped("budget_exhausted");
}
let Some(base_request) = parse_model_request(body) else {
return skipped("unparseable_request");
};
let gate_defs = state
.config
.routing
.as_ref()
.map_or(&[][..], |cfg| &cfg.gate_defs);
let gates = resolve_gates(
&route.gates,
gate_defs,
&state.providers,
&auth,
&state.config.prices,
&state.http,
);
let (budget, max_rungs) = state
.config
.routing
.as_ref()
.map_or((None, u32::MAX), |cfg| {
(
cfg.budget.per_request_usd,
cfg.escalation.max_rungs_per_request,
)
});
let mut base_request = base_request;
base_request.cache_prefix = state
.config
.routing
.as_ref()
.is_some_and(|r| r.escalation.prompt_cache);
let ctx = EnforceCtx {
condense: routing_cfg_condense(state),
ladder: &route.ladder,
gates: &gates,
health: &state.gate_health,
base_request: &base_request,
providers: &state.providers,
auth: &auth,
prices: &state.config.prices,
budget_per_request_usd: budget,
max_rungs,
speculation: 0,
serve_threshold: None,
elastic: None,
features: features.clone(),
start_rung: 0,
tenant_id: tenant.clone(),
session_id,
prompt_hash: prompt_hash(&state.config.prompt_salt, body),
api: api.to_owned(),
policy_id: "shadow".to_owned(),
};
let (outcome, trace) = route_enforce(ctx).await;
let spent = trace.final_.total_cost_usd;
state.shadow_ledger.debit(&tenant, route_ix, spent, now);
let would_pass = matches!(outcome, EngineOutcome::Served(_))
&& trace.final_.served_from == firstpass_core::ServedFrom::Attempt;
let would_serve_rung = if would_pass {
trace.final_.served_rung
} else {
None
};
ShadowSignal {
would_serve_rung,
would_pass,
projected_cost_usd: spent,
actual_cost_usd,
skipped: None,
}
}
fn pair_outcome(served_from: firstpass_core::ServedFrom) -> Option<f64> {
match served_from {
firstpass_core::ServedFrom::Attempt => Some(1.0),
firstpass_core::ServedFrom::BestAttempt => Some(0.0),
firstpass_core::ServedFrom::Cache | firstpass_core::ServedFrom::Error => None,
}
}
#[allow(clippy::too_many_arguments)]
fn spawn_paired_challenger(
state: &AppState,
route: &firstpass_core::Route,
challenger: &firstpass_core::Challenger,
fingerprint: String,
route_ix: usize,
body: bytes::Bytes,
auth: Auth,
features: firstpass_core::Features,
tenant: String,
session_id: String,
api: &str,
served_from: firstpass_core::ServedFrom,
incumbent_cost_usd: f64,
incumbent_trace_id: uuid::Uuid,
) {
let Some(x_inc) = pair_outcome(served_from) else {
return;
};
if state.experiments.challenger_blocked(&tenant, route_ix) {
return;
}
let max_usd_per_day = match challenger.max_usd_per_day {
Some(v) => v,
None => {
tracing::warn!(
tenant,
route = route_ix,
"paired challenger is missing max_usd_per_day; skipping evaluation"
);
return;
}
};
let state = state.clone();
let route = route.clone();
let challenger = challenger.clone();
let api = api.to_owned();
tokio::spawn(async move {
if !state
.experiments
.ensure_synced(&tenant, route_ix, &fingerprint)
.await
{
tracing::debug!(
tenant,
route = route_ix,
"paired challenger: ensure_synced failed (log error), skipping evaluation"
);
return;
}
if state.experiments.challenger_blocked(&tenant, route_ix) {
return;
}
let now = jiff::Timestamp::now();
if !state
.paired_ledger
.may_spend(&tenant, route_ix, max_usd_per_day, now)
{
tracing::debug!(
tenant,
route = route_ix,
"paired challenger: daily budget exhausted, skipping"
);
return;
}
let Some(base_request) = parse_model_request(&body) else {
return;
};
let gate_defs = state
.config
.routing
.as_ref()
.map_or(&[][..], |cfg| &cfg.gate_defs);
let gates = resolve_gates(
&route.gates,
gate_defs,
&state.providers,
&auth,
&state.config.prices,
&state.http,
);
let (budget, max_rungs) = state
.config
.routing
.as_ref()
.map_or((None, u32::MAX), |cfg| {
(
cfg.budget.per_request_usd,
cfg.escalation.max_rungs_per_request,
)
});
let mut base_request = base_request;
base_request.cache_prefix = state
.config
.routing
.as_ref()
.is_some_and(|r| r.escalation.prompt_cache);
let ctx = EnforceCtx {
condense: routing_cfg_condense(&state),
ladder: &challenger.ladder,
gates: &gates,
health: &state.gate_health,
base_request: &base_request,
providers: &state.providers,
auth: &auth,
prices: &state.config.prices,
budget_per_request_usd: budget,
max_rungs,
speculation: 0,
serve_threshold: None,
elastic: None,
features,
start_rung: 0,
tenant_id: tenant.clone(),
session_id: session_id.clone(),
prompt_hash: prompt_hash(&state.config.prompt_salt, &body),
api: api.clone(),
policy_id: "paired-challenger@v0".to_owned(),
};
let (_, ch_trace) = route_enforce(ctx).await;
let pair_ts = jiff::Timestamp::now();
let challenger_cost = ch_trace.final_.total_cost_usd;
let ch_errored = ch_trace.final_.served_from == firstpass_core::ServedFrom::Error;
let ch_trunc = crate::experiment::truncated_attempts(&ch_trace);
state
.paired_ledger
.debit(&tenant, route_ix, challenger_cost, pair_ts);
state.experiments.observe_health(
&tenant,
route_ix,
&fingerprint,
firstpass_core::Arm::Challenger,
ch_errored,
ch_trunc,
);
let x_ch = match ch_trace.final_.served_from {
firstpass_core::ServedFrom::Attempt => 1.0,
firstpass_core::ServedFrom::BestAttempt | firstpass_core::ServedFrom::Error => 0.0,
firstpass_core::ServedFrom::Cache => return,
};
let appended = state
.experiments
.record_pair(
&tenant,
route_ix,
&fingerprint,
x_inc,
x_ch,
incumbent_cost_usd,
challenger_cost,
ch_errored,
ch_trunc,
pair_ts.as_second(),
ch_trace.trace_id,
incumbent_trace_id,
)
.await;
let pair_row = crate::store::ExperimentPairRow {
tenant: tenant.clone(),
route_ix,
fingerprint: fingerprint.clone(),
trace_id: ch_trace.trace_id,
x_inc,
x_ch,
challenger_cost_usd: challenger_cost,
ch_error: ch_errored,
ch_truncated: ch_trunc,
ts: pair_ts,
in_log: if appended {
crate::store::InLogStatus::InLog
} else {
crate::store::InLogStatus::NotInLog
},
};
if appended {
if let Err(e) = crate::store::append_experiment_pair(&state.config.db_path, &pair_row) {
tracing::warn!(%e, "paired challenger: failed to persist InLog row to store");
}
} else {
match crate::store::append_experiment_pair(&state.config.db_path, &pair_row) {
Ok(()) => {
state.experiments.apply_pair_locally(
&tenant,
route_ix,
&fingerprint,
x_inc,
x_ch,
challenger_cost,
ch_trace.trace_id,
);
}
Err(e) => {
tracing::warn!(
%e,
"paired challenger: SQLite write failed; evidence withheld for this pair"
);
}
}
}
tracing::debug!(
tenant,
route = route_ix,
x_inc,
x_ch,
challenger_cost_usd = challenger_cost,
"paired challenger evaluation complete"
);
});
}
const MODE_PROFILE_HEADER: &str = "x-firstpass-mode";
fn resolve_mode(headers: &HeaderMap, route: &Route, config: &ProxyConfig) -> RoutingMode {
if let Some(val) = header_str(headers, MODE_PROFILE_HEADER) {
match val.trim().to_ascii_lowercase().as_str() {
"observe" => return RoutingMode::Observe,
"cost" => return RoutingMode::Cost,
"balanced" => return RoutingMode::Balanced,
"quality" => return RoutingMode::Quality,
"latency" => return RoutingMode::Latency,
"max" => return RoutingMode::Max,
other => {
tracing::warn!(
value = other,
"unknown x-firstpass-mode value; ignoring \
(valid: observe|cost|balanced|quality|latency|max)"
);
}
}
}
if let Some(m) = route.routing_mode {
return m;
}
config.default_routing_mode
}
async fn count_tokens_passthrough(
State(state): State<AppState>,
Extension(TenantId(_tenant)): Extension<TenantId>,
headers: HeaderMap,
body: Bytes,
) -> Response {
match crate::upstream::forward_anthropic_path(
&state.http,
&state.config.upstream_anthropic,
"/v1/messages/count_tokens",
&headers,
body,
)
.await
{
Ok((status, resp_headers, resp_body)) => (status, resp_headers, resp_body).into_response(),
Err(err) => err.into_response(),
}
}
async fn messages(
State(state): State<AppState>,
Extension(TenantId(tenant)): Extension<TenantId>,
headers: HeaderMap,
body: Bytes,
) -> Response {
let mut session_header = header_session(&headers, &state.config.prompt_salt);
if let Some(routing) = state.config.routing.as_ref() {
let features = extract_features(&headers, &body);
if let Some(route) = routing
.route_for(&features)
.filter(|r| r.mode == Mode::Enforce && !r.ladder.is_empty())
{
session_header = session_header
.take()
.or_else(|| body_session_bytes(&body, &state.config.prompt_salt));
let route_ix = routing
.routes
.iter()
.position(|r| std::ptr::eq(r, route))
.unwrap_or(0);
let route = route.clone();
let routing_mode = resolve_mode(&headers, &route, &state.config);
if routing_mode == RoutingMode::Observe {
return observe_passthrough(state, headers, body, session_header, tenant).await;
}
if state
.guardrails
.is_demoted(&tenant, route_ix, jiff::Timestamp::now().as_second())
{
return observe_passthrough(state, headers, body, session_header, tenant).await;
}
if let Some(rollout) = route.rollout.as_ref() {
let key_value = match rollout.key {
firstpass_core::RolloutKey::Session => session_header
.clone()
.unwrap_or_else(|| firstpass_core::rollout::request_identity(&body)),
firstpass_core::RolloutKey::Request => {
firstpass_core::rollout::request_identity(&body)
}
firstpass_core::RolloutKey::Tenant => tenant.to_string(),
};
let decision =
firstpass_core::rollout::decide(&state.config.prompt_salt, rollout, &key_value);
if !decision.enforced {
let shadow_ctx = route.shadow.map(|_| {
(
body.clone(),
Auth::from_headers(&headers, firstpass_core::Dialect::Anthropic),
features.clone(),
tenant.clone(),
session_header
.clone()
.unwrap_or_else(|| Uuid::now_v7().to_string()),
)
});
let resp =
observe_passthrough(state.clone(), headers, body, session_header, tenant)
.await;
if let Some((b, auth, feats, ten, sess)) = shadow_ctx {
spawn_shadow(
&state,
&route,
route_ix,
b,
auth,
feats,
ten,
sess,
"anthropic",
);
}
return resp;
}
}
if enforce_can_handle_route(
&features,
&body,
routing.escalation.enforce_structured,
&route,
&state.providers,
Dialect::Anthropic,
) {
return handle_enforce(
&state,
&headers,
&body,
features,
&route,
route_ix,
session_header,
tenant,
routing_mode,
)
.await;
}
tracing::info!(
"enforce route matched but structured request can't be routed faithfully (flag/ladder); serving via observe passthrough"
);
}
}
observe_passthrough(state, headers, body, session_header, tenant).await
}
fn enforce_can_handle_route(
features: &Features,
body: &[u8],
enforce_structured: bool,
route: &Route,
providers: &crate::provider::ProviderRegistry,
inbound: Dialect,
) -> bool {
enforce_can_handle(
features,
body,
enforce_structured,
&route.ladder,
providers,
inbound,
) && route.challenger.as_ref().is_none_or(|c| {
enforce_can_handle(
features,
body,
enforce_structured,
&c.ladder,
providers,
inbound,
)
})
}
fn enforce_can_handle(
features: &Features,
body: &[u8],
enforce_structured: bool,
ladder: &[String],
providers: &crate::provider::ProviderRegistry,
inbound: Dialect,
) -> bool {
let structured = features.tool_count > 0
|| features.has_images
|| match inbound {
Dialect::Anthropic => messages_have_tool_blocks(body),
Dialect::Openai => openai_messages_have_tool_calls(body),
Dialect::Gemini => false,
};
if !structured {
return true;
}
if !enforce_structured {
return false;
}
let all_verbatim = ladder.iter().all(|rung| {
let provider_id = rung.split('/').next().unwrap_or_default();
providers
.get(provider_id)
.is_some_and(|p| p.carries_structured_verbatim(inbound))
});
if all_verbatim {
return true;
}
if inbound == Dialect::Openai && !openai_has_http_images(body) {
let all_anthropic = ladder.iter().all(|rung| {
let pid = rung.split('/').next().unwrap_or_default();
providers
.get(pid)
.is_some_and(|p| p.carries_structured_verbatim(Dialect::Anthropic))
});
if all_anthropic {
return true;
}
}
false
}
fn messages_have_tool_blocks(body: &[u8]) -> bool {
serde_json::from_slice::<Value>(body)
.ok()
.and_then(|json| {
json.get("messages")
.and_then(Value::as_array)
.map(|messages| messages.iter().any(message_has_tool_block))
})
.unwrap_or(false)
}
fn message_has_tool_block(message: &Value) -> bool {
message
.get("content")
.and_then(Value::as_array)
.is_some_and(|blocks| {
blocks.iter().any(|block| {
matches!(
block.get("type").and_then(Value::as_str),
Some("tool_use" | "tool_result")
)
})
})
}
fn is_stream_request(body: &[u8]) -> bool {
serde_json::from_slice::<Value>(body)
.ok()
.and_then(|json| json.get("stream").and_then(Value::as_bool))
.unwrap_or(false)
}
fn header_str(headers: &HeaderMap, name: &str) -> Option<String> {
headers
.get(name)
.and_then(|v| v.to_str().ok())
.map(str::to_owned)
}
const AGENT_SESSION_HEADERS: [&str; 3] =
["x-claude-code-session-id", "session_id", "conversation_id"];
const SESSION_MAX_BYTES: usize = 256;
fn cap_session(s: &str) -> &str {
let mut end = s.len().min(SESSION_MAX_BYTES);
while !s.is_char_boundary(end) {
end -= 1;
}
&s[..end]
}
fn hashed_session(kind: &str, salt: &str, raw: &str) -> Option<String> {
let h = prompt_hash(salt, cap_session(raw).as_bytes());
Some(format!("{kind}:{}", h.get(..16)?))
}
fn header_session(headers: &HeaderMap, salt: &str) -> Option<String> {
if let Some(v) = header_str(headers, SESSION_HEADER) {
let v = v.trim();
if !v.is_empty() {
return Some(cap_session(v).to_owned());
}
}
for name in AGENT_SESSION_HEADERS {
if let Some(v) = headers.get(name).and_then(|v| v.to_str().ok()) {
let v = v.trim();
if !v.is_empty() {
return hashed_session("agent", salt, v);
}
}
}
None
}
fn body_session_bytes(body: &[u8], salt: &str) -> Option<String> {
body_session(&serde_json::from_slice::<Value>(body).ok()?, salt)
}
fn body_session(json: &Value, salt: &str) -> Option<String> {
if let Some(u) = json.pointer("/metadata/user_id").and_then(Value::as_str)
&& u.contains("session_")
{
return hashed_session("meta", salt, u);
}
let msgs = json.get("messages").and_then(Value::as_array);
let input = json.get("input");
let n_items = msgs
.map(Vec::len)
.or_else(|| input.and_then(Value::as_array).map(Vec::len))
.unwrap_or(0);
let has_tools = json
.get("tools")
.and_then(Value::as_array)
.is_some_and(|t| !t.is_empty());
if n_items <= 1 && !has_tools {
return json
.get("previous_response_id")
.and_then(Value::as_str)
.filter(|p| !p.trim().is_empty())
.and_then(|p| hashed_session("prev", salt, p.trim()));
}
if let Some(k) = json.get("prompt_cache_key").and_then(Value::as_str) {
let k = k.trim();
if !k.is_empty() {
return hashed_session("pck", salt, k);
}
}
let mut tools: Vec<&str> = json
.get("tools")
.and_then(Value::as_array)
.into_iter()
.flatten()
.filter_map(|t| {
t.get("name")
.or_else(|| t.pointer("/function/name"))
.or_else(|| t.get("type"))
.and_then(Value::as_str)
})
.collect();
tools.sort_unstable();
let mut prefix = vec![
json.get("system").cloned().unwrap_or(Value::Null),
json.get("instructions").cloned().unwrap_or(Value::Null),
Value::from(tools),
];
if let Some(msgs) = msgs {
for m in msgs {
prefix.push(m.clone());
if m.get("role").and_then(Value::as_str) == Some("user") {
break;
}
}
} else if let Some(input) = input {
let (role, text) = match input {
Value::String(s) => ("user", s.clone()),
other => {
let first = other.as_array().and_then(|a| a.first())?;
let role = first.get("role").and_then(Value::as_str).unwrap_or("user");
let text = match first.get("content") {
Some(Value::String(s)) => s.clone(),
Some(Value::Array(parts)) => parts
.iter()
.filter_map(|p| p.get("text").and_then(Value::as_str))
.collect::<Vec<_>>()
.join("\n"),
_ => first.to_string(),
};
(role, text)
}
};
prefix.push(serde_json::json!([role, text]));
}
let bytes = serde_json::to_vec(&prefix).ok()?;
let h = prompt_hash(salt, &bytes);
Some(format!("conv:{}", h.get(..16)?))
}
fn trajectory_signals(body: &[u8]) -> TrajectorySignals {
let Ok(json) = serde_json::from_slice::<Value>(body) else {
return TrajectorySignals::default();
};
let Some(messages) = json.get("messages").and_then(Value::as_array) else {
return TrajectorySignals::default();
};
const WINDOW: usize = 12;
let recent = &messages[messages.len().saturating_sub(WINDOW)..];
let assistant_turns = u32::try_from(
messages
.iter()
.filter(|m| m.get("role").and_then(Value::as_str) == Some("assistant"))
.count(),
)
.unwrap_or(u32::MAX);
let mut s = TrajectorySignals {
assistant_turns,
..Default::default()
};
let mut seen: Vec<u64> = Vec::new();
for m in recent {
match m.get("role").and_then(Value::as_str) {
Some("assistant") => {}
Some("tool") => {
s.tool_results += 1;
if openai_tool_content_looks_like_error(m) {
s.tool_errors += 1;
}
}
_ => {}
}
if let Some(blocks) = m.get("content").and_then(Value::as_array) {
for b in blocks {
match b.get("type").and_then(Value::as_str) {
Some("tool_result") => {
s.tool_results += 1;
if b.get("is_error").and_then(Value::as_bool).unwrap_or(false) {
s.tool_errors += 1;
}
}
Some("tool_use") => {
let fp = tool_call_fingerprint(
b.get("name").and_then(Value::as_str).unwrap_or_default(),
b.get("input"),
);
if seen.contains(&fp) {
s.repeated_tool_calls += 1;
} else {
seen.push(fp);
}
}
_ => {}
}
}
}
if let Some(calls) = m.get("tool_calls").and_then(Value::as_array) {
for c in calls {
let f = c.get("function");
let fp = tool_call_fingerprint(
f.and_then(|f| f.get("name"))
.and_then(Value::as_str)
.unwrap_or_default(),
f.and_then(|f| f.get("arguments")),
);
if seen.contains(&fp) {
s.repeated_tool_calls += 1;
} else {
seen.push(fp);
}
}
}
}
s
}
fn tool_call_fingerprint(name: &str, args: Option<&Value>) -> u64 {
use std::hash::{Hash, Hasher};
let mut h = std::collections::hash_map::DefaultHasher::new();
name.hash(&mut h);
if let Some(a) = args {
a.to_string().hash(&mut h);
}
h.finish()
}
fn openai_tool_content_looks_like_error(message: &Value) -> bool {
let content = message.get("content");
let text: &str = match content.and_then(Value::as_str) {
Some(t) => t,
None => {
let Some(t) = content.and_then(Value::as_array).and_then(|parts| {
parts
.iter()
.find_map(|p| p.get("text").and_then(Value::as_str))
}) else {
return false;
};
t
}
};
let lowered: String = text
.chars()
.take(128)
.collect::<String>()
.trim_start()
.chars()
.take(64)
.collect::<String>()
.to_ascii_lowercase();
lowered.starts_with("error")
|| lowered.starts_with("exception")
|| lowered.starts_with("traceback")
|| lowered.starts_with("failed")
|| lowered.starts_with("fatal")
}
fn extract_features(headers: &HeaderMap, body: &[u8]) -> Features {
let (_model, tool_count, has_images) = request_features(body);
let mut f = Features::new(TaskKind::Other);
f.agent = header_str(headers, AGENT_HEADER);
f.subagent = header_str(headers, SUBAGENT_HEADER);
f.tool_count = tool_count;
f.has_images = has_images;
f.prompt_token_bucket = token_bucket(body.len() as u64);
f.hour_bucket = hour_bucket(jiff::Timestamp::now());
f.difficulty_hint = DifficultyHint::score(trajectory_signals(body)).as_u8();
f
}
#[allow(clippy::too_many_arguments)]
async fn handle_enforce(
state: &AppState,
headers: &HeaderMap,
body: &Bytes,
features: Features,
route: &Route,
route_ix: usize,
session_header: Option<String>,
tenant: String,
routing_mode: RoutingMode,
) -> Response {
if is_stream_request(body) {
let (tx, rx) = tokio::sync::oneshot::channel::<Result<Value, ProxyError>>();
let (state_c, headers_c, body_c, route_c) =
(state.clone(), headers.clone(), body.clone(), route.clone());
tokio::spawn(async move {
let out = enforce_pipeline(
&state_c,
&headers_c,
&body_c,
features,
&route_c,
route_ix,
session_header,
tenant,
routing_mode,
)
.await;
let _ = tx.send(out);
});
return sse_keepalive_response(rx, Box::new(anthropic_sse_from_message), sse_error_event);
}
match enforce_pipeline(
state,
headers,
body,
features,
route,
route_ix,
session_header,
tenant,
routing_mode,
)
.await
{
Ok(message) => (axum::http::StatusCode::OK, Json(message)).into_response(),
Err(e) => e.into_response(),
}
}
#[allow(clippy::too_many_arguments)] async fn enforce_pipeline_inner(
state: &AppState,
body: &Bytes,
base_request: ModelRequest,
auth: Auth,
features: Features,
route: &Route,
session_header: Option<String>,
tenant: String,
api: &str,
routing_mode: RoutingMode,
route_ix: usize,
allow_challenger: bool,
) -> Result<ModelResponse, ProxyError> {
let gate_defs = state
.config
.routing
.as_ref()
.map_or(&[][..], |cfg| &cfg.gate_defs);
let gates = resolve_gates(
&route.gates,
gate_defs,
&state.providers,
&auth,
&state.config.prices,
&state.http,
);
let session_id = session_header.unwrap_or_else(|| Uuid::now_v7().to_string());
let live_arm_info: Option<(firstpass_core::Arm, String)> = route
.challenger
.as_ref()
.filter(|ch| allow_challenger && !ch.is_paired())
.map(|ch| {
let bucket = firstpass_core::Challenger::bucket(&state.config.prompt_salt, &session_id);
(ch.arm_of(bucket), ch.fingerprint(&route.ladder))
});
let live_sync_ok = if let Some((firstpass_core::Arm::Challenger, ref fp)) = live_arm_info {
state.experiments.ensure_synced(&tenant, route_ix, fp).await
} else {
true
};
let experiment = route
.challenger
.as_ref()
.filter(|_| allow_challenger)
.map(|ch| {
let bucket = firstpass_core::Challenger::bucket(&state.config.prompt_salt, &session_id);
let assigned = ch.arm_of(bucket);
let arm = if ch.is_paired()
|| (!live_sync_ok && !ch.is_paired() && assigned == firstpass_core::Arm::Challenger)
|| (assigned == firstpass_core::Arm::Challenger
&& state.experiments.challenger_blocked(&tenant, route_ix))
{
firstpass_core::Arm::Incumbent
} else {
assigned
};
(
ch,
firstpass_core::trace::ExperimentRecord {
arm,
percent: ch.percent,
bucket,
fingerprint: ch.fingerprint(&route.ladder),
},
)
});
let on_challenger = experiment
.as_ref()
.is_some_and(|(_, rec)| rec.arm == firstpass_core::Arm::Challenger);
let ladder: &[String] = match &experiment {
Some((ch, rec)) if rec.arm == firstpass_core::Arm::Challenger => &ch.ladder,
_ => &route.ladder,
};
let (budget, max_rungs, speculation, serve_threshold, elastic) =
match state.config.routing.as_ref() {
Some(cfg) => (
cfg.budget.per_request_usd,
cfg.escalation.max_rungs_per_request,
cfg.escalation.speculation,
cfg.escalation.serve_threshold,
cfg.escalation.elastic.as_ref(),
),
None => (None, 3, 0, None, None),
};
let certified = state.eprocess.as_ref().and_then(|e| {
e.lock()
.ok()
.and_then(|g| g.certified_threshold().map(|c| (c, g.alpha(), g.delta())))
});
if let Some((c, _, _)) = &certified {
metrics::gauge!("firstpass_eprocess_certified_threshold").set(c.threshold);
metrics::gauge!("firstpass_eprocess_e_value").set(c.e_value);
}
let aci = state
.adaptive
.as_ref()
.and_then(|a| a.lock().ok().map(|g| (g.threshold(), g.alpha())));
let risk = risk_certificate(certified, aci, serve_threshold);
let serve_threshold = risk.as_ref().map(|r| r.threshold);
let preset = routing_mode.preset();
let max_rungs = if let Some(delta) = preset.max_rungs_delta {
(max_rungs as i32 + delta).max(1) as u32
} else {
max_rungs
};
let speculation = preset.speculation.unwrap_or(speculation);
let bandit_ctx = crate::bandit::ContextBucket::from_features(&features);
let decision_prior: Option<Vec<f64>> = match state.prior.as_ref().filter(|_| !on_challenger) {
Some(prior_client) => {
prior_client
.fetch(&crate::prior::query_text(&base_request))
.await
}
None => None,
};
let (greedy_rung, base_policy_id, ts_propensity) = if on_challenger {
(0, "challenger-ladder@v0".to_owned(), None)
} else {
let (chosen, ts_p) = match state.bandit.as_ref() {
Some(bandit) => bandit
.lock()
.ok()
.map(|mut b| {
b.choose_start_with_prior(
&bandit_ctx,
&route.ladder,
&state.config.prices,
decision_prior.as_deref(),
)
})
.unwrap_or((0, None)),
None => match (state.prior.as_ref(), decision_prior.as_deref()) {
(Some(prior_client), Some(prior)) => crate::bandit::StartRungBandit::new(0, 1.0)
.with_prior_strength(prior_client.strength())
.choose_start_with_prior(
&bandit_ctx,
&route.ladder,
&state.config.prices,
Some(prior),
),
_ => (0, None),
},
};
let policy = if decision_prior.is_some() {
"bandit@v3-prior".to_owned()
} else if ts_p.is_some() {
"bandit@v2-ts".to_owned()
} else if chosen > 0 {
"bandit@v1".to_owned()
} else {
"static-ladder@v0".to_owned()
};
(chosen, policy, ts_p)
};
let exploration_epsilon = state
.config
.routing
.as_ref()
.and_then(|cfg| cfg.escalation.exploration.as_ref())
.map(|e| e.epsilon)
.filter(|_| !on_challenger);
let (start_rung, policy_id, explore_flag, propensity) = if ts_propensity.is_some() {
if exploration_epsilon.is_some() {
tracing::warn!(
"bandit.algorithm = thompson already logs propensities; \
[escalation.exploration] epsilon is ignored"
);
}
(greedy_rung, base_policy_id, false, ts_propensity)
} else if let Some(epsilon) = exploration_epsilon {
let k = ladder.len().max(1);
let u = u01(Uuid::now_v7().as_u128());
let (chosen, eps_branch) = if u < epsilon {
let idx = ((u / epsilon) * k as f64) as u32;
(idx.min(k as u32 - 1), true)
} else {
(greedy_rung, false)
};
let p = epsilon_propensity(chosen, greedy_rung, epsilon, k);
(chosen, format!("{base_policy_id}+eps"), eps_branch, Some(p))
} else {
(greedy_rung, base_policy_id, false, None)
};
let promotion = state.promoter.as_ref().filter(|_| !on_challenger).cloned();
let promotion = match promotion {
Some(p) => {
p.decide_async(&tenant, &session_id, std::time::Instant::now())
.await
}
None => crate::affinity::Decision::Cold,
};
let start_rung = start_rung.max(promotion.start_rung());
let promo_key = state
.promoter
.as_ref()
.filter(|_| !on_challenger)
.map(|_| (tenant.clone(), session_id.clone()));
let start_rung = if preset.start_at_top {
ladder.len().saturating_sub(1) as u32
} else {
start_rung
};
let speculation = match state
.config
.routing
.as_ref()
.and_then(|cfg| cfg.escalation.speculation_band)
{
Some([lo, hi]) if speculation > 0 && !on_challenger => {
let estimate = state
.bandit
.as_ref()
.and_then(|b| b.lock().ok())
.and_then(|b| b.pass_estimate(&bandit_ctx, start_rung));
match estimate {
Some(p) if p < lo || p > hi => {
metrics::counter!("firstpass_speculation_skipped_total").increment(1);
0
}
_ => speculation,
}
}
_ => speculation,
};
if state.bandit.is_some() && !on_challenger {
metrics::counter!(
"firstpass_bandit_start_rung",
"rung" => start_rung.to_string()
)
.increment(1);
}
let mut base_request = base_request;
base_request.cache_prefix = state
.config
.routing
.as_ref()
.is_some_and(|r| r.escalation.prompt_cache);
let paired_spawn_data: Option<(Features, String, String)> = experiment
.as_ref()
.filter(|(ch, _)| ch.is_paired())
.map(|_| (features.clone(), tenant.clone(), session_id.clone()));
let ctx = EnforceCtx {
condense: routing_cfg_condense(state),
ladder,
gates: &gates,
health: &state.gate_health,
base_request: &base_request,
providers: &state.providers,
auth: &auth,
prices: &state.config.prices,
budget_per_request_usd: budget,
max_rungs,
speculation,
serve_threshold,
elastic,
features,
start_rung,
tenant_id: tenant,
session_id,
prompt_hash: prompt_hash(&state.config.prompt_salt, body),
api: api.to_owned(),
policy_id,
};
if let Some(cache) = state.verified_cache.as_ref()
&& let Some(entry) = cache
.get(
&crate::verified_cache::VerifiedCache::compose_key(
&ctx.tenant_id,
&ctx.prompt_hash,
ladder,
),
std::time::Instant::now(),
)
.await
{
metrics::counter!("firstpass_cache_hits_total").increment(1);
let mut trace = cache_hit_trace(&ctx, &entry, mode_now());
if let Some((_, rec)) = &experiment {
trace.route_ix = Some(u32::try_from(route_ix).unwrap_or(0));
trace.experiment = Some(rec.clone());
state.experiments.observe_cost(
&trace.tenant_id,
route_ix,
&rec.fingerprint,
rec.arm,
trace.final_.total_cost_usd,
);
}
offer_trace(&state.traces, state.spill.as_ref(), trace);
return Ok(entry.response);
}
let (outcome, mut trace) = route_enforce(ctx).await;
trace.route_ix = Some(u32::try_from(route_ix).unwrap_or(0));
trace.policy.explore = explore_flag;
trace.policy.propensity = propensity;
if routing_mode != RoutingMode::Balanced {
trace.policy.mode_profile = Some(routing_mode.as_str().to_owned());
}
trace.decision_prior = decision_prior;
trace.risk = risk;
if let Some((_, rec)) = &experiment {
state.experiments.observe_cost(
&trace.tenant_id,
route_ix,
&rec.fingerprint,
rec.arm,
trace.final_.total_cost_usd,
);
state.experiments.observe_health(
&trace.tenant_id,
route_ix,
&rec.fingerprint,
rec.arm,
trace.final_.served_from == firstpass_core::ServedFrom::Error,
crate::experiment::truncated_attempts(&trace),
);
trace.experiment = Some(rec.clone());
}
if let Some(bandit) = state.bandit.as_ref().filter(|_| !on_challenger)
&& let Ok(mut b) = bandit.lock()
{
for attempt in &trace.attempts {
b.observe(&bandit_ctx, attempt.rung, attempt.verdict);
}
}
if let Some(promoter) = state.promoter.as_ref()
&& let Some((tenant_key, session_key)) = promo_key.as_ref()
&& let Some(served) = trace.final_.served_rung
{
promoter
.record_async(
tenant_key,
session_key,
served,
served > start_rung,
std::time::Instant::now(),
)
.await;
}
if let Some(predictor) = state.predictor.as_ref().filter(|_| !on_challenger)
&& let Ok(mut p) = predictor.lock()
{
let predicted = p.predict(&trace.request.features, start_rung);
for attempt in &trace.attempts {
match attempt.verdict {
Verdict::Pass => p.update(&trace.request.features, attempt.rung, true),
Verdict::Fail => p.update(&trace.request.features, attempt.rung, false),
Verdict::Abstain => {} }
}
trace.predicted_pass = Some(predicted);
metrics::histogram!("firstpass_predictor_pass_prob").record(predicted);
}
if let Some(probe_cfg) = state
.config
.routing
.as_ref()
.and_then(|c| c.escalation.probe)
&& u01(Uuid::now_v7().as_u128()) < probe_cfg.sample_rate
{
let probe_rung = (start_rung as usize).min(ladder.len().saturating_sub(1));
if let Some(probe_model_str) = ladder.get(probe_rung).cloned() {
let probe_provider = ModelRef::parse(&probe_model_str)
.ok()
.and_then(|m| state.providers.get(&m.provider));
if let Some(probe_provider) = probe_provider {
let mut join_set = tokio::task::JoinSet::new();
for _ in 0..probe_cfg.k {
let mut probe_req = base_request.clone();
probe_req.model = probe_model_str.clone();
let probe_auth = auth.clone();
let prov = probe_provider.clone();
join_set.spawn(async move { prov.complete(&probe_req, &probe_auth).await });
}
let fail_closed_owned: std::collections::HashSet<String> = gates
.iter()
.filter(|g| g.abstain_fails_closed())
.map(|g| g.id().to_owned())
.collect();
let mut gate_pass_count = 0u32;
let mut probe_cost_usd = 0.0f64;
while let Some(task_result) = join_set.join_next().await {
let Ok(Ok(probe_resp)) = task_result else {
continue; };
probe_cost_usd += state
.config
.prices
.cost_usd_with_cache(
&probe_model_str,
probe_resp.in_tokens,
probe_resp.cache_write_tokens,
probe_resp.cache_read_tokens,
probe_resp.out_tokens,
)
.unwrap_or(0.0);
let mut probe_gate_req = base_request.clone();
probe_gate_req.model = probe_model_str.clone();
let mut probe_gate_results = Vec::with_capacity(gates.len());
for g in &gates {
if !state.gate_health.enabled(&trace.tenant_id, g.id()) {
continue;
}
let r = g.evaluate(&probe_gate_req, &probe_resp).await;
probe_gate_results.push(r);
}
let fail_closed_refs: std::collections::HashSet<&str> =
fail_closed_owned.iter().map(|s| s.as_str()).collect();
let verdict = aggregate_with_policy(&probe_gate_results, &fail_closed_refs);
let passes = match serve_threshold {
None => verdict == Verdict::Pass,
Some(t) => crate::calibrate::gate_score(&probe_gate_results, verdict) >= t,
};
if passes {
gate_pass_count += 1;
}
}
let regime = ProbeRegime::classify(gate_pass_count, probe_cfg.k);
let regime_label = match regime {
ProbeRegime::ConfidentPass => "confident_pass",
ProbeRegime::ConfidentFail => "confident_fail",
ProbeRegime::Ambiguous => "ambiguous",
};
metrics::counter!(
"firstpass_probe_regime_total",
"regime" => regime_label
)
.increment(1);
metrics::gauge!("firstpass_probe_cost_usd_total").increment(probe_cost_usd);
trace.probe = Some(ProbeSignal {
k: probe_cfg.k,
gate_pass_count,
regime,
probe_cost_usd,
});
}
}
}
if let Some(cache) = state.verified_cache.as_ref()
&& let EngineOutcome::Served(resp) = &outcome
&& crate::verified_cache::is_cacheable(&trace)
{
let now = std::time::Instant::now();
cache
.put(
&crate::verified_cache::VerifiedCache::compose_key(
&trace.tenant_id,
&trace.request.prompt_hash,
ladder,
),
trace.trace_id,
crate::verified_cache::Entry::new(resp.clone(), &trace, now),
now,
)
.await;
metrics::counter!("firstpass_cache_stores_total").increment(1);
}
if let Some((ch, rec)) = &experiment
&& ch.is_paired()
{
let trace_id_str = trace.trace_id.to_string();
if ch.paired_sampled(&state.config.prompt_salt, &trace_id_str)
&& let Some((spawn_feats, spawn_tenant, spawn_session)) = paired_spawn_data
{
let served_from = trace.final_.served_from;
let incumbent_cost = trace.final_.total_cost_usd;
spawn_paired_challenger(
state,
route,
ch,
rec.fingerprint.clone(),
route_ix,
body.clone(),
auth.clone(),
spawn_feats,
spawn_tenant,
spawn_session,
api,
served_from,
incumbent_cost,
trace.trace_id,
);
}
}
offer_trace(&state.traces, state.spill.as_ref(), trace);
match outcome {
EngineOutcome::Served(resp) => Ok(resp),
EngineOutcome::Failed(msg) => Err(ProxyError::Engine(msg)),
}
}
#[allow(clippy::too_many_arguments)]
async fn enforce_pipeline(
state: &AppState,
headers: &HeaderMap,
body: &Bytes,
features: Features,
route: &Route,
route_ix: usize,
session_header: Option<String>,
tenant: String,
routing_mode: RoutingMode,
) -> Result<Value, ProxyError> {
let Some(base_request) = parse_model_request(body) else {
return Err(ProxyError::BadRequest(
"request body is not a valid Anthropic Messages request".to_owned(),
));
};
let auth = Auth::from_headers(headers, firstpass_core::Dialect::Anthropic);
let resp = enforce_pipeline_inner(
state,
body,
base_request,
auth,
features,
route,
session_header,
tenant,
"anthropic.messages",
routing_mode,
route_ix,
true,
)
.await?;
Ok(anthropic_response_json(&resp))
}
#[allow(clippy::too_many_arguments)]
async fn enforce_pipeline_openai(
state: &AppState,
headers: &HeaderMap,
body: &Bytes,
features: Features,
route: &Route,
route_ix: usize,
session_header: Option<String>,
tenant: String,
routing_mode: RoutingMode,
) -> Result<Value, ProxyError> {
enforce_pipeline_openai_as(
state,
headers,
body,
body,
"openai.chat_completions",
features,
route,
route_ix,
session_header,
tenant,
routing_mode,
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn enforce_pipeline_openai_as(
state: &AppState,
headers: &HeaderMap,
body: &Bytes,
receipt_body: &Bytes,
api: &str,
features: Features,
route: &Route,
route_ix: usize,
session_header: Option<String>,
tenant: String,
routing_mode: RoutingMode,
) -> Result<Value, ProxyError> {
let providers = &state.providers;
let all_openai_of = |ladder: &[String]| {
ladder.iter().all(|rung| {
let pid = rung.split('/').next().unwrap_or_default();
providers
.get(pid)
.is_some_and(|p| p.carries_structured_verbatim(Dialect::Openai))
})
};
let all_openai = all_openai_of(&route.ladder);
let allow_challenger = route
.challenger
.as_ref()
.is_none_or(|c| all_openai_of(&c.ladder) == all_openai);
let Some(base_request) = parse_openai_request(body, all_openai) else {
return Err(ProxyError::BadRequest(
"request body is not a valid OpenAI Chat Completions request".to_owned(),
));
};
let auth = Auth::from_headers(headers, firstpass_core::Dialect::Openai);
let resp = enforce_pipeline_inner(
state,
receipt_body,
base_request,
auth,
features,
route,
session_header,
tenant,
api,
routing_mode,
route_ix,
allow_challenger,
)
.await?;
Ok(openai_response_json(&resp))
}
const SSE_KEEPALIVE_EVERY: Duration = Duration::from_secs(5);
fn sse_keepalive_response(
rx: tokio::sync::oneshot::Receiver<Result<Value, ProxyError>>,
format_message: Box<dyn Fn(&Value) -> String + Send>,
format_error: fn(&ProxyError) -> String,
) -> Response {
let mut ticks = tokio::time::interval(SSE_KEEPALIVE_EVERY);
ticks.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
ticks.reset(); let stream = KeepaliveStream {
rx: Some(rx),
ticks,
format_message,
format_error,
};
(
axum::http::StatusCode::OK,
[(
axum::http::header::CONTENT_TYPE,
"text/event-stream; charset=utf-8",
)],
axum::body::Body::from_stream(stream),
)
.into_response()
}
struct KeepaliveStream {
rx: Option<tokio::sync::oneshot::Receiver<Result<Value, ProxyError>>>,
ticks: tokio::time::Interval,
format_message: Box<dyn Fn(&Value) -> String + Send>,
format_error: fn(&ProxyError) -> String,
}
impl futures_core::Stream for KeepaliveStream {
type Item = Result<Bytes, std::convert::Infallible>;
fn poll_next(
mut self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<Option<Self::Item>> {
use std::task::Poll;
let Some(rx) = self.rx.as_mut() else {
return Poll::Ready(None); };
if let Poll::Ready(out) = std::pin::Pin::new(rx).poll(cx) {
let frame = match out {
Ok(Ok(message)) => (self.format_message)(&message),
Ok(Err(e)) => (self.format_error)(&e),
Err(_) => (self.format_error)(&ProxyError::Internal(
"enforce pipeline task dropped".to_owned(),
)),
};
self.rx = None;
return Poll::Ready(Some(Ok(Bytes::from(frame))));
}
if self.ticks.poll_tick(cx).is_ready() {
return Poll::Ready(Some(Ok(Bytes::from_static(b": firstpass routing\n\n"))));
}
Poll::Pending
}
}
fn sse_error_event(e: &ProxyError) -> String {
let mut out = String::new();
sse_event(
&mut out,
"error",
&serde_json::json!({
"type": "error",
"error": { "type": "api_error", "message": e.client_message() }
}),
);
out
}
fn parse_model_request(body: &[u8]) -> Option<ModelRequest> {
let json: Value = serde_json::from_slice(body).ok()?;
let raw = json.clone();
let messages_json = json.get("messages")?.as_array()?;
let messages = messages_json
.iter()
.map(|m| ChatMessage {
role: m
.get("role")
.and_then(Value::as_str)
.unwrap_or("user")
.to_owned(),
content: m
.get("content")
.cloned()
.unwrap_or_else(|| Value::String(String::new())),
})
.collect();
let system = json
.get("system")
.and_then(Value::as_str)
.map(str::to_owned);
let max_tokens = json
.get("max_tokens")
.and_then(Value::as_u64)
.and_then(|n| u32::try_from(n).ok())
.unwrap_or(1024);
let tools = json.get("tools").cloned().unwrap_or(Value::Null);
Some(ModelRequest {
model: json
.get("model")
.and_then(Value::as_str)
.unwrap_or_default()
.to_owned(),
system,
messages,
max_tokens,
tools,
raw,
cache_prefix: false,
})
}
fn openai_finish_reason_to_anthropic(finish_reason: &str) -> &str {
match finish_reason {
"tool_calls" => "tool_use",
"length" => "max_tokens",
"stop" => "end_turn",
"content_filter" => "end_turn",
other => other,
}
}
fn openai_message_to_anthropic_content(msg: &Value) -> Value {
let mut blocks: Vec<Value> = Vec::new();
if let Some(text) = msg
.get("content")
.and_then(Value::as_str)
.filter(|s| !s.is_empty())
{
blocks.push(serde_json::json!({ "type": "text", "text": text }));
}
if let Some(tool_calls) = msg.get("tool_calls").and_then(Value::as_array) {
for tc in tool_calls {
let id = tc
.get("id")
.and_then(Value::as_str)
.filter(|s| !s.is_empty())
.map_or_else(
|| format!("toolu_{}", Uuid::now_v7().simple()),
str::to_owned,
);
let name = tc
.pointer("/function/name")
.and_then(Value::as_str)
.unwrap_or("");
let input = tc
.pointer("/function/arguments")
.and_then(Value::as_str)
.and_then(|s| serde_json::from_str::<Value>(s).ok())
.filter(Value::is_object)
.unwrap_or_else(|| serde_json::json!({}));
if !name.is_empty() {
blocks.push(serde_json::json!({
"type": "tool_use",
"id": id,
"name": name,
"input": input,
}));
}
}
}
if blocks.is_empty() {
serde_json::json!([{ "type": "text", "text": "" }])
} else {
Value::Array(blocks)
}
}
fn gemini_finish_reason_to_anthropic(reason: &str) -> &str {
match reason {
"MAX_TOKENS" => "max_tokens",
"STOP" => "end_turn",
other => other,
}
}
fn anthropic_response_json(resp: &ModelResponse) -> Value {
let content = if let Some(c) = resp.raw.get("content").filter(|c| c.is_array()) {
c.clone()
} else if let Some(msg) = resp.raw.pointer("/choices/0/message") {
openai_message_to_anthropic_content(msg)
} else {
serde_json::json!([{ "type": "text", "text": resp.text }])
};
let has_tool_use = content.as_array().is_some_and(|a| {
a.iter()
.any(|b| b.get("type") == Some(&Value::String("tool_use".to_owned())))
});
let stop_reason_str = resp
.raw
.get("stop_reason")
.and_then(Value::as_str)
.or_else(|| {
resp.raw
.pointer("/choices/0/finish_reason")
.and_then(Value::as_str)
.map(openai_finish_reason_to_anthropic)
})
.or_else(|| {
resp.raw
.pointer("/candidates/0/finishReason")
.and_then(Value::as_str)
.map(gemini_finish_reason_to_anthropic)
})
.unwrap_or("end_turn");
let stop_reason_str = if stop_reason_str == "tool_use" && !has_tool_use {
"end_turn"
} else {
stop_reason_str
};
let stop_reason = Value::String(stop_reason_str.to_owned());
let stop_sequence = resp
.raw
.get("stop_sequence")
.cloned()
.unwrap_or(Value::Null);
let mut usage = serde_json::json!({
"input_tokens": resp.in_tokens,
"output_tokens": resp.out_tokens,
});
if resp.cache_write_tokens > 0 {
usage["cache_creation_input_tokens"] = Value::from(resp.cache_write_tokens);
}
if resp.cache_read_tokens > 0 {
usage["cache_read_input_tokens"] = Value::from(resp.cache_read_tokens);
}
serde_json::json!({
"id": format!("msg_{}", Uuid::now_v7()),
"type": "message",
"role": "assistant",
"model": resp.model,
"content": content,
"stop_reason": stop_reason,
"stop_sequence": stop_sequence,
"usage": usage,
})
}
fn sse_event(out: &mut String, event: &str, data: &Value) {
out.push_str("event: ");
out.push_str(event);
out.push_str("\ndata: ");
out.push_str(&data.to_string());
out.push_str("\n\n");
}
fn anthropic_sse_from_message(message: &Value) -> String {
let mut out = String::new();
let mut start_msg = message.clone();
start_msg["content"] = Value::Array(Vec::new());
start_msg["stop_reason"] = Value::Null;
start_msg["stop_sequence"] = Value::Null;
start_msg["usage"]["output_tokens"] = Value::from(0u64);
sse_event(
&mut out,
"message_start",
&serde_json::json!({ "type": "message_start", "message": start_msg }),
);
let empty = Vec::new();
let blocks = message
.get("content")
.and_then(Value::as_array)
.unwrap_or(&empty);
for (i, block) in blocks.iter().enumerate() {
match block.get("type").and_then(Value::as_str) {
Some("tool_use") => {
let mut shell = block.clone();
shell["input"] = serde_json::json!({});
sse_event(
&mut out,
"content_block_start",
&serde_json::json!({ "type": "content_block_start", "index": i, "content_block": shell }),
);
let input_json = block
.get("input")
.map_or_else(|| "{}".to_owned(), std::string::ToString::to_string);
sse_event(
&mut out,
"content_block_delta",
&serde_json::json!({ "type": "content_block_delta", "index": i,
"delta": { "type": "input_json_delta", "partial_json": input_json } }),
);
}
Some("thinking") => {
let thinking = block.get("thinking").and_then(Value::as_str).unwrap_or("");
let signature = block.get("signature").and_then(Value::as_str).unwrap_or("");
sse_event(
&mut out,
"content_block_start",
&serde_json::json!({ "type": "content_block_start", "index": i,
"content_block": { "type": "thinking", "thinking": "" } }),
);
sse_event(
&mut out,
"content_block_delta",
&serde_json::json!({ "type": "content_block_delta", "index": i,
"delta": { "type": "thinking_delta", "thinking": thinking } }),
);
if !signature.is_empty() {
sse_event(
&mut out,
"content_block_delta",
&serde_json::json!({ "type": "content_block_delta", "index": i,
"delta": { "type": "signature_delta", "signature": signature } }),
);
}
}
Some("redacted_thinking") => {
sse_event(
&mut out,
"content_block_start",
&serde_json::json!({ "type": "content_block_start", "index": i,
"content_block": block }),
);
}
_ => {
let text = block.get("text").and_then(Value::as_str).unwrap_or("");
sse_event(
&mut out,
"content_block_start",
&serde_json::json!({ "type": "content_block_start", "index": i,
"content_block": { "type": "text", "text": "" } }),
);
sse_event(
&mut out,
"content_block_delta",
&serde_json::json!({ "type": "content_block_delta", "index": i,
"delta": { "type": "text_delta", "text": text } }),
);
}
}
sse_event(
&mut out,
"content_block_stop",
&serde_json::json!({ "type": "content_block_stop", "index": i }),
);
}
let out_tokens = message
.pointer("/usage/output_tokens")
.cloned()
.unwrap_or_else(|| Value::from(0));
let stop_reason = message
.get("stop_reason")
.cloned()
.unwrap_or_else(|| Value::String("end_turn".to_owned()));
let mut message_delta_delta =
serde_json::json!({ "stop_reason": stop_reason, "stop_sequence": Value::Null });
if let Some(seq) = message.get("stop_sequence").filter(|v| !v.is_null()) {
message_delta_delta["stop_sequence"] = seq.clone();
}
sse_event(
&mut out,
"message_delta",
&serde_json::json!({ "type": "message_delta", "delta": message_delta_delta,
"usage": { "output_tokens": out_tokens } }),
);
sse_event(
&mut out,
"message_stop",
&serde_json::json!({ "type": "message_stop" }),
);
out
}
async fn observe_passthrough(
state: AppState,
headers: HeaderMap,
body: Bytes,
session_header: Option<String>,
tenant: String,
) -> Response {
if is_stream_request(&body) {
return observe_stream(state, headers, body, session_header, tenant).await;
}
let start = Instant::now();
let result = forward_anthropic(
&state.http,
&state.config.upstream_anthropic,
&headers,
body.clone(),
)
.await;
let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
match result {
Ok((status, resp_headers, resp_body)) => {
let for_trace = status.is_success().then_some(resp_body.clone());
spawn_trace(
&state,
body,
for_trace,
latency_ms,
session_header,
tenant,
"anthropic",
"anthropic.messages",
None,
);
(status, resp_headers, resp_body).into_response()
}
Err(err) => {
spawn_trace(
&state,
body,
None,
latency_ms,
session_header,
tenant,
"anthropic",
"anthropic.messages",
None,
);
err.into_response()
}
}
}
async fn observe_stream(
state: AppState,
headers: HeaderMap,
body: Bytes,
session_header: Option<String>,
tenant: String,
) -> Response {
let start = Instant::now();
let result = forward_anthropic_streaming(
&state.http,
&state.config.upstream_anthropic,
&headers,
body.clone(),
)
.await;
let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
match result {
Ok((status, resp_headers, response)) => {
let on_done = stream_trace_callback(
&state,
body,
latency_ms,
session_header,
tenant,
"anthropic",
status.is_success(),
"anthropic.messages",
None,
);
let tap = SseTap::new(response.bytes_stream(), on_done);
let stream_body = Body::from_stream(tap);
(status, resp_headers, stream_body).into_response()
}
Err(err) => {
spawn_trace(
&state,
body,
None,
latency_ms,
session_header,
tenant,
"anthropic",
"anthropic.messages",
None,
);
err.into_response()
}
}
}
#[allow(clippy::too_many_arguments)]
fn stream_trace_callback(
state: &AppState,
req_body: Bytes,
latency_ms: u64,
session_header: Option<String>,
tenant: String,
provider: &'static str,
upstream_success: bool,
api: &'static str,
model_hint: Option<String>,
) -> impl FnOnce(SseUsage, SseEndReason) + Send + 'static {
let config = state.config.clone();
let traces = state.traces.clone();
let spill = state.spill.clone();
move |usage: SseUsage, reason: SseEndReason| {
let mut trace = if !upstream_success {
build_error_trace(
&config,
&req_body,
latency_ms,
session_header.as_deref(),
provider,
api,
)
} else {
match reason {
SseEndReason::UpstreamError => {
build_stream_trace_from_usage(
&config,
&req_body,
latency_ms,
session_header.as_deref(),
usage,
provider,
StreamOutcome::MidStreamError,
api,
model_hint.as_deref(),
)
}
SseEndReason::Dropped => {
let outcome = if usage.completed {
StreamOutcome::Served
} else {
StreamOutcome::Partial
};
build_stream_trace_from_usage(
&config,
&req_body,
latency_ms,
session_header.as_deref(),
usage,
provider,
outcome,
api,
model_hint.as_deref(),
)
}
SseEndReason::Eof => {
let saw_events =
usage.model.is_some() || usage.in_tokens > 0 || usage.out_tokens > 0;
let outcome = if usage.completed || !saw_events {
StreamOutcome::Served
} else {
StreamOutcome::Partial
};
build_stream_trace_from_usage(
&config,
&req_body,
latency_ms,
session_header.as_deref(),
usage,
provider,
outcome,
api,
model_hint.as_deref(),
)
}
}
};
trace.tenant_id = tenant;
offer_trace(&traces, spill.as_ref(), trace);
}
}
const SSE_LINE_BUF_CAP: usize = 64 * 1024;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum SseEndReason {
Eof,
UpstreamError,
Dropped,
}
#[derive(Debug, Default)]
struct SseUsage {
model: Option<String>,
in_tokens: u64,
cache_write_tokens: u64,
cache_read_tokens: u64,
out_tokens: u64,
stop_reason: Option<String>,
completed: bool,
}
struct SseTap {
inner: std::pin::Pin<Box<dyn Stream<Item = Result<Bytes, reqwest::Error>> + Send>>,
line_buf: Vec<u8>,
overflow: bool,
usage: SseUsage,
on_done: Option<Box<dyn FnOnce(SseUsage, SseEndReason) + Send>>,
}
impl SseTap {
fn new(
inner: impl Stream<Item = Result<Bytes, reqwest::Error>> + Send + 'static,
on_done: impl FnOnce(SseUsage, SseEndReason) + Send + 'static,
) -> Self {
Self {
inner: Box::pin(inner),
line_buf: Vec::new(),
overflow: false,
usage: SseUsage::default(),
on_done: Some(Box::new(on_done)),
}
}
}
impl Stream for SseTap {
type Item = Result<Bytes, reqwest::Error>;
fn poll_next(self: std::pin::Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
let this = self.get_mut();
match this.inner.as_mut().poll_next(cx) {
Poll::Pending => Poll::Pending,
Poll::Ready(Some(Ok(chunk))) => {
parse_sse_chunk_into(
&chunk,
&mut this.line_buf,
&mut this.overflow,
&mut this.usage,
);
Poll::Ready(Some(Ok(chunk)))
}
Poll::Ready(Some(Err(e))) => {
if let Some(on_done) = this.on_done.take() {
on_done(std::mem::take(&mut this.usage), SseEndReason::UpstreamError);
}
Poll::Ready(Some(Err(e)))
}
Poll::Ready(None) => {
if let Some(on_done) = this.on_done.take() {
on_done(std::mem::take(&mut this.usage), SseEndReason::Eof);
}
Poll::Ready(None)
}
}
}
}
impl Drop for SseTap {
fn drop(&mut self) {
if let Some(on_done) = self.on_done.take() {
on_done(std::mem::take(&mut self.usage), SseEndReason::Dropped);
}
}
}
fn parse_sse_chunk_into(
chunk: &[u8],
line_buf: &mut Vec<u8>,
overflow: &mut bool,
usage: &mut SseUsage,
) {
for &byte in chunk {
if byte == b'\n' {
if !*overflow {
process_sse_line(line_buf, usage);
}
line_buf.clear();
*overflow = false;
} else if *overflow {
} else {
line_buf.push(byte);
if line_buf.len() > SSE_LINE_BUF_CAP {
*overflow = true;
line_buf.clear(); }
}
}
}
fn process_sse_line(line: &[u8], usage: &mut SseUsage) {
let Some(data) = line.strip_prefix(b"data: ") else {
return;
};
let data = data.strip_suffix(b"\r").unwrap_or(data);
if data == b"[DONE]" {
usage.completed = true;
return;
}
let Ok(json) = serde_json::from_slice::<Value>(data) else {
return;
};
if usage.model.is_none() {
let model = json
.get("model")
.or_else(|| json.pointer("/message/model"))
.and_then(Value::as_str);
if let Some(m) = model {
usage.model = Some(m.to_owned());
}
}
match json.get("type").and_then(Value::as_str) {
Some("message_start") => {
let u = |ptr: &str| json.pointer(ptr).and_then(Value::as_u64).unwrap_or(0);
usage.in_tokens = u("/message/usage/input_tokens");
usage.cache_write_tokens = u("/message/usage/cache_creation_input_tokens");
usage.cache_read_tokens = u("/message/usage/cache_read_input_tokens");
}
Some("message_delta") => {
if let Some(n) = json.pointer("/usage/output_tokens").and_then(Value::as_u64) {
usage.out_tokens = n;
}
if let Some(stop) = json.pointer("/delta/stop_reason").and_then(Value::as_str) {
usage.stop_reason = Some(stop.to_owned());
usage.completed = true;
}
}
Some("message_stop") => usage.completed = true,
_ => {
let is_gemini = json.get("candidates").is_some() || json.get("usageMetadata").is_some();
if let Some(fr) = json
.pointer("/candidates/0/finishReason")
.and_then(Value::as_str)
.filter(|fr| !fr.is_empty() && *fr != "FINISH_REASON_UNSPECIFIED")
{
usage.stop_reason = Some(fr.to_owned());
usage.completed = true;
}
if json.pointer("/usageMetadata/promptTokenCount").is_some() {
let u = |ptr: &str| json.pointer(ptr).and_then(Value::as_u64).unwrap_or(0);
let cached = u("/usageMetadata/cachedContentTokenCount");
usage.in_tokens = u("/usageMetadata/promptTokenCount").saturating_sub(cached);
usage.cache_read_tokens = cached;
usage.out_tokens = u("/usageMetadata/candidatesTokenCount")
+ u("/usageMetadata/thoughtsTokenCount");
}
if is_gemini {
return;
}
if let Some(pt) = json.pointer("/usage/prompt_tokens").and_then(Value::as_u64) {
let cached = json
.pointer("/usage/prompt_tokens_details/cached_tokens")
.and_then(Value::as_u64)
.unwrap_or(0);
usage.in_tokens = pt.saturating_sub(cached);
usage.cache_read_tokens = cached;
}
if let Some(ct) = json
.pointer("/usage/completion_tokens")
.and_then(Value::as_u64)
{
usage.out_tokens = ct;
}
if let Some(fr) = json
.pointer("/choices/0/finish_reason")
.and_then(Value::as_str)
.filter(|fr| !fr.is_empty())
{
usage.stop_reason = Some(fr.to_owned());
usage.completed = true;
}
}
}
}
#[allow(clippy::too_many_arguments)]
fn spawn_trace(
state: &AppState,
req_body: Bytes,
resp_body: Option<Bytes>,
latency_ms: u64,
session_header: Option<String>,
tenant: String,
provider: &'static str,
api: &'static str,
model_hint: Option<String>,
) {
let config = state.config.clone();
let traces = state.traces.clone();
let spill = state.spill.clone();
tokio::spawn(async move {
let mut trace = match resp_body {
Some(resp) => build_trace(
&config,
&req_body,
&resp,
latency_ms,
session_header.as_deref(),
provider,
api,
model_hint.as_deref(),
),
None => build_error_trace(
&config,
&req_body,
latency_ms,
session_header.as_deref(),
provider,
api,
),
};
trace.tenant_id = tenant;
offer_trace(&traces, spill.as_ref(), trace);
});
}
fn session_id(session_header: Option<&str>, trace_id: Uuid) -> String {
session_header
.map(str::to_owned)
.unwrap_or_else(|| trace_id.to_string())
}
fn prompt_hash(salt: &str, body: &[u8]) -> String {
let mut salted = Vec::with_capacity(salt.len() + body.len());
salted.extend_from_slice(salt.as_bytes());
salted.extend_from_slice(body);
sha256_hex(&salted)
}
fn request_features(body: &[u8]) -> (Option<String>, u32, bool) {
let Ok(json) = serde_json::from_slice::<Value>(body) else {
return (None, 0, false);
};
let model = json.get("model").and_then(Value::as_str).map(str::to_owned);
let tool_count = json
.get("tools")
.and_then(Value::as_array)
.map_or(0, |tools| u32::try_from(tools.len()).unwrap_or(u32::MAX));
let has_images = json
.get("messages")
.and_then(Value::as_array)
.is_some_and(|messages| messages.iter().any(message_has_image));
(model, tool_count, has_images)
}
fn message_has_image(message: &Value) -> bool {
message
.get("content")
.and_then(Value::as_array)
.is_some_and(|blocks| {
blocks
.iter()
.any(|block| block.get("type").and_then(Value::as_str) == Some("image"))
})
}
#[derive(Debug, Default, Clone, PartialEq, Eq)]
struct ResponseUsage {
model: Option<String>,
in_tokens: u64,
cache_write_tokens: u64,
cache_read_tokens: u64,
out_tokens: u64,
}
fn response_usage(body: &[u8]) -> ResponseUsage {
let Ok(json) = serde_json::from_slice::<Value>(body) else {
return ResponseUsage::default();
};
let u = |ptr: &str| json.pointer(ptr).and_then(Value::as_u64).unwrap_or(0);
let model = json.get("model").and_then(Value::as_str).map(str::to_owned);
if json.pointer("/usageMetadata/promptTokenCount").is_some() {
let cached = u("/usageMetadata/cachedContentTokenCount");
return ResponseUsage {
model, in_tokens: u("/usageMetadata/promptTokenCount").saturating_sub(cached),
cache_write_tokens: 0,
cache_read_tokens: cached,
out_tokens: u("/usageMetadata/candidatesTokenCount")
+ u("/usageMetadata/thoughtsTokenCount"),
};
}
if json.pointer("/usage/prompt_tokens").is_some() {
let cached = u("/usage/prompt_tokens_details/cached_tokens");
return ResponseUsage {
model,
in_tokens: u("/usage/prompt_tokens").saturating_sub(cached),
cache_write_tokens: 0,
cache_read_tokens: cached,
out_tokens: u("/usage/completion_tokens"),
};
}
ResponseUsage {
model,
in_tokens: u("/usage/input_tokens"),
cache_write_tokens: u("/usage/cache_creation_input_tokens"),
cache_read_tokens: u("/usage/cache_read_input_tokens"),
out_tokens: u("/usage/output_tokens"),
}
}
#[allow(clippy::too_many_arguments)]
fn build_trace(
config: &ProxyConfig,
req_body: &Bytes,
resp_body: &Bytes,
latency_ms: u64,
session_header: Option<&str>,
provider: &str,
api: &str,
model_hint: Option<&str>,
) -> Trace {
let (req_model, tool_count, has_images) = request_features(req_body);
let usage = response_usage(resp_body);
let (cache_write_tokens, cache_read_tokens) =
(usage.cache_write_tokens, usage.cache_read_tokens);
let (in_tokens, out_tokens) = (usage.in_tokens, usage.out_tokens);
let model = usage
.model
.as_deref()
.or(req_model.as_deref())
.or(model_hint)
.unwrap_or("unknown")
.to_owned();
let cost_usd = config
.prices
.cost_usd_with_cache(
&format!(
"{provider}/{}",
model
.strip_prefix(&format!("{provider}/"))
.unwrap_or(&model)
),
in_tokens,
cache_write_tokens,
cache_read_tokens,
out_tokens,
)
.or_else(|_| {
let bare_req = req_model.as_deref().unwrap_or("unknown");
let bare_req = bare_req
.strip_prefix(&format!("{provider}/"))
.unwrap_or(bare_req);
let bare_resp = model
.strip_prefix(&format!("{provider}/"))
.unwrap_or(&model);
if is_snapshot_of(bare_resp, bare_req) {
config.prices.cost_usd_with_cache(
&format!("{provider}/{bare_req}"),
in_tokens,
cache_write_tokens,
cache_read_tokens,
out_tokens,
)
} else {
Err(firstpass_core::error::Error::UnknownModel(model.clone()))
}
})
.unwrap_or(0.0);
let attempt = Attempt {
rung: 0,
model,
provider: provider.to_owned(),
in_tokens,
cache_write_tokens,
cache_read_tokens,
out_tokens,
cost_usd,
latency_ms,
gates: Vec::new(),
verdict: Verdict::Pass,
truncated: serde_json::from_slice::<Value>(resp_body)
.is_ok_and(|v| crate::provider::hit_output_limit(&v)),
};
let mut trace = base_trace(config, req_body, latency_ms, session_header, api);
trace.request.features.prompt_token_bucket = token_bucket(in_tokens);
trace.request.features.tool_count = tool_count;
trace.request.features.has_images = has_images;
trace.attempts.push(attempt);
trace.final_ = FinalOutcome {
served_rung: Some(0),
served_from: ServedFrom::Attempt,
total_cost_usd: cost_usd,
gate_cost_usd: 0.0,
total_latency_ms: latency_ms,
escalations: 0,
counterfactual_baseline_usd: cost_usd,
savings_usd: 0.0,
cache_source: None,
};
trace.recompute_savings();
trace
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum StreamOutcome {
Served,
MidStreamError,
Partial,
}
#[allow(clippy::too_many_arguments)]
fn build_stream_trace_from_usage(
config: &ProxyConfig,
req_body: &Bytes,
latency_ms: u64,
session_header: Option<&str>,
usage: SseUsage,
provider: &str,
outcome: StreamOutcome,
api: &str,
model_hint: Option<&str>,
) -> Trace {
let (req_model, tool_count, has_images) = request_features(req_body);
let (in_tokens, cache_write_tokens, cache_read_tokens, out_tokens) = (
usage.in_tokens,
usage.cache_write_tokens,
usage.cache_read_tokens,
usage.out_tokens,
);
let model = usage
.model
.as_deref()
.or(req_model.as_deref())
.or(model_hint)
.unwrap_or("unknown")
.to_owned();
let cost_usd = config
.prices
.cost_usd_with_cache(
&format!(
"{provider}/{}",
model
.strip_prefix(&format!("{provider}/"))
.unwrap_or(&model)
),
in_tokens,
cache_write_tokens,
cache_read_tokens,
out_tokens,
)
.or_else(|_| {
let bare_req = req_model.as_deref().unwrap_or("unknown");
let bare_req = bare_req
.strip_prefix(&format!("{provider}/"))
.unwrap_or(bare_req);
let bare_resp = model
.strip_prefix(&format!("{provider}/"))
.unwrap_or(&model);
if is_snapshot_of(bare_resp, bare_req) {
config.prices.cost_usd_with_cache(
&format!("{provider}/{bare_req}"),
in_tokens,
cache_write_tokens,
cache_read_tokens,
out_tokens,
)
} else {
Err(firstpass_core::error::Error::UnknownModel(model.clone()))
}
})
.unwrap_or(0.0);
let truncated = matches!(
usage.stop_reason.as_deref(),
Some("max_tokens") | Some("length") | Some("MAX_TOKENS")
);
let (verdict, served_from, served_rung) = match outcome {
StreamOutcome::Served => (Verdict::Pass, ServedFrom::Attempt, Some(0u32)),
StreamOutcome::MidStreamError => (Verdict::Abstain, ServedFrom::Error, None),
StreamOutcome::Partial => (Verdict::Abstain, ServedFrom::Attempt, Some(0u32)),
};
let attempt = Attempt {
rung: 0,
model,
provider: provider.to_owned(),
in_tokens,
cache_write_tokens,
cache_read_tokens,
out_tokens,
cost_usd,
latency_ms,
gates: Vec::new(),
verdict,
truncated,
};
let mut trace = base_trace(config, req_body, latency_ms, session_header, api);
trace.request.features.prompt_token_bucket = token_bucket(in_tokens);
trace.request.features.tool_count = tool_count;
trace.request.features.has_images = has_images;
trace.attempts.push(attempt);
trace.final_ = FinalOutcome {
served_rung,
served_from,
total_cost_usd: cost_usd,
gate_cost_usd: 0.0,
total_latency_ms: latency_ms,
escalations: 0,
counterfactual_baseline_usd: cost_usd,
savings_usd: 0.0,
cache_source: None,
};
trace.recompute_savings();
trace
}
fn is_snapshot_of(resp_model: &str, req_model: &str) -> bool {
let Some(suffix) = resp_model.strip_prefix(&format!("{req_model}-")) else {
return false;
};
is_date_version_suffix(suffix)
}
fn is_date_version_suffix(s: &str) -> bool {
if s.len() == 8 && s.bytes().all(|b| b.is_ascii_digit()) {
return true;
}
if s.len() == 10 {
let b = s.as_bytes();
return b[4] == b'-'
&& b[7] == b'-'
&& b[..4].iter().all(|c| c.is_ascii_digit())
&& b[5..7].iter().all(|c| c.is_ascii_digit())
&& b[8..].iter().all(|c| c.is_ascii_digit());
}
false
}
fn build_error_trace(
config: &ProxyConfig,
req_body: &Bytes,
latency_ms: u64,
session_header: Option<&str>,
provider: &str,
api: &str,
) -> Trace {
let (_, tool_count, has_images) = request_features(req_body);
let mut trace = base_trace(config, req_body, latency_ms, session_header, api);
trace.request.features.tool_count = tool_count;
trace.request.features.has_images = has_images;
trace.upstream_provider = provider.to_owned();
trace.final_ = FinalOutcome {
served_rung: None,
served_from: ServedFrom::Error,
total_cost_usd: 0.0,
gate_cost_usd: 0.0,
total_latency_ms: latency_ms,
escalations: 0,
counterfactual_baseline_usd: 0.0,
savings_usd: 0.0,
cache_source: None,
};
trace.recompute_savings();
trace
}
fn base_trace(
config: &ProxyConfig,
req_body: &Bytes,
latency_ms: u64,
session_header: Option<&str>,
api: &str,
) -> Trace {
let trace_id = Uuid::now_v7();
let mut features = Features::new(TaskKind::Other);
features.hour_bucket = hour_bucket(jiff::Timestamp::now());
Trace {
trace_id,
prev_hash: GENESIS_HASH.to_owned(),
tenant_id: config.tenant_id.clone(),
session_id: session_id(session_header, trace_id),
ts: jiff::Timestamp::now(),
mode: Mode::Observe,
policy: PolicyRef {
id: "observe-passthrough@v0".to_owned(),
explore: false,
propensity: None,
mode_profile: None,
},
request: RequestInfo {
api: api.to_owned(),
prompt_hash: prompt_hash(&config.prompt_salt, req_body),
features,
},
attempts: Vec::new(),
deferred: Vec::new(),
final_: FinalOutcome {
served_rung: None,
served_from: ServedFrom::Error,
total_cost_usd: 0.0,
gate_cost_usd: 0.0,
total_latency_ms: latency_ms,
escalations: 0,
counterfactual_baseline_usd: 0.0,
savings_usd: 0.0,
cache_source: None,
},
probe: None,
rollout: None,
shadow: None,
route_ix: None,
predicted_pass: None,
decision_prior: None,
elastic: None,
risk: None,
experiment: None,
upstream_provider: String::new(),
}
}
fn openai_messages_have_tool_calls(body: &[u8]) -> bool {
serde_json::from_slice::<Value>(body)
.ok()
.and_then(|json| {
json.get("messages").and_then(Value::as_array).map(|msgs| {
msgs.iter().any(|m| {
m.get("tool_calls").is_some()
|| m.get("role").and_then(Value::as_str) == Some("tool")
})
})
})
.unwrap_or(false)
}
fn openai_has_http_images(body: &[u8]) -> bool {
serde_json::from_slice::<Value>(body)
.ok()
.and_then(|json| {
json.get("messages").and_then(Value::as_array).map(|msgs| {
msgs.iter().any(|m| {
m.get("content")
.and_then(Value::as_array)
.is_some_and(|parts| {
parts.iter().any(|p| {
p.get("type").and_then(Value::as_str) == Some("image_url")
&& p.pointer("/image_url/url")
.and_then(Value::as_str)
.is_some_and(|u| {
u.starts_with("http://") || u.starts_with("https://")
})
})
})
})
})
})
.unwrap_or(false)
}
fn openai_messages_have_images(body: &[u8]) -> bool {
serde_json::from_slice::<Value>(body)
.ok()
.and_then(|json| {
json.get("messages").and_then(Value::as_array).map(|msgs| {
msgs.iter().any(|m| {
m.get("content")
.and_then(Value::as_array)
.is_some_and(|parts| {
parts
.iter()
.any(|p| p.get("type").and_then(Value::as_str) == Some("image_url"))
})
})
})
})
.unwrap_or(false)
}
fn extract_openai_features(headers: &HeaderMap, body: &[u8]) -> Features {
let Ok(json) = serde_json::from_slice::<Value>(body) else {
let mut f = Features::new(TaskKind::Other);
f.hour_bucket = hour_bucket(jiff::Timestamp::now());
return f;
};
let tool_count = json
.get("tools")
.and_then(Value::as_array)
.map_or(0, |tools| u32::try_from(tools.len()).unwrap_or(u32::MAX));
let has_images = openai_messages_have_images(body);
let mut f = Features::new(TaskKind::Other);
f.agent = header_str(headers, AGENT_HEADER);
f.subagent = header_str(headers, SUBAGENT_HEADER);
f.tool_count = tool_count;
f.has_images = has_images;
f.prompt_token_bucket = token_bucket(body.len() as u64);
f.hour_bucket = hour_bucket(jiff::Timestamp::now());
f.difficulty_hint = DifficultyHint::score(trajectory_signals(body)).as_u8();
f
}
fn parse_data_url(url: &str) -> Option<(&str, &str)> {
let rest = url.strip_prefix("data:")?;
let (meta, data) = rest.split_once(',')?;
let media_type = meta.strip_suffix(";base64")?;
Some((media_type, data))
}
fn translate_openai_user_content(content: &Value) -> Option<Value> {
match content {
Value::String(_) => Some(content.clone()),
Value::Array(parts) => {
let mut blocks: Vec<Value> = Vec::with_capacity(parts.len());
for part in parts {
match part.get("type").and_then(Value::as_str) {
Some("text") => {
let text = part.get("text").and_then(Value::as_str).unwrap_or("");
blocks.push(serde_json::json!({ "type": "text", "text": text }));
}
Some("image_url") => {
let url = part.pointer("/image_url/url").and_then(Value::as_str)?;
if url.starts_with("http://") || url.starts_with("https://") {
return None; }
let (media_type, data) = parse_data_url(url)?;
blocks.push(serde_json::json!({
"type": "image",
"source": { "type": "base64", "media_type": media_type, "data": data }
}));
}
_ => {} }
}
Some(Value::Array(blocks))
}
_ => Some(Value::String(String::new())),
}
}
fn translate_openai_tools(tools: &Value) -> Value {
let Some(arr) = tools.as_array() else {
return Value::Null;
};
let converted: Vec<Value> = arr
.iter()
.map(|tool| {
let func = tool.get("function").unwrap_or(&Value::Null);
let mut out = serde_json::json!({
"name": func.get("name").cloned().unwrap_or(Value::String(String::new())),
"input_schema": func.get("parameters").cloned()
.unwrap_or_else(|| serde_json::json!({ "type": "object" })),
});
if let Some(desc) = func.get("description") {
out["description"] = desc.clone();
}
out
})
.collect();
Value::Array(converted)
}
fn translate_openai_tool_choice(tc: &Value) -> Value {
match tc {
Value::String(s) => match s.as_str() {
"auto" => serde_json::json!({ "type": "auto" }),
"required" => serde_json::json!({ "type": "any" }),
_ => serde_json::json!({ "type": "auto" }),
},
Value::Object(_) => {
if tc.get("type").and_then(Value::as_str) == Some("function") {
let name = tc.pointer("/function/name").cloned().unwrap_or(Value::Null);
serde_json::json!({ "type": "tool", "name": name })
} else {
serde_json::json!({ "type": "auto" })
}
}
_ => serde_json::json!({ "type": "auto" }),
}
}
pub fn parse_openai_request(body: &[u8], carry_raw: bool) -> Option<ModelRequest> {
let json: Value = serde_json::from_slice(body).ok()?;
let raw = if carry_raw { json.clone() } else { Value::Null };
let messages_json = json.get("messages")?.as_array()?;
let mut system: Option<String> = None;
let mut messages: Vec<ChatMessage> = Vec::with_capacity(messages_json.len());
let mut tools = Value::Null;
let mut tool_choice_override: Option<Value> = None;
for msg in messages_json {
let role = msg.get("role").and_then(Value::as_str).unwrap_or("user");
match role {
"system" => {
if let Some(s) = msg.get("content").and_then(Value::as_str) {
system = Some(s.to_owned());
}
}
"user" => {
let content_val = msg.get("content").unwrap_or(&Value::Null);
let translated = translate_openai_user_content(content_val)?;
messages.push(ChatMessage {
role: "user".to_owned(),
content: translated,
});
}
"assistant" => {
if let Some(tc_arr) = msg.get("tool_calls").and_then(Value::as_array) {
let mut blocks: Vec<Value> = Vec::new();
if let Some(text) = msg.get("content").and_then(Value::as_str)
&& !text.is_empty()
{
blocks.push(serde_json::json!({ "type": "text", "text": text }));
}
for tc in tc_arr {
let id = tc.get("id").and_then(Value::as_str).unwrap_or("");
let func = tc.get("function").unwrap_or(&Value::Null);
let name = func.get("name").and_then(Value::as_str).unwrap_or("");
let args_str = func
.get("arguments")
.and_then(Value::as_str)
.unwrap_or("{}");
let input: Value = serde_json::from_str(args_str)
.unwrap_or_else(|_| serde_json::json!({}));
blocks.push(serde_json::json!({
"type": "tool_use",
"id": id,
"name": name,
"input": input,
}));
}
messages.push(ChatMessage {
role: "assistant".to_owned(),
content: Value::Array(blocks),
});
} else {
let content = msg
.get("content")
.cloned()
.unwrap_or_else(|| Value::String(String::new()));
messages.push(ChatMessage {
role: "assistant".to_owned(),
content,
});
}
}
"tool" => {
let tool_call_id = msg
.get("tool_call_id")
.and_then(Value::as_str)
.unwrap_or("");
let content = msg
.get("content")
.cloned()
.unwrap_or_else(|| Value::String(String::new()));
let result_block = serde_json::json!({
"type": "tool_result",
"tool_use_id": tool_call_id,
"content": content,
});
messages.push(ChatMessage {
role: "user".to_owned(),
content: Value::Array(vec![result_block]),
});
}
_ => {} }
}
if !carry_raw {
if let Some(t) = json.get("tools") {
tools = translate_openai_tools(t);
}
if let Some(tc) = json.get("tool_choice") {
tool_choice_override = Some(translate_openai_tool_choice(tc));
}
} else {
tools = json.get("tools").cloned().unwrap_or(Value::Null);
}
let max_tokens = json
.get("max_tokens")
.and_then(Value::as_u64)
.and_then(|n| u32::try_from(n).ok())
.unwrap_or(1024);
let model = json
.get("model")
.and_then(Value::as_str)
.unwrap_or_default()
.to_owned();
let _ = tool_choice_override;
Some(ModelRequest {
model,
system,
messages,
max_tokens,
tools,
raw,
cache_prefix: false,
})
}
fn extract_openai_content_and_tools(raw: &Value, text: &str) -> (Value, Option<Value>) {
if let Some(blocks) = raw.get("content").and_then(Value::as_array) {
let mut text_parts: Vec<&str> = Vec::new();
let mut tool_calls: Vec<Value> = Vec::new();
for block in blocks {
match block.get("type").and_then(Value::as_str) {
Some("text") => {
if let Some(t) = block.get("text").and_then(Value::as_str) {
text_parts.push(t);
}
}
Some("tool_use") => {
let id = block.get("id").and_then(Value::as_str).unwrap_or("");
let name = block.get("name").and_then(Value::as_str).unwrap_or("");
let input_str = block
.get("input")
.map_or_else(|| "{}".to_owned(), std::string::ToString::to_string);
tool_calls.push(serde_json::json!({
"id": id,
"type": "function",
"function": { "name": name, "arguments": input_str },
}));
}
_ => {}
}
}
let content_text = if tool_calls.is_empty() || !text_parts.is_empty() {
Value::String(text_parts.join(""))
} else {
Value::Null };
let tc = if tool_calls.is_empty() {
None
} else {
Some(Value::Array(tool_calls))
};
return (content_text, tc);
}
if let Some(msg) = raw.pointer("/choices/0/message") {
let content = msg
.get("content")
.cloned()
.unwrap_or(Value::String(text.to_owned()));
let tc = msg.get("tool_calls").cloned();
return (content, tc);
}
(Value::String(text.to_owned()), None)
}
fn openai_response_json(resp: &ModelResponse) -> Value {
let (content_text, tool_calls) = extract_openai_content_and_tools(&resp.raw, &resp.text);
let finish_reason = if resp.truncated() {
"length"
} else if tool_calls.is_some() {
"tool_calls"
} else {
"stop"
};
let mut message = serde_json::json!({
"role": "assistant",
"content": content_text,
});
if let Some(tc) = tool_calls {
message["tool_calls"] = tc;
}
serde_json::json!({
"id": format!("chatcmpl-{}", Uuid::now_v7()),
"object": "chat.completion",
"created": jiff::Timestamp::now().as_second(),
"model": resp.model,
"choices": [{
"index": 0,
"message": message,
"finish_reason": finish_reason,
}],
"usage": {
"prompt_tokens": resp.in_tokens,
"completion_tokens": resp.out_tokens,
"total_tokens": resp.in_tokens + resp.out_tokens,
}
})
}
fn stream_options_include_usage(body: &[u8]) -> bool {
serde_json::from_slice::<Value>(body)
.ok()
.and_then(|v| v.pointer("/stream_options/include_usage").cloned())
.and_then(|v| v.as_bool())
.unwrap_or(false)
}
fn openai_sse_from_message(message: &Value, include_usage: bool) -> String {
let id = message
.get("id")
.and_then(Value::as_str)
.unwrap_or("chatcmpl-unknown")
.to_owned();
let created = message.get("created").cloned().unwrap_or(Value::from(0));
let model = message
.get("model")
.and_then(Value::as_str)
.unwrap_or("unknown");
let choices = message.get("choices").and_then(Value::as_array);
let msg = choices
.and_then(|c| c.first())
.and_then(|c| c.get("message"));
let content = msg
.and_then(|m| m.get("content"))
.cloned()
.unwrap_or(Value::Null);
let tool_calls = msg.and_then(|m| m.get("tool_calls")).cloned();
let finish_reason = choices
.and_then(|c| c.first())
.and_then(|c| c.get("finish_reason"))
.cloned()
.unwrap_or_else(|| Value::String("stop".to_owned()));
let mut out = String::new();
let chunk = |delta: Value| {
serde_json::json!({
"id": id,
"object": "chat.completion.chunk",
"created": created,
"model": model,
"choices": [{ "index": 0, "delta": delta, "finish_reason": Value::Null }]
})
};
let role_chunk = chunk(serde_json::json!({ "role": "assistant", "content": "" }));
out.push_str("data: ");
out.push_str(&role_chunk.to_string());
out.push_str("\n\n");
if let Value::String(text) = &content
&& !text.is_empty()
{
let content_chunk = chunk(serde_json::json!({ "content": text }));
out.push_str("data: ");
out.push_str(&content_chunk.to_string());
out.push_str("\n\n");
}
if let Some(tc) = tool_calls {
let tc_with_index: Vec<Value> = tc
.as_array()
.map(|arr| {
arr.iter()
.enumerate()
.map(|(idx, call)| {
let mut c = call.clone();
c["index"] = Value::from(idx);
c
})
.collect()
})
.unwrap_or_default();
let tc_chunk = chunk(serde_json::json!({ "tool_calls": tc_with_index }));
out.push_str("data: ");
out.push_str(&tc_chunk.to_string());
out.push_str("\n\n");
}
let finish_chunk = serde_json::json!({
"id": id,
"object": "chat.completion.chunk",
"created": created,
"model": model,
"choices": [{ "index": 0, "delta": {}, "finish_reason": finish_reason }]
});
out.push_str("data: ");
out.push_str(&finish_chunk.to_string());
out.push_str("\n\n");
if include_usage && let Some(usage) = message.get("usage") {
let usage_chunk = serde_json::json!({
"id": id,
"object": "chat.completion.chunk",
"created": created,
"model": model,
"choices": [],
"usage": usage
});
out.push_str("data: ");
out.push_str(&usage_chunk.to_string());
out.push_str("\n\n");
}
out.push_str("data: [DONE]\n\n");
out
}
#[allow(clippy::too_many_arguments)]
async fn handle_enforce_openai(
state: &AppState,
headers: &HeaderMap,
body: &Bytes,
features: Features,
route: &Route,
route_ix: usize,
session_header: Option<String>,
tenant: String,
routing_mode: RoutingMode,
) -> Response {
if is_stream_request(body) {
let include_usage = stream_options_include_usage(body);
let (tx, rx) = tokio::sync::oneshot::channel::<Result<Value, ProxyError>>();
let (state_c, headers_c, body_c, route_c) =
(state.clone(), headers.clone(), body.clone(), route.clone());
tokio::spawn(async move {
let out = enforce_pipeline_openai(
&state_c,
&headers_c,
&body_c,
features,
&route_c,
route_ix,
session_header,
tenant,
routing_mode,
)
.await;
let _ = tx.send(out);
});
return sse_keepalive_response(
rx,
Box::new(move |v| openai_sse_from_message(v, include_usage)),
sse_error_event,
);
}
match enforce_pipeline_openai(
state,
headers,
body,
features,
route,
route_ix,
session_header,
tenant,
routing_mode,
)
.await
{
Ok(message) => (axum::http::StatusCode::OK, Json(message)).into_response(),
Err(e) => e.into_response(),
}
}
async fn observe_passthrough_openai(
state: AppState,
headers: HeaderMap,
body: Bytes,
session_header: Option<String>,
tenant: String,
) -> Response {
observe_passthrough_openai_path(
state,
headers,
body,
session_header,
tenant,
crate::upstream::OPENAI_CHAT_PATH,
)
.await
}
async fn observe_passthrough_openai_path(
state: AppState,
headers: HeaderMap,
body: Bytes,
session_header: Option<String>,
tenant: String,
path: &'static str,
) -> Response {
if is_stream_request(&body) {
return observe_stream_openai_path(state, headers, body, session_header, tenant, path)
.await;
}
let start = Instant::now();
let result = crate::upstream::forward_openai_path(
&state.http,
&state.config.upstream_openai,
path,
&headers,
body.clone(),
)
.await;
let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
match result {
Ok((status, resp_headers, resp_body)) => {
let for_trace = status.is_success().then_some(resp_body.clone());
spawn_trace(
&state,
body,
for_trace,
latency_ms,
session_header,
tenant,
"openai",
"openai.chat_completions",
None,
);
(status, resp_headers, resp_body).into_response()
}
Err(err) => {
spawn_trace(
&state,
body,
None,
latency_ms,
session_header,
tenant,
"openai",
"openai.chat_completions",
None,
);
err.into_response()
}
}
}
async fn observe_stream_openai_path(
state: AppState,
headers: HeaderMap,
body: Bytes,
session_header: Option<String>,
tenant: String,
path: &'static str,
) -> Response {
let start = Instant::now();
let result = crate::upstream::forward_openai_streaming_path(
&state.http,
&state.config.upstream_openai,
path,
&headers,
body.clone(),
)
.await;
let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
match result {
Ok((status, resp_headers, response)) => {
let on_done = stream_trace_callback(
&state,
body,
latency_ms,
session_header,
tenant,
"openai",
status.is_success(),
"openai.chat_completions",
None,
);
let tap = SseTap::new(response.bytes_stream(), on_done);
let stream_body = Body::from_stream(tap);
(status, resp_headers, stream_body).into_response()
}
Err(err) => {
spawn_trace(
&state,
body,
None,
latency_ms,
session_header,
tenant,
"openai",
"openai.chat_completions",
None,
);
err.into_response()
}
}
}
async fn chat_completions(
State(state): State<AppState>,
Extension(TenantId(tenant)): Extension<TenantId>,
headers: HeaderMap,
body: Bytes,
) -> Response {
let mut session_header = header_session(&headers, &state.config.prompt_salt);
if let Some(routing) = state.config.routing.as_ref() {
let features = extract_openai_features(&headers, &body);
if let Some(route) = routing
.route_for(&features)
.filter(|r| r.mode == Mode::Enforce && !r.ladder.is_empty())
{
session_header = session_header
.take()
.or_else(|| body_session_bytes(&body, &state.config.prompt_salt));
let route_ix = routing
.routes
.iter()
.position(|r| std::ptr::eq(r, route))
.unwrap_or(0);
let route = route.clone();
let routing_mode = resolve_mode(&headers, &route, &state.config);
if routing_mode == RoutingMode::Observe {
return observe_passthrough_openai(state, headers, body, session_header, tenant)
.await;
}
if enforce_can_handle_route(
&features,
&body,
routing.escalation.enforce_structured,
&route,
&state.providers,
Dialect::Openai,
) {
return handle_enforce_openai(
&state,
&headers,
&body,
features,
&route,
route_ix,
session_header,
tenant,
routing_mode,
)
.await;
}
tracing::info!(
"enforce route matched but OpenAI structured request can't be routed faithfully (flag/ladder); serving via observe passthrough"
);
}
}
observe_passthrough_openai(state, headers, body, session_header, tenant).await
}
async fn responses(
State(state): State<AppState>,
Extension(TenantId(tenant)): Extension<TenantId>,
headers: HeaderMap,
body: Bytes,
) -> Response {
let mut session_header = header_session(&headers, &state.config.prompt_salt);
let Ok(parsed) = serde_json::from_slice::<Value>(&body) else {
return observe_passthrough_openai_path(
state,
headers,
body,
session_header,
tenant,
crate::upstream::OPENAI_RESPONSES_PATH,
)
.await;
};
if crate::responses::has_untranslatable_content(&parsed) {
return observe_passthrough_openai_path(
state,
headers,
body,
session_header,
tenant,
crate::upstream::OPENAI_RESPONSES_PATH,
)
.await;
}
let chat_body = Bytes::from(crate::responses::request_to_chat(&parsed).to_string());
if let Some(routing) = state.config.routing.as_ref() {
let features = extract_openai_features(&headers, &chat_body);
if let Some(route) = routing
.route_for(&features)
.filter(|r| r.mode == Mode::Enforce && !r.ladder.is_empty())
{
session_header = session_header
.take()
.or_else(|| body_session(&parsed, &state.config.prompt_salt));
let route_ix = routing
.routes
.iter()
.position(|r| std::ptr::eq(r, route))
.unwrap_or(0);
let route = route.clone();
let routing_mode = resolve_mode(&headers, &route, &state.config);
if routing_mode != RoutingMode::Observe
&& enforce_can_handle_route(
&features,
&chat_body,
routing.escalation.enforce_structured,
&route,
&state.providers,
Dialect::Openai,
)
{
if is_stream_request(&body) {
let (tx, rx) = tokio::sync::oneshot::channel::<Result<Value, ProxyError>>();
tokio::spawn(async move {
let out = enforce_pipeline_openai_as(
&state,
&headers,
&chat_body,
&body,
"openai.responses",
features,
&route,
route_ix,
session_header,
tenant,
routing_mode,
)
.await;
let _ = tx.send(out);
});
return sse_keepalive_response(
rx,
Box::new(|chat| {
crate::responses::sse_from_response(
&crate::responses::response_from_chat(chat),
)
}),
responses_sse_error,
);
}
return match enforce_pipeline_openai_as(
&state,
&headers,
&chat_body,
&body,
"openai.responses",
features,
&route,
route_ix,
session_header,
tenant,
routing_mode,
)
.await
{
Ok(chat) => Json(crate::responses::response_from_chat(&chat)).into_response(),
Err(e) => e.into_response(),
};
}
}
}
observe_passthrough_openai_path(
state,
headers,
body,
session_header,
tenant,
crate::upstream::OPENAI_RESPONSES_PATH,
)
.await
}
fn responses_sse_error(e: &ProxyError) -> String {
crate::responses::sse_failed(&format!("resp_{}", Uuid::now_v7()), &e.client_message())
}
fn parse_gemini_query(raw: Option<&str>) -> (Option<String>, String) {
let Some(q) = raw.filter(|s| !s.is_empty()) else {
return (None, String::new());
};
let mut key_val: Option<String> = None;
let scrubbed: Vec<&str> = q
.split('&')
.filter(|pair| {
if let Some(v) = pair.strip_prefix("key=") {
if !v.is_empty() {
key_val = Some(percent_decode_ascii(v));
}
false } else {
!pair.is_empty()
}
})
.collect();
(key_val, scrubbed.join("&"))
}
fn percent_decode_ascii(s: &str) -> String {
let bytes = s.as_bytes();
let mut out: Vec<u8> = Vec::with_capacity(bytes.len());
let mut i = 0;
while i < bytes.len() {
if bytes[i] == b'%'
&& i + 2 < bytes.len()
&& let (Some(hi), Some(lo)) = (hex_val(bytes[i + 1]), hex_val(bytes[i + 2]))
{
out.push((hi << 4) | lo);
i += 3;
continue;
}
out.push(bytes[i]);
i += 1;
}
String::from_utf8(out).unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned())
}
fn hex_val(b: u8) -> Option<u8> {
match b {
b'0'..=b'9' => Some(b - b'0'),
b'a'..=b'f' => Some(b - b'a' + 10),
b'A'..=b'F' => Some(b - b'A' + 10),
_ => None,
}
}
async fn gemini_generate_content(
State(state): State<AppState>,
Extension(TenantId(tenant)): Extension<TenantId>,
axum::extract::Path(model_action): axum::extract::Path<String>,
axum::extract::RawQuery(raw_query): axum::extract::RawQuery,
headers: HeaderMap,
body: Bytes,
) -> Response {
let (key_from_query, scrubbed_query) = parse_gemini_query(raw_query.as_deref());
let headers: HeaderMap = if let (Some(k), None) = (
&key_from_query,
headers.get("x-goog-api-key").filter(|v| !v.is_empty()),
) {
match axum::http::HeaderValue::from_str(k) {
Ok(v) => {
let mut h = headers;
h.insert("x-goog-api-key", v);
h
}
Err(_) => headers,
}
} else {
headers
};
let (model, action) = model_action
.rsplit_once(':')
.unwrap_or((model_action.as_str(), ""));
let model_owned = model.to_owned();
if !matches!(action, "generateContent" | "streamGenerateContent") {
let path = if scrubbed_query.is_empty() {
format!("/v1beta/models/{model_action}")
} else {
format!("/v1beta/models/{model_action}?{scrubbed_query}")
};
let result = crate::upstream::forward_gemini_path(
&state.http,
&state.config.upstream_gemini,
&path,
&headers,
body,
)
.await;
return match result {
Ok((status, resp_headers, resp_body)) => {
(status, resp_headers, resp_body).into_response()
}
Err(err) => err.into_response(),
};
}
let is_streaming = action == "streamGenerateContent";
let mut session_header = header_session(&headers, &state.config.prompt_salt);
let Ok(parsed) = serde_json::from_slice::<Value>(&body) else {
return observe_passthrough_gemini(
state,
headers,
body,
session_header,
tenant,
model_action,
is_streaming,
scrubbed_query,
Some(model_owned),
)
.await;
};
if crate::gemini::has_untranslatable_content(&parsed) {
return observe_passthrough_gemini(
state,
headers,
body,
session_header,
tenant,
model_action,
is_streaming,
scrubbed_query,
Some(model_owned),
)
.await;
}
let chat_body = Bytes::from(crate::gemini::request_to_chat(&parsed, model).to_string());
if let Some(routing) = state.config.routing.as_ref() {
let features = extract_openai_features(&headers, &chat_body);
if let Some(route) = routing
.route_for(&features)
.filter(|r| r.mode == Mode::Enforce && !r.ladder.is_empty())
{
session_header = session_header
.take()
.or_else(|| body_session_bytes(&chat_body, &state.config.prompt_salt));
let route_ix = routing
.routes
.iter()
.position(|r| std::ptr::eq(r, route))
.unwrap_or(0);
let route = route.clone();
let routing_mode = resolve_mode(&headers, &route, &state.config);
if routing_mode == RoutingMode::Observe {
return observe_passthrough_gemini(
state,
headers,
body,
session_header,
tenant,
model_action,
is_streaming,
scrubbed_query,
Some(model_owned.clone()),
)
.await;
}
if state
.guardrails
.is_demoted(&tenant, route_ix, jiff::Timestamp::now().as_second())
{
return observe_passthrough_gemini(
state,
headers,
body,
session_header,
tenant,
model_action,
is_streaming,
scrubbed_query,
Some(model_owned.clone()),
)
.await;
}
if let Some(rollout) = route.rollout.as_ref() {
let key_value = match rollout.key {
firstpass_core::RolloutKey::Session => session_header
.clone()
.unwrap_or_else(|| firstpass_core::rollout::request_identity(&body)),
firstpass_core::RolloutKey::Request => {
firstpass_core::rollout::request_identity(&body)
}
firstpass_core::RolloutKey::Tenant => tenant.to_string(),
};
let decision =
firstpass_core::rollout::decide(&state.config.prompt_salt, rollout, &key_value);
if !decision.enforced {
return observe_passthrough_gemini(
state,
headers,
body,
session_header,
tenant,
model_action,
is_streaming,
scrubbed_query,
Some(model_owned.clone()),
)
.await;
}
}
if enforce_can_handle_route(
&features,
&chat_body,
routing.escalation.enforce_structured,
&route,
&state.providers,
Dialect::Openai,
) {
if is_streaming {
let (tx, rx) = tokio::sync::oneshot::channel::<Result<Value, ProxyError>>();
tokio::spawn(async move {
let out = enforce_pipeline_gemini_as(
&state,
&headers,
&chat_body,
&body,
"gemini.generate_content",
features,
&route,
route_ix,
session_header,
tenant,
routing_mode,
)
.await;
let _ = tx.send(out);
});
return sse_keepalive_response(
rx,
Box::new(|chat| {
crate::gemini::sse_from_response(&crate::gemini::response_from_chat(
chat,
))
}),
gemini_sse_error,
);
}
return match enforce_pipeline_gemini_as(
&state,
&headers,
&chat_body,
&body,
"gemini.generate_content",
features,
&route,
route_ix,
session_header,
tenant,
routing_mode,
)
.await
{
Ok(chat) => Json(crate::gemini::response_from_chat(&chat)).into_response(),
Err(e) => e.into_response(),
};
}
}
}
observe_passthrough_gemini(
state,
headers,
body,
session_header,
tenant,
model_action,
is_streaming,
scrubbed_query,
Some(model_owned),
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn observe_passthrough_gemini(
state: AppState,
headers: HeaderMap,
body: Bytes,
session_header: Option<String>,
tenant: String,
model_action: String,
is_streaming: bool,
scrubbed_query: String,
model_hint: Option<String>,
) -> Response {
let path = if scrubbed_query.is_empty() {
format!("/v1beta/models/{model_action}")
} else {
format!("/v1beta/models/{model_action}?{scrubbed_query}")
};
if is_streaming {
let start = Instant::now();
let result = crate::upstream::forward_gemini_streaming_path(
&state.http,
&state.config.upstream_gemini,
&path,
&headers,
body.clone(),
)
.await;
let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
match result {
Ok((status, resp_headers, response)) => {
let on_done = stream_trace_callback(
&state,
body,
latency_ms,
session_header,
tenant,
"google", status.is_success(),
"gemini.generate_content",
model_hint,
);
let tap = SseTap::new(response.bytes_stream(), on_done);
let stream_body = Body::from_stream(tap);
return (status, resp_headers, stream_body).into_response();
}
Err(err) => {
spawn_trace(
&state,
body,
None,
latency_ms,
session_header,
tenant,
"google", "gemini.generate_content",
model_hint,
);
return err.into_response();
}
}
}
let start = Instant::now();
let result = crate::upstream::forward_gemini_path(
&state.http,
&state.config.upstream_gemini,
&path,
&headers,
body.clone(),
)
.await;
let latency_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
match result {
Ok((status, resp_headers, resp_body)) => {
let for_trace = status.is_success().then_some(resp_body.clone());
spawn_trace(
&state,
body,
for_trace,
latency_ms,
session_header,
tenant,
"google", "gemini.generate_content",
model_hint,
);
(status, resp_headers, resp_body).into_response()
}
Err(err) => {
spawn_trace(
&state,
body,
None,
latency_ms,
session_header,
tenant,
"google", "gemini.generate_content",
model_hint,
);
err.into_response()
}
}
}
#[allow(clippy::too_many_arguments)]
async fn enforce_pipeline_gemini_as(
state: &AppState,
headers: &HeaderMap,
chat_body: &Bytes,
receipt_body: &Bytes,
api: &str,
features: Features,
route: &Route,
route_ix: usize,
session_header: Option<String>,
tenant: String,
routing_mode: RoutingMode,
) -> Result<Value, ProxyError> {
let providers = &state.providers;
let all_openai_of = |ladder: &[String]| {
ladder.iter().all(|rung| {
let pid = rung.split('/').next().unwrap_or_default();
providers
.get(pid)
.is_some_and(|p| p.carries_structured_verbatim(Dialect::Openai))
})
};
let all_openai = all_openai_of(&route.ladder);
let allow_challenger = route
.challenger
.as_ref()
.is_none_or(|c| all_openai_of(&c.ladder) == all_openai);
let Some(base_request) = parse_openai_request(chat_body, all_openai) else {
return Err(ProxyError::BadRequest(
"Gemini request could not be translated to a valid Chat Completions request".to_owned(),
));
};
let auth = Auth::from_headers(headers, Dialect::Gemini);
let resp = enforce_pipeline_inner(
state,
receipt_body,
base_request,
auth,
features,
route,
session_header,
tenant,
api,
routing_mode,
route_ix,
allow_challenger,
)
.await?;
Ok(openai_response_json(&resp))
}
fn gemini_sse_error(e: &ProxyError) -> String {
crate::gemini::sse_error_response(&e.client_message())
}
#[cfg(test)]
mod tests {
use bytes::Bytes;
use serde_json::json;
fn sess(h: &[(&str, &str)], body: serde_json::Value) -> Option<String> {
let mut headers = HeaderMap::new();
for (k, v) in h {
headers.insert(
axum::http::HeaderName::from_bytes(k.as_bytes()).unwrap(),
v.parse().unwrap(),
);
}
header_session(&headers, "salt")
.or_else(|| body_session_bytes(body.to_string().as_bytes(), "salt"))
}
#[test]
fn caller_session_priority_order() {
let body = json!({
"metadata": {"user_id": "u_session_abc"}, "prompt_cache_key": "k",
"tools": [{"name": "t"}], "messages": [{"role": "user", "content": "hi"}],
});
let all = [("x-firstpass-session", "explicit"), ("session_id", "ag")];
assert_eq!(sess(&all, body.clone()).as_deref(), Some("explicit"));
let agent = sess(&all[1..], body.clone()).unwrap();
assert!(agent.starts_with("agent:"));
let meta = sess(&[], body.clone()).unwrap();
assert!(meta.starts_with("meta:"));
let mut b = body.clone();
b["metadata"] = json!({"user_id": "plain-user"});
let pck = sess(&[], b.clone()).unwrap();
assert!(pck.starts_with("pck:"));
b.as_object_mut().unwrap().remove("prompt_cache_key");
assert!(sess(&[], b).unwrap().starts_with("conv:"));
for (id, raw) in [(&agent, "ag"), (&meta, "u_session_abc"), (&pck, "k")] {
assert_eq!(id.len(), id.find(':').unwrap() + 1 + 16, "{id}");
assert!(
!id[id.find(':').unwrap() + 1..].contains(raw),
"{id} leaks {raw}"
);
}
let plain =
json!({"prompt_cache_key": "k", "messages": [{"role": "user", "content": "hi"}]});
assert_eq!(sess(&[], plain), None);
}
#[test]
fn caller_session_caps_and_trims_header_values() {
let long = "a".repeat(300);
assert_eq!(
sess(&[("conversation_id", &long)], json!({}))
.unwrap()
.len(),
"agent:".len() + 16
);
let v = sess(&[("x-firstpass-session", &long)], json!({})).unwrap();
assert_eq!(v.len(), 256, "explicit header is verbatim but capped");
assert_eq!(cap_session(&"é".repeat(300)).len(), 256); assert_eq!(sess(&[("session_id", " ")], json!({})), None);
}
#[test]
fn blank_explicit_header_is_absent() {
let body =
json!({"tools": [{"name": "t"}], "messages": [{"role": "user", "content": "x"}]});
for blank in ["", " "] {
let id = sess(&[("x-firstpass-session", blank)], body.clone()).unwrap();
assert!(
id.starts_with("conv:"),
"blank header must fall through, got {id:?}"
);
}
assert_eq!(
sess(&[("x-firstpass-session", " a ")], json!({})).as_deref(),
Some("a")
);
}
#[test]
fn responses_string_input_turn_one_matches_array_turn_two() {
let tools = json!([{"type": "function", "name": "shell"}]);
let t1 = json!({"instructions": "i", "tools": tools, "input": "list files"});
let t2 = json!({"instructions": "i", "tools": tools, "input": [
{"role": "user", "content": [{"type": "input_text", "text": "list files"}]},
{"type": "function_call_output", "call_id": "c", "output": "a b"}]});
let a = sess(&[], t1).unwrap();
assert!(a.starts_with("conv:"));
assert_eq!(Some(a), sess(&[], t2));
let p = json!({"input": "more", "previous_response_id": "resp_1"});
assert!(sess(&[], p).unwrap().starts_with("prev:"));
}
#[test]
fn agent_turn_one_and_two_share_a_conv_id_and_tool_names_split_it() {
let u1 = json!({"role": "user", "content": "fix it"});
let turn1 =
json!({"system": "s", "tools": [{"name": "bash"}, {"name": "edit"}], "messages": [u1]});
let turn2 = json!({"system": "s", "tools": [{"name": "edit"}, {"name": "bash"}], "messages": [
u1, {"role": "assistant", "content": "ok"}, {"role": "user", "content": "go on"}]});
let a = sess(&[], turn1.clone()).unwrap();
assert_eq!(
Some(a.clone()),
sess(&[], turn2),
"turn 1 and 2 group, tool order irrelevant"
);
let mut other = turn1;
other["tools"] = json!([{"name": "bash"}, {"name": "grep"}]);
assert_ne!(
Some(a),
sess(&[], other),
"different toolset, different session"
);
}
#[tokio::test]
async fn observe_only_derives_session_from_headers_never_the_body() {
let app = axum::Router::new().route(
"/v1/messages",
axum::routing::post(|| async { (axum::http::StatusCode::OK, "{}") }),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let base_url = format!("http://{}", listener.local_addr().unwrap());
tokio::spawn(async move {
let _ = axum::serve(listener, app).await;
});
let config = ProxyConfig::from_lookup(|k| match k {
"FIRSTPASS_UPSTREAM_ANTHROPIC" => Some(base_url.clone()),
_ => None,
})
.unwrap();
let (traces, mut rx) = mpsc::channel(4);
let state = AppState {
config: Arc::new(config),
http: reqwest::Client::new(),
providers: ProviderRegistry::from_map(HashMap::new()),
gate_health: Arc::new(GateHealthRegistry::new()),
shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
experiments: Arc::default(),
guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
traces,
adaptive: None,
eprocess: None,
bandit: None,
promoter: None,
verified_cache: None,
predictor: None,
prior: None,
tenant_rate_limiter: None,
spill: None,
};
let body = Bytes::from_static(
br#"{"model":"claude-haiku-4-5","tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
);
let mut got = Vec::new();
for hdr in [None, Some("abc")] {
let mut headers = HeaderMap::new();
if let Some(h) = hdr {
headers.insert("session_id", h.parse().unwrap());
}
let _ = messages(
State(state.clone()),
Extension(TenantId("default".to_owned())),
headers,
body.clone(),
)
.await;
got.push(rx.recv().await.unwrap());
}
assert!(
!got[0].session_id.starts_with("conv:"),
"{}",
got[0].session_id
);
assert!(
got[1].session_id.starts_with("agent:"),
"{}",
got[1].session_id
);
}
#[test]
fn caller_session_anthropic_conversation_is_stable_across_turns() {
let u1 = json!({"role": "user", "content": "fix the bug"});
let t1 = json!({"system": "sys", "tools": [{"name": "t"}], "messages": [u1]});
let t2 = json!({"system": "sys", "tools": [{"name": "t"}], "messages": [
u1, {"role": "assistant", "content": "ok"}, {"role": "user", "content": "more"}]});
let other = json!({"system": "sys", "tools": [{"name": "t"}],
"messages": [{"role": "user", "content": "different task"}]});
let a = sess(&[], t1).unwrap();
assert!(a.starts_with("conv:") && a.len() == 5 + 16);
assert_eq!(Some(a.clone()), sess(&[], t2));
assert_ne!(Some(a), sess(&[], other));
let plain =
json!({"metadata": {"user_id": "bob"}, "messages": [{"role": "user", "content": "x"}]});
assert_eq!(sess(&[], plain), None);
assert_eq!(sess(&[], json!("not an object")), None);
}
#[test]
fn caller_session_openai_chat_and_responses_shapes() {
let chat = |tail: &str| {
json!({"messages": [
{"role": "system", "content": "s"}, {"role": "user", "content": "q"},
{"role": "assistant", "content": "a"}, {"role": "user", "content": tail}]})
};
let c = sess(&[], chat("one")).unwrap();
assert_eq!(Some(c.clone()), sess(&[], chat("two")));
let diff = json!({"messages": [
{"role": "system", "content": "s"}, {"role": "user", "content": "other"},
{"role": "assistant", "content": "a"}]});
assert_ne!(Some(c), sess(&[], diff));
assert_eq!(
sess(&[], json!({"messages": [{"role": "user", "content": "q"}]})),
None
);
let resp = |items: serde_json::Value| json!({"instructions": "i", "input": items});
let first = json!({"role": "user", "content": "q"});
let r1 = sess(
&[],
resp(json!([first, {"role": "user", "content": "more"}])),
)
.unwrap();
assert!(r1.starts_with("conv:"));
assert_eq!(
Some(r1.clone()),
sess(
&[],
resp(
json!([first, {"role": "assistant", "content": "x"}, {"role": "user", "content": "y"}])
)
)
);
assert_ne!(
Some(r1),
sess(&[], resp(json!([{"role": "user", "content": "z"}, first])))
);
assert_eq!(sess(&[], resp(json!("hello"))), None);
assert!(
sess(
&[],
json!({"input": "hello", "tools": [{"type": "function"}]})
)
.is_some()
);
}
#[test]
fn risk_certificate_follows_adr_0011_precedence() {
use super::{ThresholdSource, risk_certificate};
let c = firstpass_core::eprocess::Certification {
threshold: 0.6,
e_value: 30.0,
certified_at_round: 200,
};
let r = risk_certificate(Some((c, 0.1, 0.05)), Some((0.4, 0.1)), Some(0.9)).unwrap();
assert_eq!(r.source, ThresholdSource::Eprocess);
assert_eq!(
(r.threshold, r.e_value, r.delta, r.certified_at_round),
(0.6, Some(30.0), Some(0.05), Some(200))
);
let r = risk_certificate(None, Some((0.4, 0.1)), Some(0.9)).unwrap();
assert_eq!((r.source, r.threshold), (ThresholdSource::Aci, 0.4));
assert_eq!((r.alpha, r.e_value), (Some(0.1), None));
let r = risk_certificate(None, None, Some(0.9)).unwrap();
assert_eq!(
(r.source, r.threshold, r.alpha),
(ThresholdSource::Fixed, 0.9, None)
);
assert!(risk_certificate(None, None, None).is_none());
}
use super::*;
#[test]
fn anthropic_tool_errors_are_extracted() {
let body = br#"{"messages":[
{"role":"assistant","content":[{"type":"tool_use","name":"bash","input":{"cmd":"a"}}]},
{"role":"user","content":[{"type":"tool_result","is_error":true,"content":"boom"}]},
{"role":"assistant","content":[{"type":"tool_use","name":"bash","input":{"cmd":"b"}}]},
{"role":"user","content":[{"type":"tool_result","is_error":true,"content":"boom"}]}
]}"#;
let s = trajectory_signals(body);
assert_eq!(
s.tool_errors, 2,
"both failing tool_results must be counted"
);
assert_eq!(s.tool_results, 2);
assert_eq!(s.assistant_turns, 2);
assert!(
DifficultyHint::score(s) >= DifficultyHint::Medium,
"an all-failing window must not read as an easy session"
);
}
#[test]
fn openai_tool_errors_are_extracted_from_a_different_shape() {
let body = br#"{"messages":[
{"role":"assistant","tool_calls":[{"function":{"name":"run","arguments":"{\"x\":1}"}}]},
{"role":"tool","content":"Error: command not found"},
{"role":"assistant","tool_calls":[{"function":{"name":"run","arguments":"{\"x\":2}"}}]},
{"role":"tool","content":"Traceback (most recent call last): ..."}
]}"#;
let s = trajectory_signals(body);
assert_eq!(s.tool_results, 2, "role:tool messages are tool results");
assert_eq!(
s.tool_errors, 2,
"both error-shaped tool outputs must count"
);
assert_eq!(s.assistant_turns, 2);
}
#[test]
fn a_flood_of_leading_whitespace_does_not_defeat_the_bound() {
let flood = format!("{}Error: boom", " ".repeat(4_000_000));
let small = format!("{}Error: boom", " ".repeat(400_000));
let m_big = serde_json::json!({ "role": "tool", "content": flood });
let m_small = serde_json::json!({ "role": "tool", "content": small });
let t = |m: &serde_json::Value| {
let start = std::time::Instant::now();
for _ in 0..20 {
std::hint::black_box(openai_tool_content_looks_like_error(m));
}
start.elapsed()
};
let _warm = t(&m_small);
let small_t = t(&m_small);
let big_t = t(&m_big);
assert!(
big_t < small_t * 5 + std::time::Duration::from_millis(5),
"10x the leading whitespace took {big_t:?} vs {small_t:?} — trimming is scanning the \
whole payload before the bound applies"
);
let indented = serde_json::json!({ "role": "tool", "content": " Error: boom" });
assert!(
openai_tool_content_looks_like_error(&indented),
"ordinary leading whitespace must still be trimmed"
);
}
#[test]
fn a_long_real_conversation_can_actually_reach_the_deep_signal() {
let mut msgs = Vec::new();
for i in 0..40 {
msgs.push(format!(
r#"{{"role":"assistant","content":[{{"type":"tool_use","name":"b","input":{{"i":{i}}}}}]}}"#
));
msgs.push(
r#"{"role":"user","content":[{"type":"tool_result","is_error":true}]}"#.to_owned(),
);
}
let body = format!(r#"{{"messages":[{}]}}"#, msgs.join(","));
let s = trajectory_signals(body.as_bytes());
assert!(
s.assistant_turns >= 8,
"a 40-turn failing conversation must register as deep, got {} assistant turns — the \
window is counting turns it cannot see",
s.assistant_turns
);
assert_eq!(
DifficultyHint::score(s),
DifficultyHint::High,
"the hardest possible conversation must reach the top bucket, or High is dead code"
);
}
#[test]
fn openai_array_form_tool_content_is_read() {
let arr = serde_json::json!({
"role": "tool",
"content": [{"type": "text", "text": "Error: command not found"}]
});
assert!(
openai_tool_content_looks_like_error(&arr),
"an error in array-form content must be detected"
);
let clean = serde_json::json!({
"role": "tool",
"content": [{"type": "text", "text": "All 42 tests passed"}]
});
assert!(
!openai_tool_content_looks_like_error(&clean),
"clean array-form content must not be a failure"
);
for weird in [
serde_json::json!({"role": "tool", "content": []}),
serde_json::json!({"role": "tool", "content": [{"type": "image"}]}),
serde_json::json!({"role": "tool", "content": [null, 42]}),
] {
assert!(!openai_tool_content_looks_like_error(&weird));
}
let body = br#"{"messages":[
{"role":"tool","content":[{"type":"text","text":"Error: boom"}]},
{"role":"tool","content":[{"type":"text","text":"Error: boom again"}]},
{"role":"tool","content":[{"type":"text","text":"Error: still broken"}]}
]}"#;
let s = trajectory_signals(body);
assert_eq!(s.tool_errors, 3, "array-form errors must reach the signals");
assert!(DifficultyHint::score(s) >= DifficultyHint::Medium);
}
#[test]
fn tool_output_merely_mentioning_errors_is_not_counted_as_failure() {
let body = br#"{"messages":[
{"role":"tool","content":"0 errors, 0 warnings"},
{"role":"tool","content":"All 42 tests passed (no failures)"},
{"role":"tool","content":"lint: checked 12 files for errors"}
]}"#;
let s = trajectory_signals(body);
assert_eq!(s.tool_results, 3);
assert_eq!(
s.tool_errors, 0,
"clean output that mentions the word 'error' must not be scored as a failure"
);
}
#[test]
fn a_multibyte_character_does_not_defeat_the_prefix_bound() {
let padded = format!("{}é{}", "x".repeat(63), "y".repeat(200_000));
let msg = serde_json::json!({ "role": "tool", "content": padded });
assert!(
!openai_tool_content_looks_like_error(&msg),
"a huge non-error payload must not be classified as a failure"
);
let big = format!("{}é{}", "x".repeat(63), "y".repeat(2_000_000));
let small = format!("{}é{}", "x".repeat(63), "y".repeat(200_000));
let m_big = serde_json::json!({ "role": "tool", "content": big });
let m_small = serde_json::json!({ "role": "tool", "content": small });
let t = |m: &serde_json::Value| {
let start = std::time::Instant::now();
for _ in 0..20 {
std::hint::black_box(openai_tool_content_looks_like_error(m));
}
start.elapsed()
};
let _warm = t(&m_small);
let small_t = t(&m_small);
let big_t = t(&m_big);
assert!(
big_t < small_t * 5 + std::time::Duration::from_millis(5),
"scanning a 10x larger payload took {big_t:?} vs {small_t:?} — the prefix bound is not \
holding, so work scales with attacker-controlled input size"
);
let err = format!("Error: boom 💥{}", "z".repeat(200_000));
let msg2 = serde_json::json!({ "role": "tool", "content": err });
assert!(
openai_tool_content_looks_like_error(&msg2),
"bounding the scan must not cost the detection it exists to perform"
);
let late = format!("{}Error: too late", "w".repeat(200));
let msg3 = serde_json::json!({ "role": "tool", "content": late });
assert!(
!openai_tool_content_looks_like_error(&msg3),
"the scan must stay bounded to the head of the message"
);
}
#[test]
fn identical_repeated_tool_calls_are_detected() {
let body = br#"{"messages":[
{"role":"assistant","content":[{"type":"tool_use","name":"ls","input":{"p":"/x"}}]},
{"role":"assistant","content":[{"type":"tool_use","name":"ls","input":{"p":"/x"}}]},
{"role":"assistant","content":[{"type":"tool_use","name":"ls","input":{"p":"/x"}}]},
{"role":"user","content":[{"type":"tool_result","content":"ok"}]}
]}"#;
let s = trajectory_signals(body);
assert_eq!(
s.repeated_tool_calls, 2,
"three identical calls are two repeats"
);
let distinct = br#"{"messages":[
{"role":"assistant","content":[{"type":"tool_use","name":"ls","input":{"p":"/x"}}]},
{"role":"assistant","content":[{"type":"tool_use","name":"ls","input":{"p":"/y"}}]}
]}"#;
assert_eq!(trajectory_signals(distinct).repeated_tool_calls, 0);
}
#[test]
fn malformed_bodies_yield_no_signal_and_never_panic() {
for body in [
&b""[..],
b"not json at all",
b"{}",
br#"{"messages":"not-an-array"}"#,
br#"{"messages":[{"role":"assistant","content":"plain text"}]}"#,
br#"{"messages":[null,42,"x"]}"#,
] {
let s = trajectory_signals(body);
assert_eq!(
DifficultyHint::score(s),
DifficultyHint::None,
"a body with no usable trajectory must score None, not a difficulty level"
);
}
let quiet_success = br#"{"messages":[{"content":[{"type":"tool_result"}]}]}"#;
let s = trajectory_signals(quiet_success);
assert_eq!(s.tool_results, 1);
assert_eq!(s.tool_errors, 0);
assert_eq!(
DifficultyHint::score(s),
DifficultyHint::Low,
"a tool result that reported no error is evidence of health, not absence of evidence"
);
}
#[test]
fn only_the_recent_window_counts_so_old_failures_do_not_ratchet() {
let mut msgs = String::from("{\"messages\":[");
for _ in 0..30 {
msgs.push_str(r#"{"role":"user","content":[{"type":"tool_result","is_error":true}]},"#);
}
for i in 0..12 {
msgs.push_str(&format!(
r#"{{"role":"user","content":[{{"type":"tool_result","is_error":false,"content":"ok{i}"}}]}}{}"#,
if i == 11 { "" } else { "," }
));
}
msgs.push_str("]}");
let s = trajectory_signals(msgs.as_bytes());
assert_eq!(
s.tool_errors, 0,
"failures outside the window must not count; a recovered session is not a hard one"
);
assert!(s.tool_results > 0, "the recent clean results must be seen");
}
#[test]
fn the_hint_reaches_the_feature_vector_for_both_dialects() {
let hard = br#"{"messages":[
{"role":"assistant","content":[{"type":"tool_use","name":"b","input":{"c":1}}]},
{"role":"user","content":[{"type":"tool_result","is_error":true}]},
{"role":"assistant","content":[{"type":"tool_use","name":"b","input":{"c":1}}]},
{"role":"user","content":[{"type":"tool_result","is_error":true}]},
{"role":"assistant","content":[{"type":"tool_use","name":"b","input":{"c":1}}]},
{"role":"user","content":[{"type":"tool_result","is_error":true}]}
]}"#;
let h = HeaderMap::new();
assert!(
extract_features(&h, hard).difficulty_hint > 0,
"a visibly struggling Anthropic conversation must carry a non-zero hint"
);
assert!(
extract_openai_features(&h, hard).difficulty_hint > 0,
"the OpenAI extractor must populate the hint too, not silently leave it zero"
);
let plain = br#"{"messages":[{"role":"user","content":"hello"}]}"#;
assert_eq!(
extract_features(&h, plain).difficulty_hint,
0,
"a single-shot request must be unchanged by this feature"
);
}
fn test_config() -> ProxyConfig {
ProxyConfig::from_lookup(|_| None).unwrap()
}
#[test]
fn openai_observe_receipt_uses_openai_usage_and_price() {
let config = test_config();
let req = Bytes::from_static(br#"{"model":"gpt-4.1-mini","messages":[]}"#);
let resp = Bytes::from_static(
br#"{"model":"gpt-4.1-mini","choices":[{"finish_reason":"length"}],
"usage":{"prompt_tokens":1000,"completion_tokens":200,
"prompt_tokens_details":{"cached_tokens":400}}}"#,
);
let trace = build_trace(
&config,
&req,
&resp,
7,
None,
"openai",
"openai.chat_completions",
None,
);
let a = &trace.attempts[0];
assert_eq!(a.provider, "openai");
assert_eq!(a.in_tokens, 600, "uncached prompt tokens");
assert_eq!(a.cache_read_tokens, 400);
assert_eq!(a.out_tokens, 200);
assert!(a.truncated, "finish_reason length");
let want = config
.prices
.cost_usd_with_cache("openai/gpt-4.1-mini", 600, 0, 400, 200)
.expect("gpt-4.1-mini is priced");
assert!(want > 0.0);
assert!(
(a.cost_usd - want).abs() < 1e-12,
"{} vs {want}",
a.cost_usd
);
let mut u = SseUsage::default();
let mut buf = Vec::new();
let mut overflow = false;
parse_sse_chunk_into(
b"data: {\"model\":\"gpt-4.1-mini\",\"choices\":[],\"usage\":{\"prompt_tokens\":1000,\"completion_tokens\":200,\"prompt_tokens_details\":{\"cached_tokens\":400}}}\n",
&mut buf,
&mut overflow,
&mut u,
);
assert_eq!(
(u.in_tokens, u.cache_read_tokens, u.out_tokens),
(600, 400, 200)
);
}
#[test]
fn build_trace_maps_request_and_response_fields() {
let config = test_config();
let req = Bytes::from_static(
br#"{"model":"claude-haiku-4-5","tools":[{"name":"a"}],"messages":[]}"#,
);
let resp = Bytes::from_static(
br#"{"model":"claude-haiku-4-5","usage":{"input_tokens":1200,"output_tokens":300}}"#,
);
let trace = build_trace(
&config,
&req,
&resp,
42,
Some("sess-1"),
"anthropic",
"anthropic.messages",
None,
);
assert_eq!(trace.request.api, "anthropic.messages");
assert_eq!(trace.session_id, "sess-1");
assert_eq!(trace.attempts.len(), 1);
let attempt = &trace.attempts[0];
assert_eq!(attempt.model, "claude-haiku-4-5");
assert_eq!(attempt.provider, "anthropic");
assert_eq!(attempt.in_tokens, 1200);
assert_eq!(attempt.out_tokens, 300);
assert!(attempt.cost_usd > 0.0);
assert_eq!(trace.request.features.tool_count, 1);
assert!(!trace.request.features.has_images);
assert_eq!(trace.final_.served_rung, Some(0));
}
#[test]
fn build_trace_falls_back_to_trace_id_session_when_header_absent() {
let config = test_config();
let req = Bytes::from_static(b"{}");
let resp = Bytes::from_static(b"{}");
let trace = build_trace(
&config,
&req,
&resp,
1,
None,
"anthropic",
"anthropic.messages",
None,
);
assert_eq!(trace.session_id, trace.trace_id.to_string());
}
#[test]
fn build_error_trace_has_no_attempts_and_served_from_error() {
let config = test_config();
let req = Bytes::from_static(br#"{"model":"claude-haiku-4-5"}"#);
let trace = build_error_trace(&config, &req, 7, None, "anthropic", "anthropic.messages");
assert!(trace.attempts.is_empty());
assert_eq!(trace.final_.served_from, ServedFrom::Error);
assert_eq!(trace.final_.served_rung, None);
}
#[test]
fn message_with_image_block_sets_has_images() {
let req = br#"{"messages":[{"role":"user","content":[{"type":"image"}]}]}"#;
let (_, _, has_images) = request_features(req);
assert!(has_images);
}
#[test]
fn prompt_hash_never_contains_raw_prompt_text() {
let hash = prompt_hash("salt", b"super secret prompt");
assert!(!hash.contains("secret"));
assert_eq!(hash.len(), 64);
}
#[test]
fn parse_model_request_preserves_content_verbatim_and_projects_text() {
let body = br#"{"model":"m","system":"sys","max_tokens":50,
"messages":[{"role":"user","content":[{"type":"text","text":"a"},{"type":"text","text":"b"}]},
{"role":"assistant","content":"c"}]}"#;
let req = parse_model_request(body).unwrap();
assert_eq!(req.system.as_deref(), Some("sys"));
assert_eq!(req.max_tokens, 50);
assert_eq!(req.messages.len(), 2);
assert_eq!(
req.messages[0].content,
serde_json::json!([{"type":"text","text":"a"},{"type":"text","text":"b"}])
);
assert_eq!(req.messages[1].content, Value::String("c".to_owned()));
assert_eq!(req.messages[0].text_view(), "a\nb");
assert_eq!(req.messages[1].text_view(), "c");
}
#[test]
fn tool_and_image_blocks_survive_the_request_round_trip() {
let body = br#"{"model":"m","max_tokens":50,"messages":[
{"role":"assistant","content":[{"type":"tool_use","id":"t1","name":"calc","input":{"x":1}}]},
{"role":"user","content":[
{"type":"tool_result","tool_use_id":"t1","content":"2"},
{"type":"image","source":{"type":"base64","media_type":"image/png","data":"AA=="}}
]}]}"#;
let req = parse_model_request(body).unwrap();
let round_tripped = serde_json::to_value(&req.messages).unwrap();
assert_eq!(
round_tripped,
serde_json::json!([
{"role":"assistant","content":[{"type":"tool_use","id":"t1","name":"calc","input":{"x":1}}]},
{"role":"user","content":[
{"type":"tool_result","tool_use_id":"t1","content":"2"},
{"type":"image","source":{"type":"base64","media_type":"image/png","data":"AA=="}}
]}
])
);
}
#[test]
fn text_message_serializes_byte_identical_to_a_plain_string() {
let m = ChatMessage::text("user", "hello");
assert_eq!(
serde_json::to_string(&m).unwrap(),
r#"{"role":"user","content":"hello"}"#
);
}
#[test]
fn parse_model_request_rejects_non_message_bodies() {
assert!(parse_model_request(b"not json").is_none());
assert!(parse_model_request(br#"{"no":"messages"}"#).is_none());
}
use crate::provider::{MockProvider, ModelResponse, Provider, ProviderError, ProviderRegistry};
use axum::extract::State;
use std::collections::HashMap;
use std::sync::Arc;
use tokio::sync::mpsc;
fn model_resp(model: &str, text: &str) -> ModelResponse {
ModelResponse {
model: model.to_owned(),
text: text.to_owned(),
in_tokens: 1000,
cache_write_tokens: 0,
cache_read_tokens: 0,
out_tokens: 400,
raw: serde_json::Value::Null,
}
}
fn enforce_state(
ladder: &[&str],
gates: &[&str],
outcomes: Vec<(&str, Result<ModelResponse, ProviderError>)>,
) -> (AppState, mpsc::Receiver<Trace>) {
let toml = format!(
"[[route]]\nmatch = {{}}\nmode = \"enforce\"\nladder = [{}]\ngates = [{}]\n",
ladder
.iter()
.map(|m| format!("\"{m}\""))
.collect::<Vec<_>>()
.join(", "),
gates
.iter()
.map(|g| format!("\"{g}\""))
.collect::<Vec<_>>()
.join(", "),
);
let config = ProxyConfig::from_lookup(|k| match k {
"FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
"FIRSTPASS_MODE" => Some("enforce".to_owned()),
_ => None,
})
.unwrap();
let mut outs = HashMap::new();
for (model, out) in outcomes {
outs.insert(model.to_owned(), out);
}
let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
map.insert(
"anthropic".to_owned(),
Arc::new(MockProvider::new("anthropic", outs)),
);
let providers = ProviderRegistry::from_map(map);
let (traces, rx) = mpsc::channel(64);
let state = AppState {
config: Arc::new(config),
http: reqwest::Client::new(),
providers,
gate_health: Arc::new(GateHealthRegistry::new()),
shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
experiments: Arc::default(),
guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
traces,
adaptive: None,
eprocess: None,
bandit: None,
promoter: None,
verified_cache: None,
predictor: None,
prior: None,
tenant_rate_limiter: None,
spill: None,
};
(state, rx)
}
fn user_body() -> Bytes {
Bytes::from_static(
br#"{"model":"ignored","max_tokens":64,"messages":[{"role":"user","content":"hi"}]}"#,
)
}
async fn body_json(resp: Response) -> Value {
let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
.await
.unwrap();
serde_json::from_slice(&bytes).unwrap()
}
#[tokio::test]
async fn mode_profile_stamped_on_trace_when_non_balanced() {
let (state, mut rx) = enforce_state(
&["anthropic/claude-haiku-4-5"],
&["non-empty"],
vec![(
"anthropic/claude-haiku-4-5",
Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
)],
);
let mut headers = HeaderMap::new();
headers.insert("x-firstpass-mode", "quality".parse().unwrap());
let resp = messages(
State(state),
Extension(TenantId("default".to_owned())),
headers,
user_body(),
)
.await;
assert_eq!(resp.status(), axum::http::StatusCode::OK);
let trace = rx.try_recv().expect("trace enqueued");
assert_eq!(
trace.policy.mode_profile.as_deref(),
Some("quality"),
"mode_profile must be stamped when quality mode is active"
);
}
#[tokio::test]
async fn mode_profile_absent_from_trace_when_balanced() {
let (state, mut rx) = enforce_state(
&["anthropic/claude-haiku-4-5"],
&["non-empty"],
vec![(
"anthropic/claude-haiku-4-5",
Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
)],
);
let resp = messages(
State(state),
Extension(TenantId("default".to_owned())),
HeaderMap::new(),
user_body(),
)
.await;
assert_eq!(resp.status(), axum::http::StatusCode::OK);
let trace = rx.try_recv().expect("trace enqueued");
assert!(
trace.policy.mode_profile.is_none(),
"mode_profile must be absent when Balanced (byte-identical invariant)"
);
}
#[tokio::test]
async fn enforce_serves_a_tool_only_turn_from_the_cheap_rung() {
let haiku = "anthropic/claude-haiku-4-5";
let mut tool_turn = model_resp(haiku, "");
tool_turn.raw = serde_json::json!({
"model": "claude-haiku-4-5",
"stop_reason": "tool_use",
"content": [{"type": "tool_use", "id": "toolu_1", "name": "bash",
"input": {"command": "ls"}}],
"usage": {"input_tokens": 10, "output_tokens": 5}
});
let (state, mut rx) = enforce_state(
&[haiku, "anthropic/claude-sonnet-5"],
&["non-empty"],
vec![
(haiku, Ok(tool_turn)),
(
"anthropic/claude-sonnet-5",
Ok(model_resp(
"anthropic/claude-sonnet-5",
"should not be called",
)),
),
],
);
let body_with_tools = Bytes::from_static(br#"{
"model": "ignored",
"max_tokens": 64,
"messages": [{"role": "user", "content": "run ls"}],
"tools": [{"name": "bash", "input_schema": {"type": "object", "properties": {"command": {"type": "string"}}}}]
}"#);
let resp = messages(
State(state),
Extension(TenantId("default".to_owned())),
HeaderMap::new(),
body_with_tools,
)
.await;
assert_eq!(resp.status(), axum::http::StatusCode::OK);
let body = axum::body::to_bytes(resp.into_body(), 1 << 20)
.await
.unwrap();
let json: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(json["stop_reason"], "tool_use", "{json}");
let trace = rx.try_recv().expect("trace enqueued");
assert_eq!(trace.attempts.len(), 1, "no escalation on a tool step");
assert_eq!(trace.final_.served_rung, Some(0));
}
#[tokio::test]
async fn enforce_serves_first_pass_and_returns_anthropic_shape() {
let (state, mut rx) = enforce_state(
&["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"],
&["non-empty"],
vec![(
"anthropic/claude-haiku-4-5",
Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
)],
);
let resp = messages(
State(state),
Extension(TenantId("default".to_owned())),
HeaderMap::new(),
user_body(),
)
.await;
assert_eq!(resp.status(), axum::http::StatusCode::OK);
let json = body_json(resp).await;
assert_eq!(json["type"], "message");
assert_eq!(json["content"][0]["text"], "hello");
assert_eq!(json["model"], "anthropic/claude-haiku-4-5");
let trace = rx.try_recv().expect("a trace was enqueued");
assert_eq!(trace.mode, Mode::Enforce);
assert_eq!(trace.final_.served_rung, Some(0));
assert_eq!(trace.attempts.len(), 1);
}
#[tokio::test]
async fn enforce_escalates_then_serves_and_traces_two_attempts() {
let (state, mut rx) = enforce_state(
&["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"],
&["non-empty"],
vec![
(
"anthropic/claude-haiku-4-5",
Ok(model_resp("anthropic/claude-haiku-4-5", " ")),
), (
"anthropic/claude-sonnet-5",
Ok(model_resp("anthropic/claude-sonnet-5", "answer")),
),
],
);
let resp = messages(
State(state),
Extension(TenantId("default".to_owned())),
HeaderMap::new(),
user_body(),
)
.await;
let json = body_json(resp).await;
assert_eq!(json["content"][0]["text"], "answer");
let trace = rx.try_recv().expect("trace enqueued");
assert_eq!(trace.attempts.len(), 2);
assert_eq!(trace.final_.escalations, 1);
assert_eq!(trace.final_.served_rung, Some(1));
}
#[tokio::test]
async fn enforce_all_rungs_error_returns_502() {
let (state, mut rx) = enforce_state(
&["anthropic/claude-haiku-4-5"],
&["non-empty"],
vec![(
"anthropic/claude-haiku-4-5",
Err(ProviderError::Transport("down".into())),
)],
);
let resp = messages(
State(state),
Extension(TenantId("default".to_owned())),
HeaderMap::new(),
user_body(),
)
.await;
assert_eq!(resp.status(), axum::http::StatusCode::BAD_GATEWAY);
assert!(rx.try_recv().is_ok());
}
#[tokio::test]
async fn no_routing_config_falls_through_to_observe_not_enforce() {
let config = ProxyConfig::from_lookup(|k| match k {
"FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
_ => None,
})
.unwrap();
let (traces, _rx) = mpsc::channel(64);
let state = AppState {
config: Arc::new(config),
http: reqwest::Client::new(),
providers: ProviderRegistry::new("http://127.0.0.1:1", "http://127.0.0.1:1"),
gate_health: Arc::new(GateHealthRegistry::new()),
shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
experiments: Arc::default(),
guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
traces,
adaptive: None,
eprocess: None,
bandit: None,
promoter: None,
verified_cache: None,
predictor: None,
prior: None,
tenant_rate_limiter: None,
spill: None,
};
let resp = messages(
State(state),
Extension(TenantId("default".to_owned())),
HeaderMap::new(),
user_body(),
)
.await;
assert_ne!(resp.status(), axum::http::StatusCode::OK);
}
fn bare_enforce_route() -> Route {
use firstpass_core::config::{Match, Mode};
Route {
match_: Match::default(),
mode: Mode::Enforce,
ladder: vec!["anthropic/claude-haiku-4-5".to_owned()],
gates: vec![],
deferred_gates: vec![],
routing_mode: None,
rollout: None,
shadow: None,
challenger: None,
}
}
#[test]
fn balanced_preset_application_is_noop() {
let base_max_rungs = 3u32;
let base_speculation = 2u32;
let preset = RoutingMode::Balanced.preset();
let max_rungs = if let Some(d) = preset.max_rungs_delta {
(base_max_rungs as i32 + d).max(1) as u32
} else {
base_max_rungs
};
let speculation = preset.speculation.unwrap_or(base_speculation);
let start_at_top = preset.start_at_top;
assert_eq!(max_rungs, 3, "Balanced must not change max_rungs");
assert_eq!(speculation, 2, "Balanced must not change speculation");
assert!(!start_at_top, "Balanced must not set start_at_top");
}
#[test]
fn resolve_mode_header_wins_over_route_and_global() {
let mut headers = HeaderMap::new();
headers.insert("x-firstpass-mode", "cost".parse().unwrap());
let mut route = bare_enforce_route();
route.routing_mode = Some(RoutingMode::Quality); let mut config = test_config();
config.default_routing_mode = RoutingMode::Max; assert_eq!(
resolve_mode(&headers, &route, &config),
RoutingMode::Cost,
"header must win"
);
}
#[test]
fn resolve_mode_route_wins_over_global_when_no_header() {
let mut route = bare_enforce_route();
route.routing_mode = Some(RoutingMode::Latency);
let mut config = test_config();
config.default_routing_mode = RoutingMode::Max;
assert_eq!(
resolve_mode(&HeaderMap::new(), &route, &config),
RoutingMode::Latency,
"route must beat global default"
);
}
#[test]
fn resolve_mode_global_when_header_and_route_absent() {
let mut config = test_config();
config.default_routing_mode = RoutingMode::Quality;
assert_eq!(
resolve_mode(&HeaderMap::new(), &bare_enforce_route(), &config),
RoutingMode::Quality
);
}
#[test]
fn resolve_mode_unknown_header_falls_through_to_route() {
let mut headers = HeaderMap::new();
headers.insert("x-firstpass-mode", "turbo-mode".parse().unwrap());
let mut route = bare_enforce_route();
route.routing_mode = Some(RoutingMode::Cost);
let config = test_config();
assert_eq!(
resolve_mode(&headers, &route, &config),
RoutingMode::Cost,
"unknown header value must fall through to route"
);
}
#[test]
fn resolve_mode_header_case_insensitive() {
let mut headers = HeaderMap::new();
headers.insert("x-firstpass-mode", "QUALITY".parse().unwrap());
assert_eq!(
resolve_mode(&headers, &bare_enforce_route(), &test_config()),
RoutingMode::Quality
);
}
#[test]
fn resolve_mode_no_mode_set_returns_balanced() {
assert_eq!(
resolve_mode(&HeaderMap::new(), &bare_enforce_route(), &test_config()),
RoutingMode::Balanced
);
}
#[test]
fn capabilities_json_includes_routing_modes() {
let modes: Vec<&'static str> = RoutingMode::ALL.iter().map(|m| m.as_str()).collect();
assert!(modes.contains(&"balanced"));
assert!(modes.contains(&"cost"));
assert!(modes.contains(&"quality"));
assert!(modes.contains(&"latency"));
assert!(modes.contains(&"max"));
assert!(modes.contains(&"observe"));
}
#[test]
fn model_list_dedups_across_ladders_keeping_cheapest_first_order() {
let config = firstpass_core::config::Config::parse(
r#"
[[route]]
mode = "enforce"
ladder = ["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"]
[[route]]
mode = "enforce"
ladder = ["anthropic/claude-sonnet-5", "anthropic/claude-opus-4-8"]
"#,
)
.expect("test config parses");
let out = model_list(Some(&config));
let ids: Vec<&str> = out["data"]
.as_array()
.expect("data is an array")
.iter()
.map(|m| m["id"].as_str().expect("id is a string"))
.collect();
assert_eq!(
ids,
[
"anthropic/claude-haiku-4-5",
"anthropic/claude-sonnet-5",
"anthropic/claude-opus-4-8",
],
"sonnet appears once, and ladder order is preserved"
);
assert_eq!(out["object"], "list");
assert_eq!(out["data"][0]["owned_by"], "anthropic");
let haiku_in = out["data"][0]["firstpass"]["input_per_mtok"]
.as_f64()
.expect("a first-party rung has a price");
assert!(haiku_in > 0.0, "expected a real price, got {haiku_in}");
let opus_in = out["data"][2]["firstpass"]["input_per_mtok"]
.as_f64()
.expect("a first-party rung has a price");
assert!(haiku_in < opus_in, "{haiku_in} should undercut {opus_in}");
}
#[tokio::test]
async fn healthz_names_the_service_so_a_client_can_tell_us_from_a_stranger() {
let body = healthz().await.into_response();
let bytes = axum::body::to_bytes(body.into_body(), 64 * 1024)
.await
.expect("healthz body");
let json: serde_json::Value = serde_json::from_slice(&bytes).expect("healthz is json");
assert_eq!(json["status"], "ok");
assert_eq!(json["service"], "firstpass");
}
#[test]
fn model_list_is_empty_without_routing_config() {
let out = model_list(None);
assert_eq!(out["object"], "list");
assert_eq!(
out["data"].as_array().expect("data is an array").len(),
0,
"an observe-only deployment advertises no models"
);
}
#[test]
fn detects_stream_requests() {
assert!(is_stream_request(br#"{"stream": true}"#));
assert!(!is_stream_request(br#"{"stream": false}"#));
assert!(!is_stream_request(br#"{"model":"m"}"#));
assert!(!is_stream_request(b"not json"));
}
#[test]
fn detects_tool_blocks_in_messages() {
let with =
br#"{"messages":[{"role":"user","content":[{"type":"tool_result","content":"42"}]}]}"#;
let without = br#"{"messages":[{"role":"user","content":"hi"}]}"#;
assert!(messages_have_tool_blocks(with));
assert!(!messages_have_tool_blocks(without));
}
#[test]
fn enforce_only_handles_plain_text() {
let plain =
Bytes::from_static(br#"{"model":"m","messages":[{"role":"user","content":"hi"}]}"#);
let tools = Bytes::from_static(
br#"{"model":"m","tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
);
let f_plain = extract_features(&HeaderMap::new(), &plain);
let f_tools = extract_features(&HeaderMap::new(), &tools);
let anthropic_ladder = vec!["anthropic/claude-haiku-4-5".to_owned()];
let providers = test_registry();
assert!(enforce_can_handle(
&f_plain,
&plain,
false,
&anthropic_ladder,
&providers,
Dialect::Anthropic,
));
assert!(!enforce_can_handle(
&f_tools,
&tools,
false,
&anthropic_ladder,
&providers,
Dialect::Anthropic,
));
}
#[test]
fn structured_enforce_routes_tools_and_streaming() {
let tools = Bytes::from_static(
br#"{"model":"m","tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
);
let streaming_tools = Bytes::from_static(
br#"{"model":"m","stream":true,"tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
);
let f = extract_features(&HeaderMap::new(), &tools);
let anthropic_ladder = vec![
"anthropic/claude-haiku-4-5".to_owned(),
"anthropic/claude-sonnet-5".to_owned(),
];
let providers = test_registry();
assert!(enforce_can_handle(
&f,
&tools,
true,
&anthropic_ladder,
&providers,
Dialect::Anthropic,
));
assert!(enforce_can_handle(
&f,
&streaming_tools,
true,
&anthropic_ladder,
&providers,
Dialect::Anthropic,
));
}
fn test_registry() -> crate::provider::ProviderRegistry {
crate::provider::ProviderRegistry::new("http://localhost", "http://localhost")
}
#[test]
fn fidelity_guard_blocks_structured_on_non_verbatim_ladder() {
let tools = Bytes::from_static(
br#"{"model":"m","tools":[{"name":"t"}],"messages":[{"role":"user","content":"hi"}]}"#,
);
let plain =
Bytes::from_static(br#"{"model":"m","messages":[{"role":"user","content":"hi"}]}"#);
let f_tools = extract_features(&HeaderMap::new(), &tools);
let f_plain = extract_features(&HeaderMap::new(), &plain);
let providers = test_registry();
let mixed_ladder = vec![
"openai/gpt-4.1-mini".to_owned(),
"anthropic/claude-sonnet-5".to_owned(),
];
assert!(!enforce_can_handle(
&f_tools,
&tools,
true,
&mixed_ladder,
&providers,
Dialect::Anthropic,
));
assert!(enforce_can_handle(
&f_plain,
&plain,
true,
&mixed_ladder,
&providers,
Dialect::Anthropic,
));
}
#[test]
fn enforce_sse_reemission_preserves_text_and_tool_use() {
let resp = ModelResponse {
model: "anthropic/claude-haiku-4-5".to_owned(),
text: "let me check".to_owned(),
in_tokens: 5,
cache_write_tokens: 0,
cache_read_tokens: 0,
out_tokens: 7,
raw: serde_json::json!({
"content": [
{ "type": "text", "text": "let me check" },
{ "type": "tool_use", "id": "tu_1", "name": "get_weather", "input": { "city": "Paris" } }
]
}),
};
let sse = anthropic_sse_from_message(&anthropic_response_json(&resp));
let frames: Vec<Value> = sse
.lines()
.filter_map(|l| l.strip_prefix("data: "))
.map(|d| serde_json::from_str::<Value>(d).expect("each SSE data frame is valid JSON"))
.collect();
assert_eq!(frames.first().unwrap()["type"], "message_start");
assert_eq!(frames.last().unwrap()["type"], "message_stop");
assert!(frames.iter().any(|f| f["delta"]["type"] == "text_delta"
&& f["delta"]["text"] == "let me check"));
assert!(
frames
.iter()
.any(|f| f["content_block"]["type"] == "tool_use"
&& f["content_block"]["name"] == "get_weather"
&& f["content_block"]["id"] == "tu_1")
);
assert!(
frames
.iter()
.any(|f| f["delta"]["type"] == "input_json_delta"
&& f["delta"]["partial_json"] == r#"{"city":"Paris"}"#)
);
}
#[tokio::test]
async fn derived_session_survives_the_observe_fallback() {
let toml = "[[price]]\nmodel = \"anthropic/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/m\"]\ngates = [\"non-empty\"]\n[escalation]\nenforce_structured = false\n";
let config = ProxyConfig::from_lookup(|k| match k {
"FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
"FIRSTPASS_MODE" => Some("enforce".to_owned()),
"FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
_ => None,
})
.unwrap();
let (traces, mut rx) = mpsc::channel(8);
let state = AppState {
config: Arc::new(config),
http: reqwest::Client::new(),
providers: ProviderRegistry::from_map(HashMap::new()),
gate_health: Arc::new(GateHealthRegistry::new()),
shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
experiments: Arc::default(),
guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
traces,
adaptive: None,
eprocess: None,
bandit: None,
promoter: None,
verified_cache: None,
predictor: None,
prior: None,
tenant_rate_limiter: None,
spill: None,
};
let body = Bytes::from_static(
br#"{"model":"m","tools":[{"name":"get_weather"}],"messages":[{"role":"user","content":"hi"}]}"#,
);
let _ = messages(
State(state),
Extension(TenantId("default".to_owned())),
HeaderMap::new(),
body,
)
.await;
let t = rx.recv().await.expect("observe fallback records a trace");
assert!(t.session_id.starts_with("conv:"), "{}", t.session_id);
}
#[tokio::test]
async fn enforce_falls_back_to_observe_for_tool_requests() {
let toml = "[[price]]\nmodel = \"anthropic/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/m\"]\ngates = [\"non-empty\"]\n";
let config = ProxyConfig::from_lookup(|k| match k {
"FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
"FIRSTPASS_MODE" => Some("enforce".to_owned()),
"FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
_ => None,
})
.unwrap();
let mut outs = HashMap::new();
outs.insert(
"anthropic/m".to_owned(),
Ok(model_resp("anthropic/m", "hello")),
);
let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
map.insert(
"anthropic".to_owned(),
Arc::new(MockProvider::new("anthropic", outs)),
);
let (traces, _rx) = mpsc::channel(64);
let state = AppState {
config: Arc::new(config),
http: reqwest::Client::new(),
providers: ProviderRegistry::from_map(map),
gate_health: Arc::new(GateHealthRegistry::new()),
shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
experiments: Arc::default(),
guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
traces,
adaptive: None,
eprocess: None,
bandit: None,
promoter: None,
verified_cache: None,
predictor: None,
prior: None,
tenant_rate_limiter: None,
spill: None,
};
let plain =
Bytes::from_static(br#"{"model":"m","messages":[{"role":"user","content":"hi"}]}"#);
let resp = messages(
State(state.clone()),
Extension(TenantId("default".to_owned())),
HeaderMap::new(),
plain,
)
.await;
assert_eq!(
resp.status(),
axum::http::StatusCode::OK,
"plain text should enforce"
);
let tools = Bytes::from_static(
br#"{"model":"m","tools":[{"name":"get_weather"}],"messages":[{"role":"user","content":"hi"}]}"#,
);
let resp = messages(
State(state.clone()),
Extension(TenantId("default".to_owned())),
HeaderMap::new(),
tools.clone(),
)
.await;
assert_eq!(
resp.status(),
axum::http::StatusCode::OK,
"tool request must route through enforce by default (ADR 0005 default-on)"
);
let toml_off = "[[price]]\nmodel = \"anthropic/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/m\"]\ngates = [\"non-empty\"]\n[escalation]\nenforce_structured = false\n";
let config_off = ProxyConfig::from_lookup(|k| match k {
"FIRSTPASS_CONFIG_TOML" => Some(toml_off.to_owned()),
"FIRSTPASS_MODE" => Some("enforce".to_owned()),
"FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
_ => None,
})
.unwrap();
let state_off = AppState {
config: Arc::new(config_off),
..state.clone()
};
let resp = messages(
State(state_off),
Extension(TenantId("default".to_owned())),
HeaderMap::new(),
tools,
)
.await;
assert_ne!(
resp.status(),
axum::http::StatusCode::OK,
"with enforce_structured = false a tool request must fall back to observe"
);
let toolres = Bytes::from_static(
br#"{"model":"m","messages":[{"role":"user","content":[{"type":"tool_result","tool_use_id":"x","content":"42"}]}]}"#,
);
let resp = messages(
State(state),
Extension(TenantId("default".to_owned())),
HeaderMap::new(),
toolres,
)
.await;
assert_eq!(
resp.status(),
axum::http::StatusCode::OK,
"tool_result blocks route through enforce by default too (verbatim carry)"
);
}
#[tokio::test]
async fn panel_is_public_markup_but_receipts_stay_behind_auth() {
use tower::ServiceExt;
let (state, _rx) = enforce_state(
&["anthropic/claude-haiku-4-5"],
&["non-empty"],
vec![(
"anthropic/claude-haiku-4-5",
Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
)],
);
let router = app(state).expect("prometheus recorder installs");
let page = router
.clone()
.oneshot(
axum::http::Request::builder()
.uri("/panel")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(page.status(), axum::http::StatusCode::OK);
let html = axum::body::to_bytes(page.into_body(), 1 << 20)
.await
.unwrap();
let html = String::from_utf8_lossy(&html);
assert!(
html.contains("firstpass") && html.contains("/v1/receipts"),
"the page must fetch its data rather than embed it"
);
assert!(
!html.contains("http://") && !html.contains("https://"),
"the panel must not reference any external origin"
);
assert!(html.contains("const esc ="), "the escaper must exist");
for raw in [
"${a.model}",
"${a.rung}",
"${a.verdict}",
"${r.trace_id}",
"${r.served_rung}",
] {
assert!(
!html.contains(raw),
"receipt field interpolated unescaped into innerHTML: {raw}"
);
}
let data = router
.oneshot(
axum::http::Request::builder()
.uri("/v1/receipts?limit=5")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(data.status(), axum::http::StatusCode::OK);
let body = axum::body::to_bytes(data.into_body(), 1 << 20)
.await
.unwrap();
let v: Value = serde_json::from_slice(&body).unwrap();
assert!(v.get("receipts").and_then(Value::as_array).is_some());
}
async fn feedback_state() -> (AppState, std::path::PathBuf, String) {
feedback_state_with(false).await
}
async fn feedback_state_with(served: bool) -> (AppState, std::path::PathBuf, String) {
let db = std::env::temp_dir().join(format!("firstpass-feedback-{}.db", Uuid::now_v7()));
let (tx, handle) = crate::store::open(&db).unwrap();
let mut trace = build_error_trace(
&ProxyConfig::from_lookup(|_| None).unwrap(),
&Bytes::from_static(b"{}"),
5,
Some("sess-fb"),
"anthropic",
"anthropic.messages",
);
trace.attempts.push(Attempt {
rung: 0,
model: "anthropic/claude-haiku-4-5".into(),
provider: "anthropic".into(),
in_tokens: 10,
cache_write_tokens: 0,
cache_read_tokens: 0,
out_tokens: 5,
cost_usd: 0.001,
latency_ms: 5,
gates: vec![],
verdict: Verdict::Pass,
truncated: false,
});
let trace_id = trace.trace_id.to_string();
if served {
trace.final_.served_rung = Some(0);
}
tx.try_send(trace).unwrap();
drop(tx);
handle.await.unwrap();
let db_str = db.to_string_lossy().into_owned();
let config = ProxyConfig::from_lookup(move |k| match k {
"FIRSTPASS_DB" => Some(db_str.clone()),
_ => None,
})
.unwrap();
let (traces, _rx) = mpsc::channel(64);
let state = AppState {
config: Arc::new(config),
http: reqwest::Client::new(),
providers: ProviderRegistry::new("http://127.0.0.1:1", "http://127.0.0.1:1"),
gate_health: Arc::new(GateHealthRegistry::new()),
shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
experiments: Arc::default(),
guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
traces,
adaptive: None,
eprocess: None,
bandit: None,
promoter: None,
verified_cache: None,
predictor: None,
prior: None,
tenant_rate_limiter: None,
spill: None,
};
(state, db, trace_id)
}
#[tokio::test]
async fn feedback_without_a_score_still_advances_the_eprocess() {
use firstpass_core::eprocess::{DEFAULT_BET, EProcessRiskControl};
let (mut state, _db, trace_id) = feedback_state_with(true).await;
let grid: Vec<f64> = (0..=20).map(|i| f64::from(i) / 20.0).collect();
let ep = Arc::new(std::sync::Mutex::new(EProcessRiskControl::new(
0.2,
0.05,
DEFAULT_BET,
&grid,
)));
state.eprocess = Some(ep.clone());
assert_eq!(ep.lock().unwrap().rounds(), 0);
let body = Bytes::from(
serde_json::json!({
"trace_id": trace_id, "gate_id": "tests", "verdict": "pass", "reporter": "ci"
})
.to_string(),
);
let resp = feedback(
State(state.clone()),
Extension(TenantId("default".to_owned())),
body,
)
.await
.into_response();
assert_eq!(resp.status(), axum::http::StatusCode::ACCEPTED);
assert_eq!(
ep.lock().unwrap().rounds(),
1,
"a scoreless feedback call must still advance the controller — keying on the optional \
payload score left it permanently inert"
);
}
#[tokio::test]
async fn feedback_nudges_the_adaptive_threshold() {
use firstpass_core::conformal::AdaptiveConformal;
let (mut state, _db, trace_id) = feedback_state().await;
let aci = Arc::new(std::sync::Mutex::new(AdaptiveConformal::new(0.1, 0.2, 0.5)));
state.adaptive = Some(aci.clone());
let before = aci.lock().unwrap().threshold();
let fail = Bytes::from(
serde_json::json!({ "trace_id": trace_id, "gate_id": "tests", "verdict": "fail", "reporter": "ci" })
.to_string(),
);
assert_eq!(
feedback(
State(state.clone()),
Extension(TenantId("default".to_owned())),
fail
)
.await
.status(),
axum::http::StatusCode::ACCEPTED
);
let after_fail = aci.lock().unwrap().threshold();
assert!(
after_fail > before,
"served fail should raise the live threshold: {before} -> {after_fail}"
);
let pass = Bytes::from(
serde_json::json!({ "trace_id": trace_id, "gate_id": "tests", "verdict": "pass", "reporter": "ci" })
.to_string(),
);
let _ = feedback(
State(state),
Extension(TenantId("default".to_owned())),
pass,
)
.await;
assert!(aci.lock().unwrap().threshold() < after_fail);
}
#[tokio::test]
async fn feedback_records_a_deferred_verdict_without_breaking_the_chain() {
let (state, db, trace_id) = feedback_state().await;
let body = Bytes::from(
serde_json::json!({
"trace_id": trace_id,
"gate_id": "tests",
"verdict": "pass",
"score": 1.0,
"reporter": "ci",
})
.to_string(),
);
let resp = feedback(
State(state),
Extension(TenantId("default".to_owned())),
body,
)
.await;
assert_eq!(resp.status(), axum::http::StatusCode::ACCEPTED);
let view = crate::store::load_trace_view(&db, "default", &trace_id)
.unwrap()
.unwrap();
assert_eq!(view.deferred.len(), 1);
assert_eq!(view.deferred[0].gate_id, "tests");
let traces = crate::store::load_all_traces(&db).unwrap();
firstpass_core::verify_chain(&traces, GENESIS_HASH).unwrap();
let _ = std::fs::remove_file(&db);
}
#[tokio::test]
async fn feedback_for_unknown_trace_is_404() {
let (state, db, _trace_id) = feedback_state().await;
let body = Bytes::from(
serde_json::json!({
"trace_id": "does-not-exist",
"gate_id": "tests",
"verdict": "pass",
"reporter": "ci",
})
.to_string(),
);
let resp = feedback(
State(state),
Extension(TenantId("default".to_owned())),
body,
)
.await;
assert_eq!(resp.status(), axum::http::StatusCode::NOT_FOUND);
let _ = std::fs::remove_file(&db);
}
#[tokio::test]
async fn feedback_across_tenants_is_404_not_403() {
let (state, db, trace_id) = feedback_state().await;
let body = Bytes::from(
serde_json::json!({
"trace_id": trace_id,
"gate_id": "tests",
"verdict": "pass",
"score": 1.0,
"reporter": "attacker",
})
.to_string(),
);
let resp = feedback(
State(state),
Extension(TenantId("tenant-b".to_owned())),
body,
)
.await;
assert_eq!(
resp.status(),
axum::http::StatusCode::NOT_FOUND,
"cross-tenant feedback must look exactly like a missing trace"
);
let _ = std::fs::remove_file(&db);
}
#[tokio::test]
async fn feedback_rejects_bad_verdict_and_score() {
let (state, db, trace_id) = feedback_state().await;
let bad_verdict = Bytes::from(
serde_json::json!({ "trace_id": trace_id, "gate_id": "g", "verdict": "maybe", "reporter": "x" })
.to_string(),
);
assert_eq!(
feedback(
State(state.clone()),
Extension(TenantId("default".to_owned())),
bad_verdict
)
.await
.status(),
axum::http::StatusCode::BAD_REQUEST
);
let bad_score = Bytes::from(
serde_json::json!({ "trace_id": trace_id, "gate_id": "g", "verdict": "pass", "score": 9.0, "reporter": "x" })
.to_string(),
);
assert_eq!(
feedback(
State(state),
Extension(TenantId("default".to_owned())),
bad_score
)
.await
.status(),
axum::http::StatusCode::BAD_REQUEST
);
let _ = std::fs::remove_file(&db);
}
#[tokio::test]
async fn metrics_endpoint_renders_after_a_real_request() {
use tower::ServiceExt;
let (state, mut rx) = enforce_state(
&["anthropic/claude-haiku-4-5"],
&["non-empty"],
vec![(
"anthropic/claude-haiku-4-5",
Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
)],
);
let router = app(state).expect("prometheus recorder installs");
let req = axum::http::Request::builder()
.method("POST")
.uri("/v1/messages")
.header("content-type", "application/json")
.body(Body::from(user_body()))
.unwrap();
let resp = router.clone().oneshot(req).await.unwrap();
assert_eq!(resp.status(), axum::http::StatusCode::OK);
rx.try_recv().expect("a trace was enqueued");
let metrics_req = axum::http::Request::builder()
.method("GET")
.uri("/metrics")
.body(Body::empty())
.unwrap();
let metrics_resp = router.oneshot(metrics_req).await.unwrap();
assert_eq!(metrics_resp.status(), axum::http::StatusCode::OK);
let bytes = axum::body::to_bytes(metrics_resp.into_body(), 1 << 20)
.await
.unwrap();
let body = String::from_utf8(bytes.to_vec()).unwrap();
assert!(
body.contains("firstpass_enforce_latency_ms"),
"metrics body missing enforce latency histogram: {body}"
);
assert!(
body.contains("firstpass_served_total"),
"metrics body missing served counter: {body}"
);
}
#[tokio::test]
async fn metrics_are_dimensioned_by_provider_rung_and_gate() {
use tower::ServiceExt;
let (state, mut rx) = enforce_state(
&["anthropic/claude-haiku-4-5"],
&["non-empty"],
vec![(
"anthropic/claude-haiku-4-5",
Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
)],
);
let router = app(state).expect("prometheus recorder installs");
let req = axum::http::Request::builder()
.method("POST")
.uri("/v1/messages")
.header("content-type", "application/json")
.body(Body::from(user_body()))
.unwrap();
assert_eq!(
router.clone().oneshot(req).await.unwrap().status(),
axum::http::StatusCode::OK
);
rx.try_recv().expect("a trace was enqueued");
let metrics_req = axum::http::Request::builder()
.method("GET")
.uri("/metrics")
.body(Body::empty())
.unwrap();
let bytes = axum::body::to_bytes(
router.oneshot(metrics_req).await.unwrap().into_body(),
1 << 20,
)
.await
.unwrap();
let body = String::from_utf8(bytes.to_vec()).unwrap();
assert!(
body.contains(r#"firstpass_attempt_total{provider="anthropic",rung="0"}"#),
"attempt counter is not dimensioned by provider+rung: {body}"
);
assert!(
body.contains("firstpass_attempt_latency_ms")
&& body.contains(r#"provider="anthropic""#),
"attempt latency is not dimensioned by provider: {body}"
);
assert!(
body.contains("firstpass_attempt_cost_usd_total"),
"per-attempt cost series missing: {body}"
);
assert!(
body.contains(r#"firstpass_gate_verdict_total{gate_id="non-empty",verdict="pass"}"#),
"gate verdicts are not dimensioned by gate_id+verdict: {body}"
);
}
fn auth_state(require_auth: bool, keys_json: Option<String>) -> AppState {
auth_state_rated(require_auth, keys_json, None)
}
fn auth_state_rated(
require_auth: bool,
keys_json: Option<String>,
rate_per_sec: Option<u32>,
) -> AppState {
let config = ProxyConfig::from_lookup(|k| match k {
"FIRSTPASS_REQUIRE_AUTH" => require_auth.then(|| "true".to_owned()),
"FIRSTPASS_TENANT_KEYS_JSON" => keys_json.clone(),
"FIRSTPASS_TENANT_RATE_PER_SEC" => rate_per_sec.map(|n| n.to_string()),
_ => None,
})
.unwrap();
let (traces, _rx) = mpsc::channel(64);
std::mem::forget(_rx);
let providers: HashMap<String, Arc<dyn Provider>> = HashMap::new();
let tenant_rate_limiter = build_tenant_rate_limiter(&config);
AppState {
config: Arc::new(config),
http: reqwest::Client::new(),
providers: ProviderRegistry::from_map(providers),
gate_health: Arc::new(GateHealthRegistry::new()),
shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
experiments: Arc::default(),
guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
traces,
adaptive: None,
eprocess: None,
bandit: None,
promoter: None,
verified_cache: None,
predictor: None,
prior: None,
tenant_rate_limiter,
spill: None,
}
}
fn cap_request(auth_header: Option<&str>) -> axum::http::Request<Body> {
let mut b = axum::http::Request::builder()
.method("GET")
.uri("/v1/capabilities");
if let Some(h) = auth_header {
b = b.header("authorization", h);
}
b.body(Body::empty()).unwrap()
}
#[tokio::test]
async fn auth_off_allows_unauthenticated_request() {
use tower::ServiceExt;
let router = app(auth_state(false, None)).expect("router");
let resp = router.oneshot(cap_request(None)).await.unwrap();
assert_eq!(resp.status(), axum::http::StatusCode::OK);
}
#[tokio::test]
async fn auth_on_missing_key_is_401_opaque() {
use tower::ServiceExt;
let hash = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
let keys = format!("{{\"tenant-a\": {hash:?}}}");
let router = app(auth_state(true, Some(keys))).expect("router");
let resp = router.oneshot(cap_request(None)).await.unwrap();
assert_eq!(resp.status(), axum::http::StatusCode::UNAUTHORIZED);
let json = body_json(resp).await;
assert_eq!(json["error"]["type"], "unauthorized");
let msg = json["error"]["message"].as_str().unwrap();
assert!(!msg.contains("tenant"), "no tenant oracle in body: {msg}");
}
#[tokio::test]
async fn auth_on_invalid_key_is_401() {
use tower::ServiceExt;
let hash = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
let keys = format!("{{\"tenant-a\": {hash:?}}}");
let router = app(auth_state(true, Some(keys))).expect("router");
let resp = router
.oneshot(cap_request(Some("Bearer wrong-key")))
.await
.unwrap();
assert_eq!(resp.status(), axum::http::StatusCode::UNAUTHORIZED);
}
#[tokio::test]
async fn auth_on_valid_key_proceeds() {
use tower::ServiceExt;
let hash = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
let keys = format!("{{\"tenant-a\": {hash:?}}}");
let router = app(auth_state(true, Some(keys))).expect("router");
let resp = router
.oneshot(cap_request(Some("Bearer tenant-a.key-a")))
.await
.unwrap();
assert_eq!(resp.status(), axum::http::StatusCode::OK);
}
fn two_tenant_state(rate_per_sec: Option<u32>) -> AppState {
let hash_a = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
let hash_b = crate::tenant_auth::TenantKeys::hash_key("key-b").unwrap();
let keys = format!("{{\"tenant-a\": {hash_a:?}, \"tenant-b\": {hash_b:?}}}");
let mut state = auth_state_rated(true, Some(keys), rate_per_sec);
if let Some(n) = rate_per_sec.and_then(std::num::NonZeroU32::new) {
state.tenant_rate_limiter = Some(Arc::new(governor::RateLimiter::keyed(
governor::Quota::per_hour(n),
)));
}
state
}
#[test]
fn tenant_rate_limiter_allows_a_burst_of_exactly_the_configured_rate() {
let config = ProxyConfig::from_lookup(|k| {
(k == "FIRSTPASS_TENANT_RATE_PER_SEC").then(|| "2".to_owned())
})
.unwrap();
let limiter = build_tenant_rate_limiter(&config).expect("rate configured");
let t = "tenant-a".to_owned();
assert!(limiter.check_key(&t).is_ok());
assert!(limiter.check_key(&t).is_ok());
assert!(
limiter.check_key(&t).is_err(),
"third in the same instant is over the burst"
);
assert!(
limiter.check_key(&"tenant-b".to_owned()).is_ok(),
"buckets are per tenant"
);
}
#[tokio::test]
async fn tenant_exceeding_rate_limit_gets_429_opaque() {
use tower::ServiceExt;
let router = app(two_tenant_state(Some(1))).expect("router");
let (r1, r2, r3, r4) = tokio::join!(
router
.clone()
.oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
router
.clone()
.oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
router
.clone()
.oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
router
.clone()
.oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
);
let responses = [r1.unwrap(), r2.unwrap(), r3.unwrap(), r4.unwrap()];
let ok = responses
.iter()
.filter(|r| r.status() == axum::http::StatusCode::OK)
.count();
assert!(ok >= 1, "the burst's first request must pass");
let limited: Vec<_> = responses
.into_iter()
.filter(|r| r.status() == axum::http::StatusCode::TOO_MANY_REQUESTS)
.collect();
assert!(
!limited.is_empty(),
"a 4-request burst against 1 req/sec must trip the limiter"
);
let json = body_json(limited.into_iter().next().unwrap()).await;
assert_eq!(json["error"]["type"], "rate_limited");
let msg = json["error"]["message"].as_str().unwrap();
assert!(!msg.contains('1'), "no limit value in body: {msg}");
}
#[tokio::test]
async fn rate_limit_buckets_are_independent_per_tenant() {
use tower::ServiceExt;
let router = app(two_tenant_state(Some(1))).expect("router");
let (a1, a2, a3) = tokio::join!(
router
.clone()
.oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
router
.clone()
.oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
router
.clone()
.oneshot(cap_request(Some("Bearer tenant-a.key-a"))),
);
let a_limited = [a1.unwrap(), a2.unwrap(), a3.unwrap()]
.iter()
.filter(|r| r.status() == axum::http::StatusCode::TOO_MANY_REQUESTS)
.count();
assert!(a_limited >= 1, "tenant A's burst must trip its limiter");
let b1 = router
.clone()
.oneshot(cap_request(Some("Bearer tenant-b.key-b")))
.await
.unwrap();
assert_eq!(b1.status(), axum::http::StatusCode::OK);
}
#[tokio::test]
async fn key_verify_does_not_stall_the_async_worker() {
use std::sync::atomic::{AtomicUsize, Ordering};
use tower::ServiceExt;
let router = app(two_tenant_state(None)).expect("router");
let ticks = Arc::new(AtomicUsize::new(0));
let ticker = tokio::spawn({
let ticks = ticks.clone();
async move {
loop {
tokio::time::sleep(std::time::Duration::from_millis(1)).await;
ticks.fetch_add(1, Ordering::Relaxed);
}
}
});
let resp = router
.oneshot(cap_request(Some("Bearer tenant-a.key-a")))
.await
.unwrap();
let during = ticks.load(Ordering::Relaxed);
ticker.abort();
assert_eq!(resp.status(), axum::http::StatusCode::OK);
assert!(
during >= 2,
"worker was blocked during the verify ({during} ticks)"
);
}
#[tokio::test]
async fn rate_limit_unset_never_429s() {
use tower::ServiceExt;
let router = app(two_tenant_state(None)).expect("router");
for _ in 0..20 {
let resp = router
.clone()
.oneshot(cap_request(Some("Bearer tenant-a.key-a")))
.await
.unwrap();
assert_eq!(resp.status(), axum::http::StatusCode::OK);
}
}
#[test]
fn u01_is_deterministic_and_in_range() {
let s1 = u01(0xDEAD_BEEF_CAFE_1234_u128);
let s2 = u01(0xDEAD_BEEF_CAFE_1234_u128);
assert_eq!(s1, s2, "u01 must be deterministic for the same seed");
assert!((0.0..1.0).contains(&s1), "u01 must return [0, 1), got {s1}");
let s3 = u01(0x1234_5678_9ABC_DEF0_u128);
assert_ne!(s1, s3, "different seeds should give different values");
for i in 0u64..256 {
let v = u01(i as u128);
assert!((0.0..1.0).contains(&v), "seed {i}: u01={v} out of [0,1)");
}
}
#[test]
fn epsilon_propensity_formula() {
let epsilon = 0.2_f64;
let k = 3_usize;
let greedy = 1_u32;
let p_greedy = epsilon_propensity(greedy, greedy, epsilon, k);
let expected_greedy = (1.0 - epsilon) + epsilon / k as f64;
assert!(
(p_greedy - expected_greedy).abs() < 1e-12,
"{p_greedy} != {expected_greedy}"
);
let p_other = epsilon_propensity(0, greedy, epsilon, k);
let expected_other = epsilon / k as f64;
assert!(
(p_other - expected_other).abs() < 1e-12,
"{p_other} != {expected_other}"
);
for chosen in 0..k as u32 {
let p = epsilon_propensity(chosen, greedy, epsilon, k);
assert!(
p > 0.0 && p <= 1.0,
"propensity {p} out of (0,1] for chosen={chosen}"
);
}
}
#[test]
fn epsilon_branch_and_greedy_branch_both_occur_over_many_seeds() {
let epsilon = 0.3_f64;
let mut saw_explore = false;
let mut saw_greedy = false;
for i in 0u64..200 {
let u = u01(i as u128);
if u < epsilon {
saw_explore = true;
} else {
saw_greedy = true;
}
if saw_explore && saw_greedy {
break;
}
}
assert!(
saw_explore,
"epsilon branch must fire with epsilon=0.3 over 200 seeds"
);
assert!(
saw_greedy,
"greedy branch must occur with epsilon=0.3 over 200 seeds"
);
}
#[test]
fn epsilon_propensity_sums_to_one_over_all_rungs() {
let epsilon = 0.15_f64;
let k = 4_usize;
let greedy = 2_u32;
let total: f64 = (0..k as u32)
.map(|r| epsilon_propensity(r, greedy, epsilon, k))
.sum();
assert!(
(total - 1.0).abs() < 1e-12,
"propensities must sum to 1, got {total}"
);
}
#[tokio::test(start_paused = true)]
async fn keepalive_stream_ticks_then_emits_final_frame() {
let (tx, rx) = tokio::sync::oneshot::channel::<Result<Value, ProxyError>>();
let mut ticks = tokio::time::interval(SSE_KEEPALIVE_EVERY);
ticks.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
ticks.reset();
let mut stream = KeepaliveStream {
rx: Some(rx),
ticks,
format_message: Box::new(anthropic_sse_from_message),
format_error: sse_error_event,
};
async fn next(
stream: &mut KeepaliveStream,
) -> Option<Option<Result<Bytes, std::convert::Infallible>>> {
std::future::poll_fn(|cx| {
std::task::Poll::Ready(
match futures_core::Stream::poll_next(std::pin::Pin::new(&mut *stream), cx) {
std::task::Poll::Ready(item) => Some(item),
std::task::Poll::Pending => None,
},
)
})
.await
}
assert!(
next(&mut stream).await.is_none(),
"no frame before an interval"
);
tokio::time::advance(SSE_KEEPALIVE_EVERY + Duration::from_millis(1)).await;
let frame = next(&mut stream)
.await
.expect("keepalive due")
.unwrap()
.unwrap();
assert!(
frame.starts_with(b": "),
"keepalive must be an SSE comment (ignored by every conforming parser)"
);
let message = serde_json::json!({
"id": "msg_1", "type": "message", "role": "assistant", "model": "m",
"content": [{ "type": "text", "text": "done" }],
"usage": { "input_tokens": 1, "output_tokens": 1 }
});
tx.send(Ok(message)).unwrap();
let frame = next(&mut stream)
.await
.expect("final frame")
.unwrap()
.unwrap();
let text = String::from_utf8(frame.to_vec()).unwrap();
assert!(text.contains("event: message_start"));
assert!(text.contains("event: message_stop"));
let eos = next(&mut stream)
.await
.expect("stream must end after the final frame");
assert!(eos.is_none(), "end-of-stream after the final frame");
}
#[tokio::test]
async fn streaming_enforce_serves_full_sse_sequence() {
let toml = "[[price]]\nmodel = \"anthropic/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/m\"]\ngates = [\"non-empty\"]\n";
let config = ProxyConfig::from_lookup(|k| match k {
"FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
"FIRSTPASS_MODE" => Some("enforce".to_owned()),
"FIRSTPASS_UPSTREAM_ANTHROPIC" => Some("http://127.0.0.1:1".to_owned()),
_ => None,
})
.unwrap();
let mut outs = HashMap::new();
outs.insert(
"anthropic/m".to_owned(),
Ok(model_resp("anthropic/m", "gated answer")),
);
let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
map.insert(
"anthropic".to_owned(),
Arc::new(MockProvider::new("anthropic", outs)),
);
let (traces, _rx) = mpsc::channel(64);
let state = AppState {
config: Arc::new(config),
http: reqwest::Client::new(),
providers: ProviderRegistry::from_map(map),
gate_health: Arc::new(GateHealthRegistry::new()),
shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
experiments: Arc::default(),
guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
traces,
adaptive: None,
eprocess: None,
bandit: None,
promoter: None,
verified_cache: None,
predictor: None,
prior: None,
tenant_rate_limiter: None,
spill: None,
};
let body = Bytes::from_static(
br#"{"model":"m","stream":true,"messages":[{"role":"user","content":"hi"}]}"#,
);
let resp = messages(
State(state),
Extension(TenantId("default".to_owned())),
HeaderMap::new(),
body,
)
.await;
assert_eq!(resp.status(), axum::http::StatusCode::OK);
assert!(
resp.headers()
.get(axum::http::header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.is_some_and(|ct| ct.starts_with("text/event-stream")),
"streaming client must get SSE"
);
let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
.await
.unwrap();
let text = String::from_utf8(bytes.to_vec()).unwrap();
assert!(text.contains("event: message_start"));
assert!(text.contains("gated answer"));
assert!(text.contains("event: message_stop"));
}
#[test]
fn parse_openai_request_plain_text() {
let body = br#"{"model":"gpt-4o","max_tokens":256,"messages":[{"role":"user","content":"hello"}]}"#;
let req = parse_openai_request(body, false).expect("must parse");
assert_eq!(req.model, "gpt-4o");
assert_eq!(req.max_tokens, 256);
assert_eq!(req.messages.len(), 1);
assert_eq!(req.messages[0].role, "user");
assert_eq!(req.messages[0].content, Value::String("hello".to_owned()));
assert!(req.system.is_none());
assert_eq!(req.raw, Value::Null);
}
#[test]
fn parse_openai_request_system_message() {
let body = br#"{"model":"gpt-4o","messages":[{"role":"system","content":"be concise"},{"role":"user","content":"hi"}]}"#;
let req = parse_openai_request(body, false).expect("must parse");
assert_eq!(req.system.as_deref(), Some("be concise"));
assert_eq!(req.messages.len(), 1);
assert_eq!(req.messages[0].role, "user");
}
#[test]
fn parse_openai_request_tool_calls_translate_to_tool_use() {
let body = br#"{
"model":"gpt-4o",
"messages":[
{"role":"user","content":"what's the weather?"},
{"role":"assistant","content":null,"tool_calls":[
{"id":"call_1","type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Paris\"}"}}
]},
{"role":"tool","tool_call_id":"call_1","content":"15C, cloudy"}
]
}"#;
let req = parse_openai_request(body, false).expect("must parse");
assert_eq!(req.messages.len(), 3);
let asst = &req.messages[1];
assert_eq!(asst.role, "assistant");
let blocks = asst.content.as_array().expect("content array");
assert_eq!(blocks[0]["type"], "tool_use");
assert_eq!(blocks[0]["name"], "get_weather");
assert_eq!(blocks[0]["id"], "call_1");
assert_eq!(blocks[0]["input"]["city"], "Paris");
let tool_msg = &req.messages[2];
assert_eq!(tool_msg.role, "user");
let result_blocks = tool_msg.content.as_array().expect("result blocks");
assert_eq!(result_blocks[0]["type"], "tool_result");
assert_eq!(result_blocks[0]["tool_use_id"], "call_1");
}
#[test]
fn parse_openai_request_tools_translate_to_anthropic_format() {
let body = br#"{
"model":"gpt-4o",
"messages":[{"role":"user","content":"use a tool"}],
"tools":[{"type":"function","function":{"name":"search","description":"web search","parameters":{"type":"object","properties":{"q":{"type":"string"}}}}}]
}"#;
let req = parse_openai_request(body, false).expect("must parse");
let tools = req.tools.as_array().expect("tools array");
assert_eq!(tools.len(), 1);
assert_eq!(tools[0]["name"], "search");
assert_eq!(tools[0]["description"], "web search");
assert_eq!(tools[0]["input_schema"]["type"], "object");
}
#[test]
fn parse_openai_request_raw_carry_preserves_full_body() {
let body =
br#"{"model":"gpt-4o","max_tokens":100,"messages":[{"role":"user","content":"hi"}]}"#;
let req = parse_openai_request(body, true).expect("must parse");
assert!(req.raw.is_object(), "raw must be the full JSON object");
assert_eq!(req.raw["model"], "gpt-4o");
assert_eq!(req.raw["max_tokens"], 100);
assert!(req.tools.is_null(), "no tools in this request");
}
#[test]
fn parse_openai_request_http_image_returns_none() {
let body = br#"{"model":"gpt-4o","messages":[{"role":"user","content":[
{"type":"text","text":"describe this"},
{"type":"image_url","image_url":{"url":"https://example.com/cat.png"}}
]}]}"#;
let result = parse_openai_request(body, false);
assert!(result.is_none(), "http image URL must fail translation");
}
#[test]
fn parse_openai_request_data_url_image_translates_to_anthropic_base64() {
let body = br#"{"model":"gpt-4o","messages":[{"role":"user","content":[
{"type":"text","text":"describe"},
{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgo="}}
]}]}"#;
let req = parse_openai_request(body, false).expect("data URL must parse");
let blocks = req.messages[0].content.as_array().expect("blocks");
let img = blocks
.iter()
.find(|b| b["type"] == "image")
.expect("image block");
assert_eq!(img["source"]["type"], "base64");
assert_eq!(img["source"]["media_type"], "image/png");
assert_eq!(img["source"]["data"], "iVBORw0KGgo=");
}
#[test]
fn openai_response_json_renders_text_response() {
let resp = ModelResponse {
model: "gpt-4o".to_owned(),
text: "Hello!".to_owned(),
in_tokens: 10,
cache_write_tokens: 0,
cache_read_tokens: 0,
out_tokens: 5,
raw: serde_json::json!({
"content": [{ "type": "text", "text": "Hello!" }]
}),
};
let json = openai_response_json(&resp);
assert_eq!(json["object"], "chat.completion");
assert_eq!(json["model"], "gpt-4o");
assert_eq!(json["choices"][0]["message"]["role"], "assistant");
assert_eq!(json["choices"][0]["message"]["content"], "Hello!");
assert_eq!(json["choices"][0]["finish_reason"], "stop");
assert_eq!(json["usage"]["prompt_tokens"], 10);
assert_eq!(json["usage"]["completion_tokens"], 5);
}
#[test]
fn openai_response_json_renders_tool_call() {
let resp = ModelResponse {
model: "gpt-4o".to_owned(),
text: String::new(),
in_tokens: 20,
cache_write_tokens: 0,
cache_read_tokens: 0,
out_tokens: 15,
raw: serde_json::json!({
"content": [{
"type": "tool_use",
"id": "call_abc",
"name": "search",
"input": {"q": "Rust async"}
}]
}),
};
let json = openai_response_json(&resp);
assert_eq!(json["choices"][0]["finish_reason"], "tool_calls");
let tc = &json["choices"][0]["message"]["tool_calls"][0];
assert_eq!(tc["id"], "call_abc");
assert_eq!(tc["type"], "function");
assert_eq!(tc["function"]["name"], "search");
assert_eq!(json["choices"][0]["message"]["content"], Value::Null);
}
#[test]
fn openai_sse_from_message_plain_text_has_role_then_content_then_stop() {
let resp = ModelResponse {
model: "gpt-4o".to_owned(),
text: "Hi there!".to_owned(),
in_tokens: 5,
cache_write_tokens: 0,
cache_read_tokens: 0,
out_tokens: 3,
raw: serde_json::json!({
"content": [{ "type": "text", "text": "Hi there!" }]
}),
};
let sse = openai_sse_from_message(&openai_response_json(&resp), false);
for line in sse.lines() {
assert!(
line.is_empty() || line.starts_with("data: "),
"bad SSE line: {line:?}"
);
}
let frames: Vec<&str> = sse
.lines()
.filter_map(|l| l.strip_prefix("data: "))
.collect();
assert_eq!(*frames.last().unwrap(), "[DONE]");
let role_frame: Value = serde_json::from_str(frames[0]).unwrap();
assert_eq!(role_frame["choices"][0]["delta"]["role"], "assistant");
assert!(frames.iter().any(|f| {
if *f == "[DONE]" {
return false;
}
serde_json::from_str::<Value>(f)
.ok()
.is_some_and(|v| v["choices"][0]["delta"]["content"] == "Hi there!")
}));
assert!(frames.iter().any(|f| {
if *f == "[DONE]" {
return false;
}
serde_json::from_str::<Value>(f)
.ok()
.is_some_and(|v| v["choices"][0]["finish_reason"] == "stop")
}));
}
#[test]
fn detects_openai_tool_calls() {
let with_tool_calls = Bytes::from_static(br#"{"messages":[
{"role":"user","content":"hi"},
{"role":"assistant","content":null,"tool_calls":[{"id":"c1","type":"function","function":{"name":"f","arguments":"{}"}}]}
]}"#);
let without = Bytes::from_static(br#"{"messages":[{"role":"user","content":"hi"}]}"#);
let with_tool_msg = Bytes::from_static(
br#"{"messages":[
{"role":"tool","tool_call_id":"c1","content":"result"}
]}"#,
);
assert!(openai_messages_have_tool_calls(&with_tool_calls));
assert!(!openai_messages_have_tool_calls(&without));
assert!(openai_messages_have_tool_calls(&with_tool_msg));
}
#[test]
fn detects_openai_http_images() {
let http_img = Bytes::from_static(
br#"{"messages":[{"role":"user","content":[
{"type":"image_url","image_url":{"url":"https://example.com/img.png"}}
]}]}"#,
);
let data_img = Bytes::from_static(
br#"{"messages":[{"role":"user","content":[
{"type":"image_url","image_url":{"url":"data:image/png;base64,abc"}}
]}]}"#,
);
let no_img = Bytes::from_static(br#"{"messages":[{"role":"user","content":"hi"}]}"#);
assert!(openai_has_http_images(&http_img));
assert!(!openai_has_http_images(&data_img));
assert!(!openai_has_http_images(&no_img));
}
#[test]
fn enforce_can_handle_openai_inbound_all_openai_ladder() {
let tools_body = Bytes::from_static(br#"{"model":"gpt-4o","messages":[{"role":"assistant","content":null,"tool_calls":[{"id":"c","type":"function","function":{"name":"f","arguments":"{}"}}]}]}"#);
let f = extract_openai_features(&HeaderMap::new(), &tools_body);
let ladder = vec!["openai/gpt-4o-mini".to_owned(), "openai/gpt-4o".to_owned()];
let providers = crate::provider::ProviderRegistry::new("http://x", "http://x");
assert!(enforce_can_handle(
&f,
&tools_body,
true,
&ladder,
&providers,
Dialect::Openai
));
}
#[test]
fn enforce_can_handle_openai_inbound_all_anthropic_ladder_no_http_image() {
let tools_body = Bytes::from_static(br#"{"model":"gpt-4o","messages":[{"role":"assistant","content":null,"tool_calls":[{"id":"c","type":"function","function":{"name":"f","arguments":"{}"}}]}]}"#);
let f = extract_openai_features(&HeaderMap::new(), &tools_body);
let ladder = vec!["anthropic/claude-haiku-4-5".to_owned()];
let providers = crate::provider::ProviderRegistry::new("http://x", "http://x");
assert!(enforce_can_handle(
&f,
&tools_body,
true,
&ladder,
&providers,
Dialect::Openai,
));
}
#[test]
fn enforce_can_handle_openai_inbound_http_image_falls_back() {
let img_body = Bytes::from_static(
br#"{"model":"gpt-4o","messages":[{"role":"user","content":[
{"type":"image_url","image_url":{"url":"https://example.com/img.png"}}
]}]}"#,
);
let f = extract_openai_features(&HeaderMap::new(), &img_body);
let ladder = vec!["anthropic/claude-haiku-4-5".to_owned()];
let providers = crate::provider::ProviderRegistry::new("http://x", "http://x");
assert!(!enforce_can_handle(
&f,
&img_body,
true,
&ladder,
&providers,
Dialect::Openai,
));
}
fn openai_enforce_state(mock_resp: ModelResponse) -> AppState {
let toml = "[[price]]\nmodel = \"mock/m\"\ninput_per_mtok = 1.0\noutput_per_mtok = 5.0\n[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"mock/m\"]\ngates = [\"non-empty\"]\n";
let config = ProxyConfig::from_lookup(|k| match k {
"FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
"FIRSTPASS_MODE" => Some("enforce".to_owned()),
_ => None,
})
.unwrap();
let mut outs = HashMap::new();
outs.insert("mock/m".to_owned(), Ok(mock_resp));
let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
map.insert("mock".to_owned(), Arc::new(MockProvider::new("mock", outs)));
let (traces, _rx) = mpsc::channel(64);
std::mem::forget(_rx);
let tenant_rate_limiter = build_tenant_rate_limiter(&config);
AppState {
config: Arc::new(config),
http: reqwest::Client::new(),
providers: ProviderRegistry::from_map(map),
gate_health: Arc::new(GateHealthRegistry::new()),
shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
experiments: Arc::default(),
guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
traces,
adaptive: None,
eprocess: None,
bandit: None,
promoter: None,
verified_cache: None,
predictor: None,
prior: None,
tenant_rate_limiter,
spill: None,
}
}
#[tokio::test]
async fn chat_completions_plain_text_enforce_returns_openai_shape() {
let mock = model_resp("mock/m", "gated answer");
let state = openai_enforce_state(mock);
let body = Bytes::from_static(
br#"{"model":"gpt-4o","messages":[{"role":"user","content":"hello"}]}"#,
);
let resp = chat_completions(
State(state),
Extension(TenantId("default".to_owned())),
HeaderMap::new(),
body,
)
.await;
assert_eq!(resp.status(), axum::http::StatusCode::OK);
let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
.await
.unwrap();
let json: Value = serde_json::from_slice(&bytes).expect("must be JSON");
assert_eq!(json["object"], "chat.completion");
assert_eq!(json["choices"][0]["message"]["role"], "assistant");
assert_eq!(json["choices"][0]["message"]["content"], "gated answer");
assert_eq!(json["choices"][0]["finish_reason"], "stop");
assert!(
json["id"]
.as_str()
.is_some_and(|id| id.starts_with("chatcmpl-"))
);
}
#[tokio::test]
async fn chat_completions_stream_true_returns_sse_with_openai_chunks() {
let mock = model_resp("mock/m", "gated answer");
let state = openai_enforce_state(mock);
let body = Bytes::from_static(
br#"{"model":"gpt-4o","stream":true,"messages":[{"role":"user","content":"hello"}]}"#,
);
let resp = chat_completions(
State(state),
Extension(TenantId("default".to_owned())),
HeaderMap::new(),
body,
)
.await;
assert_eq!(resp.status(), axum::http::StatusCode::OK);
assert!(
resp.headers()
.get(axum::http::header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.is_some_and(|ct| ct.starts_with("text/event-stream")),
"stream:true must return SSE"
);
let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
.await
.unwrap();
let text = String::from_utf8(bytes.to_vec()).unwrap();
assert!(
text.contains("chat.completion.chunk"),
"must have OpenAI chunk frames"
);
assert!(text.contains("[DONE]"), "must end with [DONE]");
assert!(
text.contains("gated answer"),
"content must be in the stream"
);
assert!(
!text.contains("message_start"),
"must not have Anthropic event types"
);
}
fn probe_state(
sample_rate: f64,
k: u32,
outcomes: Vec<(&str, Result<ModelResponse, ProviderError>)>,
) -> (AppState, mpsc::Receiver<Trace>) {
let toml = format!(
"[[route]]\nmatch = {{}}\nmode = \"enforce\"\nladder = [\"anthropic/claude-haiku-4-5\"]\ngates = [\"non-empty\"]\n\
[escalation.probe]\nk = {k}\nsample_rate = {sample_rate}\n"
);
let config = ProxyConfig::from_lookup(|k_| match k_ {
"FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
"FIRSTPASS_MODE" => Some("enforce".to_owned()),
_ => None,
})
.unwrap();
let mut outs = HashMap::new();
for (model, out) in outcomes {
outs.insert(model.to_owned(), out);
}
let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
map.insert(
"anthropic".to_owned(),
Arc::new(MockProvider::new("anthropic", outs)),
);
let (traces, rx) = mpsc::channel(64);
let state = AppState {
config: Arc::new(config),
http: reqwest::Client::new(),
providers: ProviderRegistry::from_map(map),
gate_health: Arc::new(GateHealthRegistry::new()),
shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
experiments: Arc::default(),
guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
traces,
adaptive: None,
eprocess: None,
bandit: None,
promoter: None,
verified_cache: None,
predictor: None,
prior: None,
tenant_rate_limiter: None,
spill: None,
};
(state, rx)
}
async fn run_enforce_get_trace(state: AppState, mut rx: mpsc::Receiver<Trace>) -> Trace {
let resp = messages(
State(state),
Extension(TenantId("default".to_owned())),
HeaderMap::new(),
user_body(),
)
.await;
assert_eq!(resp.status(), axum::http::StatusCode::OK);
rx.try_recv().expect("trace must be enqueued")
}
#[tokio::test]
async fn probe_off_trace_has_no_probe_field() {
let (state, rx) = enforce_state(
&["anthropic/claude-haiku-4-5"],
&["non-empty"],
vec![(
"anthropic/claude-haiku-4-5",
Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
)],
);
assert!(
state
.config
.routing
.as_ref()
.unwrap()
.escalation
.probe
.is_none(),
"probe must default to None"
);
let trace = run_enforce_get_trace(state, rx).await;
assert!(
trace.probe.is_none(),
"probe=None config must not set trace.probe"
);
}
#[tokio::test]
async fn probe_sample_rate_zero_never_fires() {
let (state, rx) = probe_state(
0.0,
5,
vec![(
"anthropic/claude-haiku-4-5",
Ok(model_resp("anthropic/claude-haiku-4-5", "hi")),
)],
);
let trace = run_enforce_get_trace(state, rx).await;
assert!(
trace.probe.is_none(),
"sample_rate=0.0 must never set trace.probe"
);
}
#[tokio::test]
async fn probe_on_all_pass_sets_confident_pass() {
let model = "anthropic/claude-haiku-4-5";
let (state, mut rx) = probe_state(1.0, 3, vec![(model, Ok(model_resp(model, "hello")))]);
let resp = messages(
State(state),
Extension(TenantId("default".to_owned())),
HeaderMap::new(),
user_body(),
)
.await;
assert_eq!(resp.status(), axum::http::StatusCode::OK);
let json = body_json(resp).await;
assert_eq!(
json["content"][0]["text"], "hello",
"served content unchanged"
);
let trace = rx.try_recv().expect("trace enqueued");
let sig = trace.probe.expect("probe must be set when sample_rate=1.0");
assert_eq!(sig.k, 3);
assert_eq!(sig.gate_pass_count, 3, "all 3 samples must pass non-empty");
assert_eq!(
sig.regime,
firstpass_core::ProbeRegime::ConfidentPass,
"all-pass → ConfidentPass"
);
assert!(
sig.probe_cost_usd > 0.0,
"k model calls must cost something"
);
}
#[tokio::test]
async fn probe_on_all_fail_sets_confident_fail() {
let model = "anthropic/claude-haiku-4-5";
let (state, mut rx) = probe_state(1.0, 3, vec![(model, Ok(model_resp(model, "")))]);
let resp = messages(
State(state),
Extension(TenantId("default".to_owned())),
HeaderMap::new(),
user_body(),
)
.await;
assert_eq!(resp.status(), axum::http::StatusCode::OK);
let trace = rx.try_recv().expect("trace enqueued");
let sig = trace.probe.expect("probe must be set when sample_rate=1.0");
assert_eq!(
sig.gate_pass_count, 0,
"empty response fails non-empty: all 0 pass"
);
assert_eq!(
sig.regime,
firstpass_core::ProbeRegime::ConfidentFail,
"0 passes → ConfidentFail"
);
}
#[tokio::test]
async fn probe_on_served_output_identical_to_probe_off() {
let model = "anthropic/claude-haiku-4-5";
let mk = |sample_rate: f64| {
probe_state(
sample_rate,
2,
vec![(model, Ok(model_resp(model, "gated answer")))],
)
};
let (state_off, mut rx_off) = mk(0.0);
let resp_off = messages(
State(state_off),
Extension(TenantId("default".to_owned())),
HeaderMap::new(),
user_body(),
)
.await;
let json_off = body_json(resp_off).await;
let trace_off = rx_off.try_recv().unwrap();
let (state_on, mut rx_on) = mk(1.0);
let resp_on = messages(
State(state_on),
Extension(TenantId("default".to_owned())),
HeaderMap::new(),
user_body(),
)
.await;
let json_on = body_json(resp_on).await;
let trace_on = rx_on.try_recv().unwrap();
assert_eq!(
json_off["content"][0]["text"], json_on["content"][0]["text"],
"served text must be identical regardless of probe"
);
assert!(
(trace_off.final_.total_cost_usd - trace_on.final_.total_cost_usd).abs() < 1e-12,
"total_cost_usd must not include probe cost: off={} on={}",
trace_off.final_.total_cost_usd,
trace_on.final_.total_cost_usd
);
assert!(trace_off.probe.is_none());
assert!(trace_on.probe.is_some());
assert!(
trace_on.probe.as_ref().unwrap().probe_cost_usd > 0.0,
"probe cost must be positive"
);
}
#[tokio::test]
async fn probe_does_not_mutate_gate_health() {
let model = "anthropic/claude-haiku-4-5";
let (mut state, rx) = probe_state(1.0, 2, vec![(model, Ok(model_resp(model, "answer")))]);
let registry = GateHealthRegistry::new().with_budget("non-empty", 2, 0.4);
registry.record("default", "non-empty", true);
assert!(
registry.enabled("default", "non-empty"),
"gate must start enabled (window not full yet)"
);
state.gate_health = Arc::new(registry);
let trace = run_enforce_get_trace(state, rx).await;
let sig = trace.probe.expect("probe must fire with sample_rate=1.0");
assert_eq!(sig.k, 2);
assert!(
sig.gate_pass_count <= 2,
"gate_pass_count must be in [0, k]"
);
}
fn predictor_state(
enabled: bool,
outcome: Result<ModelResponse, ProviderError>,
) -> (AppState, mpsc::Receiver<Trace>) {
let toml = "[[route]]\nmatch = {}\nmode = \"enforce\"\nladder = [\"anthropic/claude-haiku-4-5\"]\ngates = [\"non-empty\"]\n";
let config = ProxyConfig::from_lookup(|k_| match k_ {
"FIRSTPASS_CONFIG_TOML" => Some(toml.to_owned()),
"FIRSTPASS_MODE" => Some("enforce".to_owned()),
_ => None,
})
.unwrap();
let mut outs = HashMap::new();
outs.insert("anthropic/claude-haiku-4-5".to_owned(), outcome);
let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
map.insert(
"anthropic".to_owned(),
Arc::new(MockProvider::new("anthropic", outs)),
);
let (traces, rx) = mpsc::channel(64);
let predictor = enabled.then(|| {
Arc::new(std::sync::Mutex::new(firstpass_core::PassPredictor::new(
0.05, 1e-4,
)))
});
let state = AppState {
config: Arc::new(config),
http: reqwest::Client::new(),
providers: ProviderRegistry::from_map(map),
gate_health: Arc::new(GateHealthRegistry::new()),
shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
experiments: Arc::default(),
guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
traces,
adaptive: None,
eprocess: None,
bandit: None,
promoter: None,
verified_cache: None,
predictor,
prior: None,
tenant_rate_limiter: None,
spill: None,
};
(state, rx)
}
#[tokio::test]
async fn predictor_off_leaves_predicted_pass_none() {
let (state, rx) =
predictor_state(false, Ok(model_resp("anthropic/claude-haiku-4-5", "ok")));
let trace = run_enforce_get_trace(state, rx).await;
assert!(
trace.predicted_pass.is_none(),
"predictor off => no field (byte-identical)"
);
let j = serde_json::to_string(&trace).unwrap();
assert!(!j.contains("predicted_pass"), "None must be omitted: {j}");
}
#[tokio::test]
async fn predictor_on_records_shadow_prediction_and_serves_identically() {
let (state_off, rx_off) = predictor_state(
false,
Ok(model_resp("anthropic/claude-haiku-4-5", "served answer")),
);
let off = run_enforce_get_trace(state_off, rx_off).await;
let (state_on, rx_on) = predictor_state(
true,
Ok(model_resp("anthropic/claude-haiku-4-5", "served answer")),
);
let on = run_enforce_get_trace(state_on, rx_on).await;
assert_eq!(
on.final_.served_rung, off.final_.served_rung,
"served rung identical"
);
assert_eq!(on.attempts.len(), off.attempts.len(), "same attempts");
assert_eq!(
on.final_.total_cost_usd, off.final_.total_cost_usd,
"predictor never adds served cost"
);
let p = on
.predicted_pass
.expect("predictor on => predicted_pass recorded");
assert!(p > 0.0 && p < 1.0, "shadow prediction in (0,1): {p}");
}
fn rollout_state(
percent: f64,
key: &str,
outcomes: Vec<(&str, Result<ModelResponse, ProviderError>)>,
) -> (AppState, mpsc::Receiver<Trace>) {
let toml = format!(
"[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
ladder = [\"anthropic/claude-haiku-4-5\"]\ngates = [\"non-empty\"]\n\
[route.rollout]\npercent = {percent}\nkey = \"{key}\"\n"
);
let config = ProxyConfig::from_lookup(|k| match k {
"FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
"FIRSTPASS_MODE" => Some("enforce".to_owned()),
_ => None,
})
.unwrap();
let mut outs = HashMap::new();
for (model, out) in outcomes {
outs.insert(model.to_owned(), out);
}
let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
map.insert(
"anthropic".to_owned(),
Arc::new(MockProvider::new("anthropic", outs)),
);
let (traces, rx) = mpsc::channel(64);
let state = AppState {
config: Arc::new(config),
http: reqwest::Client::new(),
providers: ProviderRegistry::from_map(map),
gate_health: Arc::new(GateHealthRegistry::new()),
shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
experiments: Arc::default(),
guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
traces,
adaptive: None,
eprocess: None,
bandit: None,
promoter: None,
verified_cache: None,
predictor: None,
prior: None,
tenant_rate_limiter: None,
spill: None,
};
(state, rx)
}
#[tokio::test]
async fn rollout_at_zero_percent_never_enforces() {
for i in 0..25 {
let (state, mut rx) = rollout_state(
0.0,
"session",
vec![(
"anthropic/claude-haiku-4-5",
Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
)],
);
let mut headers = HeaderMap::new();
headers.insert("x-firstpass-session", format!("s{i}").parse().unwrap());
let resp = messages(
State(state),
Extension(TenantId("default".to_owned())),
headers,
user_body(),
)
.await;
let _ = resp;
if let Ok(t) = rx.try_recv() {
assert!(
t.attempts.is_empty(),
"0% rollout produced an enforced decision for session s{i}"
);
}
}
}
#[tokio::test]
async fn rollout_at_hundred_percent_always_enforces() {
for i in 0..25 {
let (state, mut rx) = rollout_state(
100.0,
"session",
vec![(
"anthropic/claude-haiku-4-5",
Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
)],
);
let mut headers = HeaderMap::new();
headers.insert("x-firstpass-session", format!("s{i}").parse().unwrap());
let resp = messages(
State(state),
Extension(TenantId("default".to_owned())),
headers,
user_body(),
)
.await;
assert_eq!(resp.status(), axum::http::StatusCode::OK);
let t = rx.try_recv().expect("trace enqueued");
assert!(
!t.attempts.is_empty(),
"100% rollout skipped enforcement for session s{i}"
);
}
}
#[tokio::test]
async fn dispatch_arm_matches_the_predicted_arm_per_session() {
let mut checked_enforced = 0;
let mut checked_control = 0;
for i in 0..24 {
let session = format!("session-{i}");
let (state, mut rx) = rollout_state(
50.0,
"session",
vec![(
"anthropic/claude-haiku-4-5",
Ok(model_resp("anthropic/claude-haiku-4-5", "hello")),
)],
);
let expected = firstpass_core::rollout::decide(
&state.config.prompt_salt,
&firstpass_core::Rollout {
percent: 50.0,
key: firstpass_core::RolloutKey::Session,
},
&session,
)
.enforced;
let mut headers = HeaderMap::new();
headers.insert("x-firstpass-session", session.parse().unwrap());
let _ = messages(
State(state),
Extension(TenantId("default".to_owned())),
headers,
user_body(),
)
.await;
let observed = rx
.try_recv()
.map(|t| !t.attempts.is_empty())
.unwrap_or(false);
assert_eq!(
observed,
expected,
"{session}: dispatch put it in the {} arm, bucketing predicted the {} arm",
if observed { "enforced" } else { "control" },
if expected { "enforced" } else { "control" }
);
if expected {
checked_enforced += 1;
} else {
checked_control += 1;
}
}
assert!(
checked_enforced > 0 && checked_control > 0,
"test saw only one arm ({checked_enforced} enforced, {checked_control} control)"
);
}
const INCUMBENT: &str = "anthropic/claude-sonnet-5";
const CHALLENGER: &str = "anthropic/claude-haiku-4-5";
fn challenger_state(percent: f64, db: &str) -> (AppState, mpsc::Receiver<Trace>) {
let toml = format!(
"[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
ladder = [\"{INCUMBENT}\"]\ngates = [\"non-empty\"]\n\
[route.challenger]\nladder = [\"{CHALLENGER}\"]\npercent = {percent}\n"
);
let db = db.to_owned();
let config = ProxyConfig::from_lookup(|k| match k {
"FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
"FIRSTPASS_MODE" => Some("enforce".to_owned()),
"FIRSTPASS_DB" => Some(db.clone()),
_ => None,
})
.unwrap();
let mut outs = HashMap::new();
for m in [INCUMBENT, CHALLENGER] {
outs.insert(m.to_owned(), Ok(model_resp(m, "answer")));
}
let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
map.insert(
"anthropic".to_owned(),
Arc::new(MockProvider::new("anthropic", outs)),
);
let experiments = Arc::new(crate::experiment::ExperimentMonitor::from_config(
config.routing.as_ref().unwrap(),
));
let (traces, rx) = mpsc::channel(64);
let state = AppState {
config: Arc::new(config),
http: reqwest::Client::new(),
providers: ProviderRegistry::from_map(map),
gate_health: Arc::new(GateHealthRegistry::new()),
shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
experiments,
guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
traces,
adaptive: None,
eprocess: None,
bandit: None,
promoter: None,
verified_cache: None,
predictor: None,
prior: None,
tenant_rate_limiter: None,
spill: None,
};
(state, rx)
}
async fn serve_session(
state: &AppState,
rx: &mut mpsc::Receiver<Trace>,
session: &str,
) -> Trace {
let mut headers = HeaderMap::new();
headers.insert("x-firstpass-session", session.parse().unwrap());
let resp = messages(
State(state.clone()),
Extension(TenantId("default".to_owned())),
headers,
user_body(),
)
.await;
assert_eq!(resp.status(), axum::http::StatusCode::OK);
rx.try_recv().expect("trace enqueued")
}
fn session_in_arm(state: &AppState, want: firstpass_core::Arm, nth: usize) -> String {
let ch = state.config.routing.as_ref().unwrap().routes[0]
.challenger
.as_ref()
.unwrap();
(0..)
.map(|i| format!("sess-{i}"))
.filter(|k| ch.arm(&state.config.prompt_salt, k) == want)
.nth(nth)
.unwrap()
}
#[tokio::test]
async fn challenger_arm_serves_its_ladder_and_both_arms_are_recorded() {
use firstpass_core::Arm;
let (state, mut rx) = challenger_state(50.0, "unused.db");
for want in [Arm::Challenger, Arm::Incumbent] {
for n in 0..5 {
let session = session_in_arm(&state, want, n);
let t = serve_session(&state, &mut rx, &session).await;
let rec = t.experiment.as_ref().expect("experiment recorded");
assert_eq!(rec.arm, want, "{session}");
assert!((rec.percent - 50.0).abs() < f64::EPSILON);
let expect_model = if want == Arm::Challenger {
CHALLENGER
} else {
INCUMBENT
};
assert_eq!(t.attempts[0].model, expect_model, "{session}");
assert_eq!(t.route_ix, Some(0));
assert!(
t.policy.id == "challenger-ladder@v0" || want == Arm::Incumbent,
"challenger arm must not claim the incumbent's start-rung policy: {}",
t.policy.id
);
}
}
let reports = state.experiments.reports();
assert_eq!(reports[0].report.challenger.requests, 5);
assert_eq!(reports[0].report.incumbent.requests, 5);
}
#[tokio::test]
async fn halted_experiment_stops_serving_the_challenger() {
use firstpass_core::{Arm, ExperimentStatus, Verdict};
let (state, mut rx) = challenger_state(50.0, "unused.db");
for i in 0..500 {
state.experiments.observe_outcome(
"default",
0,
state.experiments.fingerprint(0).unwrap(),
Arm::Incumbent,
&format!("i{i}"),
Verdict::Pass,
);
state.experiments.observe_outcome(
"default",
0,
state.experiments.fingerprint(0).unwrap(),
Arm::Challenger,
&format!("c{i}"),
Verdict::Fail,
);
}
assert_eq!(
state.experiments.status("default", 0),
ExperimentStatus::Halted
);
let session = session_in_arm(&state, Arm::Challenger, 0);
let t = serve_session(&state, &mut rx, &session).await;
assert_eq!(
t.attempts[0].model, INCUMBENT,
"halted challenger was still served"
);
assert_eq!(t.experiment.unwrap().arm, Arm::Incumbent);
}
#[tokio::test]
async fn shared_log_wiring_halt_propagates_to_replica_b() {
use firstpass_core::{Arm, ExperimentStatus, Verdict};
let shared_log = Arc::new(crate::experiment::InMemoryExperimentLog::new());
let (mut state_a, _rx_a) = challenger_state(50.0, "unused-a.db");
{
let monitor = crate::experiment::ExperimentMonitor::from_config(
state_a.config.routing.as_ref().unwrap(),
)
.with_log(Arc::clone(&shared_log) as Arc<dyn crate::experiment::ExperimentLogBackend>);
state_a.experiments = Arc::new(monitor);
}
let (mut state_b, mut rx_b) = challenger_state(50.0, "unused-b.db");
{
let monitor = crate::experiment::ExperimentMonitor::from_config(
state_b.config.routing.as_ref().unwrap(),
)
.with_log(Arc::clone(&shared_log) as Arc<dyn crate::experiment::ExperimentLogBackend>);
state_b.experiments = Arc::new(monitor);
}
let fp = state_a.experiments.fingerprint(0).unwrap().to_owned();
let mut halted = false;
for i in 0..500 {
state_a
.experiments
.record_outcome(
"default",
0,
&fp,
Arm::Incumbent,
&format!("i{i}"),
Verdict::Pass,
0.10,
)
.await;
state_a
.experiments
.record_outcome(
"default",
0,
&fp,
Arm::Challenger,
&format!("c{i}"),
Verdict::Fail,
0.02,
)
.await;
if state_a.experiments.status("default", 0) == ExperimentStatus::Halted {
halted = true;
break;
}
}
assert!(halted, "experiment never halted on replica A");
assert!(
!state_b.experiments.challenger_blocked("default", 0),
"replica B must not block before syncing"
);
let _ = state_b
.experiments
.sync_from_backend("default", 0, &fp)
.await;
assert!(
state_b.experiments.challenger_blocked("default", 0),
"replica B must block the challenger after syncing the halt from the shared log"
);
let ch_session = session_in_arm(&state_b, Arm::Challenger, 999);
let t = serve_session(&state_b, &mut rx_b, &ch_session).await;
assert_eq!(
t.experiment.as_ref().unwrap().arm,
Arm::Incumbent,
"a halted challenger-arm session must be rerouted to the incumbent on replica B"
);
assert_eq!(
t.attempts[0].model, INCUMBENT,
"the incumbent model must have been called, not the challenger"
);
}
#[tokio::test]
async fn unavailable_experiment_history_serves_the_incumbent_only() {
use firstpass_core::Arm;
let (mut state, mut rx) = challenger_state(50.0, "unused.db");
state.experiments = Arc::new(crate::experiment::ExperimentMonitor::unavailable(
state.config.routing.as_ref().unwrap(),
));
for n in 0..3 {
let session = session_in_arm(&state, Arm::Challenger, n);
let t = serve_session(&state, &mut rx, &session).await;
assert_eq!(t.attempts[0].model, INCUMBENT);
assert_eq!(t.experiment.unwrap().arm, Arm::Incumbent);
}
}
#[tokio::test]
async fn challenger_arm_cannot_share_the_incumbents_cache_entry() {
use firstpass_core::Arm;
let (mut state, mut rx) = challenger_state(50.0, "unused.db");
state.verified_cache = Some(Arc::new(crate::verified_cache::VerifiedCache::new(
Duration::from_secs(60),
100,
)));
let a = serve_session(&state, &mut rx, &session_in_arm(&state, Arm::Incumbent, 0)).await;
assert!(!a.attempts.is_empty());
let b = serve_session(&state, &mut rx, &session_in_arm(&state, Arm::Challenger, 0)).await;
assert!(
!b.attempts.is_empty(),
"challenger replayed the incumbent's cached answer"
);
assert_eq!(b.attempts[0].model, CHALLENGER);
let c = serve_session(&state, &mut rx, &session_in_arm(&state, Arm::Incumbent, 1)).await;
assert!(
c.attempts.is_empty(),
"incumbent should have been a cache hit"
);
assert_eq!(c.experiment.as_ref().unwrap().arm, Arm::Incumbent);
assert_eq!(c.route_ix, Some(0));
let d = serve_session(&state, &mut rx, &session_in_arm(&state, Arm::Challenger, 1)).await;
assert!(d.attempts.is_empty(), "challenger should hit its own entry");
assert_eq!(d.experiment.as_ref().unwrap().arm, Arm::Challenger);
}
#[tokio::test]
async fn feedback_feeds_the_experiment_once_per_session() {
use firstpass_core::Arm;
let db = std::env::temp_dir().join(format!("firstpass-exp-{}.db", Uuid::now_v7()));
let (state, mut rx) = challenger_state(50.0, &db.to_string_lossy());
let session = session_in_arm(&state, Arm::Challenger, 0);
let t = serve_session(&state, &mut rx, &session).await;
let trace_id = t.trace_id.to_string();
let (tx, handle) = crate::store::open(&db).unwrap();
tx.try_send(t).unwrap();
drop(tx);
handle.await.unwrap();
let send = |verdict: &'static str| {
let state = state.clone();
let body = Bytes::from(
serde_json::json!({
"trace_id": trace_id, "gate_id": "tests", "verdict": verdict, "reporter": "ci"
})
.to_string(),
);
async move {
feedback(
State(state),
Extension(TenantId("default".to_owned())),
body,
)
.await
}
};
assert_eq!(
send("pass").await.status(),
axum::http::StatusCode::ACCEPTED
);
assert_eq!(
send("fail").await.status(),
axum::http::StatusCode::ACCEPTED
);
assert_eq!(
send("abstain").await.status(),
axum::http::StatusCode::ACCEPTED
);
let r = &state.experiments.reports()[0].report;
assert_eq!(r.challenger.outcomes, 1);
assert!(
(r.challenger.outcome_sum - 1.0).abs() < f64::EPSILON,
"first outcome (pass) counts"
);
assert_eq!(r.incumbent.outcomes, 0);
let _ = std::fs::remove_file(&db);
}
async fn spawn_fake_prior_server(body: serde_json::Value) -> String {
use axum::routing::post;
let app = axum::Router::new().route(
"/v1/systemone",
post(move || {
let body = body.clone();
async move { axum::Json(body) }
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("bind");
let addr = listener.local_addr().expect("addr");
tokio::spawn(async move {
let _ = axum::serve(listener, app).await;
});
format!("http://{addr}")
}
fn prior_client(base_url: String, rungs: &[&str]) -> crate::prior::PriorClient {
let cfg = firstpass_core::PriorConfig {
provider: "typesafe".to_owned(),
model: "jev-latest".to_owned(),
api_key_env: "TYPESAFE_API_KEY".to_owned(),
base_url,
timeout_ms: 2000,
strength: 10.0,
rungs: rungs.iter().map(|s| (*s).to_owned()).collect(),
};
crate::prior::PriorClient::new(reqwest::Client::new(), cfg, "test-key".to_owned())
}
const OPUS: &str = "anthropic/claude-opus-4-8";
#[tokio::test]
async fn confident_prior_starts_above_zero_and_records_decision_prior() {
let base_url = spawn_fake_prior_server(serde_json::json!({
"answers": { "rung": { "choice": "r2", "confidence": 0.9,
"probabilities": { "r0": 0.05, "r1": 0.05, "r2": 0.9 } } }
}))
.await;
let (mut state, rx) = enforce_state(
&[
"anthropic/claude-haiku-4-5",
"anthropic/claude-sonnet-5",
OPUS,
],
&["non-empty"],
vec![
(
"anthropic/claude-haiku-4-5",
Ok(model_resp("anthropic/claude-haiku-4-5", "ok")),
),
(
"anthropic/claude-sonnet-5",
Ok(model_resp("anthropic/claude-sonnet-5", "ok")),
),
(OPUS, Ok(model_resp(OPUS, "ok"))),
],
);
state.prior = Some(Arc::new(prior_client(
base_url,
&[
"anthropic/claude-haiku-4-5",
"anthropic/claude-sonnet-5",
OPUS,
],
)));
let trace = run_enforce_get_trace(state, rx).await;
assert!(
!trace.attempts.is_empty(),
"at least one attempt must be recorded"
);
assert!(
trace.attempts[0].rung >= 1,
"a confident hard prior (rung 0/1 nearly hopeless) must start above rung 0, started \
at {}",
trace.attempts[0].rung
);
let decision_prior = trace
.decision_prior
.as_ref()
.expect("a successful prior fetch must record decision_prior");
assert_eq!(decision_prior.len(), 3);
assert!(
decision_prior[2] > decision_prior[0],
"cumulative prior must be monotone non-decreasing: {decision_prior:?}"
);
}
#[tokio::test]
async fn prior_server_error_fails_open_start_rung_zero_no_decision_prior() {
let unreachable = "http://127.0.0.1:1".to_owned();
let (mut state, rx) = enforce_state(
&["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"],
&["non-empty"],
vec![(
"anthropic/claude-haiku-4-5",
Ok(model_resp("anthropic/claude-haiku-4-5", "ok")),
)],
);
state.prior = Some(Arc::new(prior_client(
unreachable,
&["anthropic/claude-haiku-4-5", "anthropic/claude-sonnet-5"],
)));
let trace = run_enforce_get_trace(state, rx).await;
assert_eq!(
trace.attempts[0].rung, 0,
"a failed prior fetch must fail open to the static start rung 0"
);
assert!(
trace.decision_prior.is_none(),
"a failed prior fetch must leave decision_prior unset"
);
}
#[tokio::test]
async fn gate_failed_start_rung_is_never_served_with_prior_on() {
let base_url = spawn_fake_prior_server(serde_json::json!({
"answers": { "rung": { "choice": "r2", "confidence": 0.9,
"probabilities": { "r0": 0.05, "r1": 0.05, "r2": 0.9 } } }
}))
.await;
let (mut state, rx) = enforce_state(
&[
"anthropic/claude-haiku-4-5",
"anthropic/claude-sonnet-5",
OPUS,
],
&["non-empty"],
vec![
(
"anthropic/claude-haiku-4-5",
Ok(model_resp("anthropic/claude-haiku-4-5", "")),
),
(
"anthropic/claude-sonnet-5",
Ok(model_resp("anthropic/claude-sonnet-5", "")),
),
(OPUS, Ok(model_resp(OPUS, "ok"))),
],
);
state.prior = Some(Arc::new(prior_client(
base_url,
&[
"anthropic/claude-haiku-4-5",
"anthropic/claude-sonnet-5",
OPUS,
],
)));
let trace = run_enforce_get_trace(state, rx).await;
assert_eq!(
trace.final_.served_rung,
Some(2),
"only the top rung passes the gate, so it must be what's served, regardless of \
which rung the prior started at"
);
for attempt in &trace.attempts[..trace.attempts.len() - 1] {
assert_eq!(
attempt.verdict,
firstpass_core::Verdict::Fail,
"every attempt before the served one must be a recorded gate failure, not a \
silently-served output"
);
}
assert_eq!(
trace.attempts.last().unwrap().verdict,
firstpass_core::Verdict::Pass
);
}
fn ux_state(db: &str) -> AppState {
let (mut state, _rx) = challenger_state(50.0, db);
let hash_a = crate::tenant_auth::TenantKeys::hash_key("key-a").unwrap();
let hash_b = crate::tenant_auth::TenantKeys::hash_key("key-b").unwrap();
let keys = format!("{{\"tenant-a\": {hash_a:?}, \"tenant-b\": {hash_b:?}}}");
let toml = format!(
"[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
ladder = [\"{INCUMBENT}\"]\ngates = [\"non-empty\"]\n\
[route.challenger]\nladder = [\"{CHALLENGER}\"]\npercent = 50\n"
);
let db = db.to_owned();
state.config = Arc::new(
ProxyConfig::from_lookup(|k| match k {
"FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
"FIRSTPASS_MODE" => Some("enforce".to_owned()),
"FIRSTPASS_DB" => Some(db.clone()),
"FIRSTPASS_REQUIRE_AUTH" => Some("true".to_owned()),
"FIRSTPASS_TENANT_KEYS_JSON" => Some(keys.clone()),
_ => None,
})
.unwrap(),
);
let (traces, rx) = mpsc::channel(256);
std::mem::forget(rx);
state.traces = traces;
state
}
#[tokio::test]
async fn paired_ledger_is_seeded_from_todays_pairs_at_startup() {
let db = std::env::temp_dir().join(format!("firstpass-ledger-{}.db", Uuid::now_v7()));
let (tx, handle) = crate::store::open(&db).unwrap();
drop(tx);
handle.await.unwrap();
let now = jiff::Timestamp::now();
let row = |tenant: &str, usd: f64, ts: jiff::Timestamp| crate::store::ExperimentPairRow {
tenant: tenant.to_owned(),
route_ix: 0,
fingerprint: "fp".to_owned(),
trace_id: Uuid::now_v7(),
x_inc: 1.0,
x_ch: 1.0,
challenger_cost_usd: usd,
ch_error: false,
ch_truncated: 0,
ts,
in_log: crate::store::InLogStatus::InLog,
};
let two_days_ago = now - jiff::SignedDuration::from_hours(48);
for r in [
row("tenant-a", 0.40, now),
row("tenant-a", 0.70, now),
row("tenant-a", 9.00, two_days_ago),
row("tenant-b", 0.10, now),
] {
crate::store::append_experiment_pair(&db, &r).unwrap();
}
let toml = format!(
"[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
ladder = [\"{INCUMBENT}\"]\ngates = [\"json-valid\"]\n\
[route.challenger]\nladder = [\"{CHALLENGER}\"]\nmode = \"paired\"\n\
max_usd_per_day = 1.0\n"
);
let db_s = db.to_string_lossy().into_owned();
let cfg = ProxyConfig::from_lookup(|k| match k {
"FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
"FIRSTPASS_DB" => Some(db_s.clone()),
_ => None,
})
.unwrap();
let ledger = build_paired_ledger(&cfg, false);
assert!((ledger.spent_today("tenant-a", 0, now) - 1.10).abs() < 1e-9);
assert!((ledger.spent_today("tenant-b", 0, now) - 0.10).abs() < 1e-9);
assert!(
!ledger.may_spend("tenant-a", 0, 1.0, now),
"cap survives restart"
);
assert!(ledger.may_spend("tenant-b", 0, 1.0, now));
let _ = std::fs::remove_file(&db);
}
#[tokio::test]
async fn build_paired_ledger_seeds_offline_pairs_when_shared_log_is_true() {
let db =
std::env::temp_dir().join(format!("firstpass-offline-ledger-{}.db", Uuid::now_v7()));
let (tx, handle) = crate::store::open(&db).unwrap();
drop(tx);
handle.await.unwrap();
let now = jiff::Timestamp::now();
let toml = format!(
"[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
ladder = [\"{INCUMBENT}\"]\ngates = [\"json-valid\"]\n\
[route.challenger]\nladder = [\"{CHALLENGER}\"]\nmode = \"paired\"\n\
max_usd_per_day = 2.0\n"
);
let db_s = db.to_string_lossy().into_owned();
let cfg = ProxyConfig::from_lookup(|k| match k {
"FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
"FIRSTPASS_DB" => Some(db_s.clone()),
_ => None,
})
.unwrap();
let offline_row = crate::store::ExperimentPairRow {
tenant: "default".to_owned(),
route_ix: 0,
fingerprint: "fp".to_owned(),
trace_id: Uuid::now_v7(),
x_inc: 1.0,
x_ch: 0.0,
challenger_cost_usd: 0.7,
ch_error: false,
ch_truncated: 0,
ts: now,
in_log: crate::store::InLogStatus::NotInLog,
};
let log_row = crate::store::ExperimentPairRow {
in_log: crate::store::InLogStatus::InLog,
challenger_cost_usd: 0.4,
trace_id: Uuid::now_v7(),
..offline_row.clone()
};
crate::store::append_experiment_pair(&db, &offline_row).unwrap();
crate::store::append_experiment_pair(&db, &log_row).unwrap();
let ledger = build_paired_ledger(&cfg, true);
let spent = ledger.spent_today("default", 0, now);
assert!(
(spent - 0.7).abs() < 1e-9,
"only offline row seeded; got {spent}"
);
let ledger2 = build_paired_ledger(&cfg, false);
let spent2 = ledger2.spent_today("default", 0, now);
assert!(
(spent2 - 1.1).abs() < 1e-9,
"both rows seeded with shared_log=false; got {spent2}"
);
let _ = std::fs::remove_file(&db);
}
#[tokio::test]
async fn build_paired_ledger_unknown_rows_count_toward_spend_with_shared_log() {
let db =
std::env::temp_dir().join(format!("firstpass-unknown-ledger-{}.db", Uuid::now_v7()));
let (tx, handle) = crate::store::open(&db).unwrap();
drop(tx);
handle.await.unwrap();
let now = jiff::Timestamp::now();
let toml = format!(
"[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
ladder = [\"{INCUMBENT}\"]\ngates = [\"json-valid\"]\n\
[route.challenger]\nladder = [\"{CHALLENGER}\"]\nmode = \"paired\"\n\
max_usd_per_day = 2.0\n"
);
let db_s = db.to_string_lossy().into_owned();
let cfg = ProxyConfig::from_lookup(|k| match k {
"FIRSTPASS_CONFIG_TOML" => Some(toml.clone()),
"FIRSTPASS_DB" => Some(db_s.clone()),
_ => None,
})
.unwrap();
let unknown_row = crate::store::ExperimentPairRow {
tenant: "default".to_owned(),
route_ix: 0,
fingerprint: "fp".to_owned(),
trace_id: Uuid::now_v7(),
x_inc: 1.0,
x_ch: 0.0,
challenger_cost_usd: 0.6,
ch_error: false,
ch_truncated: 0,
ts: now,
in_log: crate::store::InLogStatus::Unknown,
};
let log_row = crate::store::ExperimentPairRow {
in_log: crate::store::InLogStatus::InLog,
challenger_cost_usd: 0.3,
trace_id: Uuid::now_v7(),
..unknown_row.clone()
};
crate::store::append_experiment_pair(&db, &unknown_row).unwrap();
crate::store::append_experiment_pair(&db, &log_row).unwrap();
let ledger = build_paired_ledger(&cfg, true);
let spent = ledger.spent_today("default", 0, now);
assert!(
(spent - 0.6).abs() < 1e-9,
"Unknown row seeded (conservative); InLog row skipped; got {spent}"
);
let _ = std::fs::remove_file(&db);
}
#[tokio::test]
async fn experiments_endpoint_reports_paired_mode_and_tenant_spend() {
use tower::ServiceExt;
let db = std::env::temp_dir().join(format!("firstpass-ux-paired-{}.db", Uuid::now_v7()));
let mut state = ux_state(&db.to_string_lossy());
let toml = format!(
"[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
ladder = [\"{INCUMBENT}\"]\ngates = [\"json-valid\"]\n\
[route.challenger]\nladder = [\"{CHALLENGER}\"]\nmode = \"paired\"\n\
sample_rate = 0.25\nmax_usd_per_day = 2.0\n"
);
let mut cfg = (*state.config).clone();
cfg.routing = Some(firstpass_core::Config::parse(&toml).expect("paired config parses"));
state.config = Arc::new(cfg);
let now = jiff::Timestamp::now();
state.paired_ledger.debit("tenant-a", 0, 0.5, now);
state.paired_ledger.debit("tenant-b", 0, 1.25, now);
let router = app(state).expect("router");
let resp = router
.clone()
.oneshot(ux_get("/v1/experiments", Some("tenant-a.key-a")))
.await
.unwrap();
assert_eq!(resp.status(), axum::http::StatusCode::OK);
let a = body_json(resp).await;
let c = &a["configured"][0];
assert_eq!(c["mode"], "paired", "{a}");
assert_eq!(c["paired"]["sample_rate"], 0.25);
assert_eq!(c["paired"]["max_usd_per_day"], 2.0);
assert_eq!(c["paired"]["spent_today_usd"], 0.5, "tenant-a's spend only");
let resp = router
.oneshot(ux_get("/v1/experiments", Some("tenant-b.key-b")))
.await
.unwrap();
let b = body_json(resp).await;
assert_eq!(b["configured"][0]["paired"]["spent_today_usd"], 1.25);
let _ = std::fs::remove_file(&db);
}
#[tokio::test]
async fn paired_challenger_that_fails_its_gate_is_recorded_as_a_failure() {
use tower::ServiceExt;
let db = std::env::temp_dir().join(format!("firstpass-paired-fail-{}.db", Uuid::now_v7()));
let (tx, handle) = crate::store::open(&db).unwrap();
drop(tx);
handle.await.unwrap();
let (mut state, _rx) = challenger_state(50.0, &db.to_string_lossy());
let toml = format!(
"[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
ladder = [\"{INCUMBENT}\"]\ngates = [\"json-valid\"]\n\
[route.challenger]\nladder = [\"{CHALLENGER}\"]\nmode = \"paired\"\n\
max_usd_per_day = 5.0\n"
);
let mut cfg = (*state.config).clone();
cfg.routing = Some(firstpass_core::Config::parse(&toml).expect("paired config parses"));
state.experiments = Arc::new(crate::experiment::ExperimentMonitor::from_config(
cfg.routing.as_ref().unwrap(),
));
state.config = Arc::new(cfg);
let mut outs = HashMap::new();
outs.insert(
INCUMBENT.to_owned(),
Ok(model_resp(INCUMBENT, "{\"ok\": 1}")),
);
outs.insert(
CHALLENGER.to_owned(),
Ok(model_resp(CHALLENGER, "not json")),
);
let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
map.insert(
"anthropic".to_owned(),
Arc::new(MockProvider::new("anthropic", outs)),
);
state.providers = ProviderRegistry::from_map(map);
let router = app(state).expect("router");
let req = axum::http::Request::builder()
.method("POST")
.uri("/v1/messages")
.body(Body::from(user_body()))
.unwrap();
let resp = router.oneshot(req).await.unwrap();
assert_eq!(resp.status(), axum::http::StatusCode::OK);
let mut pairs = Vec::new();
for _ in 0..200 {
pairs = crate::store::load_all_experiment_pairs(&db).unwrap_or_default();
if !pairs.is_empty() {
break;
}
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
assert_eq!(pairs.len(), 1, "one pair recorded");
assert_eq!(pairs[0].x_inc, 1.0, "incumbent passed json-valid");
assert_eq!(
pairs[0].x_ch, 0.0,
"challenger failed json-valid on every rung"
);
assert_eq!(
pairs[0].in_log,
crate::store::InLogStatus::NotInLog,
"no shared log is configured, so the pair cannot be in one"
);
let _ = std::fs::remove_file(&db);
}
#[tokio::test]
async fn paired_challenger_error_is_a_failure_and_arm_health_is_reported() {
use tower::ServiceExt;
let db = std::env::temp_dir().join(format!("firstpass-paired-err-{}.db", Uuid::now_v7()));
let (tx, handle) = crate::store::open(&db).unwrap();
drop(tx);
handle.await.unwrap();
let (mut state, _rx) = challenger_state(50.0, &db.to_string_lossy());
let toml = format!(
"[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
ladder = [\"{INCUMBENT}\"]\ngates = [\"json-valid\"]\n\
[route.challenger]\nladder = [\"{CHALLENGER}\"]\nmode = \"paired\"\n\
max_usd_per_day = 5.0\n"
);
let mut cfg = (*state.config).clone();
cfg.routing = Some(firstpass_core::Config::parse(&toml).expect("paired config parses"));
state.experiments = Arc::new(crate::experiment::ExperimentMonitor::from_config(
cfg.routing.as_ref().unwrap(),
));
state.config = Arc::new(cfg);
let mut cut = model_resp(INCUMBENT, "{\"ok\": 1}");
cut.raw = serde_json::json!({"stop_reason": "max_tokens"});
let mut outs = HashMap::new();
outs.insert(INCUMBENT.to_owned(), Ok(cut));
outs.insert(
CHALLENGER.to_owned(),
Err(ProviderError::Transport("timed out".into())),
);
let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
map.insert(
"anthropic".to_owned(),
Arc::new(MockProvider::new("anthropic", outs)),
);
state.providers = ProviderRegistry::from_map(map);
let monitor = state.experiments.clone();
let router = app(state).expect("router");
let req = axum::http::Request::builder()
.method("POST")
.uri("/v1/messages")
.body(Body::from(user_body()))
.unwrap();
let resp = router.oneshot(req).await.unwrap();
assert_eq!(resp.status(), axum::http::StatusCode::OK);
let mut pairs = Vec::new();
for _ in 0..200 {
pairs = crate::store::load_all_experiment_pairs(&db).unwrap_or_default();
if !pairs.is_empty() {
break;
}
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
assert_eq!(
pairs.len(),
1,
"the challenger's error still records a pair"
);
assert_eq!(pairs[0].x_inc, 1.0);
assert_eq!(pairs[0].x_ch, 0.0, "an error is a failure");
let health = monitor.reports()[0].health;
assert_eq!(health.incumbent.truncated, 1, "{health:?}");
assert_eq!(health.incumbent.errors, 0, "{health:?}");
assert_eq!(health.challenger.errors, 1, "{health:?}");
let _ = std::fs::remove_file(&db);
}
#[derive(Debug)]
struct FlakyReadLog {
inner: crate::experiment::InMemoryExperimentLog,
reads: std::sync::atomic::AtomicUsize,
always_fail: bool,
}
#[async_trait::async_trait]
impl crate::experiment::ExperimentLogBackend for FlakyReadLog {
async fn append(
&self,
tenant: &str,
route_ix: usize,
fingerprint: &str,
record: crate::experiment::LogRecord,
) -> Result<u64, crate::experiment::LogError> {
if self.always_fail {
return Err(crate::experiment::LogError::Backend("down".into()));
}
self.inner
.append(tenant, route_ix, fingerprint, record)
.await
}
async fn load_since(
&self,
tenant: &str,
route_ix: usize,
fingerprint: &str,
offset: u64,
) -> Result<Vec<Option<crate::experiment::LogRecord>>, crate::experiment::LogError>
{
let n = self.reads.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
if self.always_fail || n > 0 {
return Err(crate::experiment::LogError::Backend("down".into()));
}
self.inner
.load_since(tenant, route_ix, fingerprint, offset)
.await
}
async fn claim_session(&self, t: &str, r: usize, f: &str, s: &str) -> bool {
self.inner.claim_session(t, r, f, s).await
}
}
fn flaky_log(always_fail: bool) -> Arc<dyn crate::experiment::ExperimentLogBackend> {
Arc::new(FlakyReadLog {
inner: crate::experiment::InMemoryExperimentLog::new(),
reads: std::sync::atomic::AtomicUsize::new(0),
always_fail,
})
}
#[tokio::test]
async fn paired_spend_is_debited_locally_when_the_log_read_back_fails() {
use tower::ServiceExt;
let db = std::env::temp_dir().join(format!("firstpass-paired-flaky-{}.db", Uuid::now_v7()));
let (tx, handle) = crate::store::open(&db).unwrap();
drop(tx);
handle.await.unwrap();
let (mut state, _rx) = challenger_state(50.0, &db.to_string_lossy());
let toml = format!(
"[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
ladder = [\"{INCUMBENT}\"]\ngates = [\"json-valid\"]\n\
[route.challenger]\nladder = [\"{CHALLENGER}\"]\nmode = \"paired\"\n\
max_usd_per_day = 5.0\n"
);
let mut cfg = (*state.config).clone();
cfg.routing = Some(firstpass_core::Config::parse(&toml).expect("paired config parses"));
state.experiments = Arc::new(
crate::experiment::ExperimentMonitor::from_config(cfg.routing.as_ref().unwrap())
.with_log(flaky_log(false)),
);
state.config = Arc::new(cfg);
let mut outs = HashMap::new();
outs.insert(
INCUMBENT.to_owned(),
Ok(model_resp(INCUMBENT, "{\"ok\": 1}")),
);
outs.insert(
CHALLENGER.to_owned(),
Ok(model_resp(CHALLENGER, "{\"ok\": 1}")),
);
let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
map.insert(
"anthropic".to_owned(),
Arc::new(MockProvider::new("anthropic", outs)),
);
state.providers = ProviderRegistry::from_map(map);
let ledger = state.paired_ledger.clone();
let router = app(state).expect("router");
let req = axum::http::Request::builder()
.method("POST")
.uri("/v1/messages")
.body(Body::from(user_body()))
.unwrap();
assert_eq!(
router.oneshot(req).await.unwrap().status(),
axum::http::StatusCode::OK
);
let mut pairs = Vec::new();
for _ in 0..200 {
pairs = crate::store::load_all_experiment_pairs(&db).unwrap_or_default();
if !pairs.is_empty() {
break;
}
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
assert_eq!(pairs.len(), 1, "the challenger ran (first sync succeeded)");
let spent = ledger.spent_today("default", 0, jiff::Timestamp::now());
assert!(
spent > 0.0,
"challenger spend debited locally despite the failed read-back: {spent}"
);
let _ = std::fs::remove_file(&db);
}
#[derive(Debug)]
struct AppendFailLog;
#[async_trait::async_trait]
impl crate::experiment::ExperimentLogBackend for AppendFailLog {
async fn append(
&self,
_: &str,
_: usize,
_: &str,
_: crate::experiment::LogRecord,
) -> Result<u64, crate::experiment::LogError> {
Err(crate::experiment::LogError::Backend("rpush failed".into()))
}
async fn load_since(
&self,
_: &str,
_: usize,
_: &str,
_: u64,
) -> Result<Vec<Option<crate::experiment::LogRecord>>, crate::experiment::LogError>
{
Ok(vec![])
}
async fn claim_session(&self, _: &str, _: usize, _: &str, _: &str) -> bool {
false
}
}
#[tokio::test]
async fn paired_pair_that_missed_the_log_is_persisted_as_offline() {
use tower::ServiceExt;
let db =
std::env::temp_dir().join(format!("firstpass-paired-offline-{}.db", Uuid::now_v7()));
let (tx, handle) = crate::store::open(&db).unwrap();
drop(tx);
handle.await.unwrap();
let (mut state, _rx) = challenger_state(50.0, &db.to_string_lossy());
let toml = format!(
"[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
ladder = [\"{INCUMBENT}\"]\ngates = [\"json-valid\"]\n\
[route.challenger]\nladder = [\"{CHALLENGER}\"]\nmode = \"paired\"\n\
max_usd_per_day = 5.0\n"
);
let mut cfg = (*state.config).clone();
cfg.routing = Some(firstpass_core::Config::parse(&toml).expect("paired config parses"));
state.experiments = Arc::new(
crate::experiment::ExperimentMonitor::from_config(cfg.routing.as_ref().unwrap())
.with_log(Arc::new(AppendFailLog)),
);
state.config = Arc::new(cfg);
let mut outs = HashMap::new();
outs.insert(
INCUMBENT.to_owned(),
Ok(model_resp(INCUMBENT, "{\"ok\": 1}")),
);
outs.insert(
CHALLENGER.to_owned(),
Ok(model_resp(CHALLENGER, "{\"ok\": 1}")),
);
let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
map.insert(
"anthropic".to_owned(),
Arc::new(MockProvider::new("anthropic", outs)),
);
state.providers = ProviderRegistry::from_map(map);
let router = app(state).expect("router");
let req = axum::http::Request::builder()
.method("POST")
.uri("/v1/messages")
.body(Body::from(user_body()))
.unwrap();
assert_eq!(
router.oneshot(req).await.unwrap().status(),
axum::http::StatusCode::OK
);
let mut pairs = Vec::new();
for _ in 0..200 {
pairs = crate::store::load_all_experiment_pairs(&db).unwrap_or_default();
if !pairs.is_empty() {
break;
}
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
assert_eq!(pairs.len(), 1);
assert_eq!(
pairs[0].in_log,
crate::store::InLogStatus::NotInLog,
"the append failed, so the row must be marked offline"
);
let _ = std::fs::remove_file(&db);
}
#[tokio::test]
async fn sqlite_write_failure_withholds_local_evidence() {
use tower::ServiceExt;
let bad_db = "/nonexistent-firstpass-dir/firstpass-sqlite-fail.db";
let (mut state, _rx) = challenger_state(50.0, bad_db);
let toml = format!(
"[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
ladder = [\"{INCUMBENT}\"]\ngates = [\"json-valid\"]\n\
[route.challenger]\nladder = [\"{CHALLENGER}\"]\nmode = \"paired\"\n\
max_usd_per_day = 5.0\n"
);
let mut cfg = (*state.config).clone();
cfg.routing = Some(firstpass_core::Config::parse(&toml).expect("paired config parses"));
state.experiments = Arc::new(crate::experiment::ExperimentMonitor::from_config(
cfg.routing.as_ref().unwrap(),
));
state.config = Arc::new(cfg);
let mut outs = HashMap::new();
outs.insert(
INCUMBENT.to_owned(),
Ok(model_resp(INCUMBENT, "{\"ok\": 1}")),
);
outs.insert(
CHALLENGER.to_owned(),
Ok(model_resp(CHALLENGER, "{\"ok\": 1}")),
);
let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
map.insert(
"anthropic".to_owned(),
Arc::new(MockProvider::new("anthropic", outs)),
);
state.providers = ProviderRegistry::from_map(map);
let (ledger, monitor) = (state.paired_ledger.clone(), state.experiments.clone());
let router = app(state).expect("router");
let req = axum::http::Request::builder()
.method("POST")
.uri("/v1/messages")
.body(Body::from(user_body()))
.unwrap();
assert_eq!(
router.oneshot(req).await.unwrap().status(),
axum::http::StatusCode::OK
);
for _ in 0..200 {
if ledger.spent_today("default", 0, jiff::Timestamp::now()) > 0.0 {
break;
}
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
assert!(
ledger.spent_today("default", 0, jiff::Timestamp::now()) > 0.0,
"challenger ran"
);
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
let pairs: u64 = monitor.reports().iter().map(|r| r.report.pairs).sum();
assert_eq!(
pairs, 0,
"the durable write failed, so no evidence was applied"
);
}
#[tokio::test]
async fn live_challenger_is_not_served_when_the_cold_start_sync_fails() {
use firstpass_core::Arm;
let (mut state, mut rx) = challenger_state(50.0, ":memory:");
let cfg = state.config.routing.clone().unwrap();
state.experiments = Arc::new(
crate::experiment::ExperimentMonitor::from_config(&cfg).with_log(flaky_log(true)),
);
let session = session_in_arm(&state, Arm::Challenger, 0);
let trace = serve_session(&state, &mut rx, &session).await;
let rec = trace
.experiment
.expect("experiment record on a challenger route");
assert_eq!(rec.arm, Arm::Incumbent, "fail closed: served the incumbent");
assert_eq!(trace.attempts[0].model, INCUMBENT);
}
#[tokio::test]
async fn halted_paired_challenger_stops_spending() {
use tower::ServiceExt;
let db = std::env::temp_dir().join(format!("firstpass-paired-halt-{}.db", Uuid::now_v7()));
let (tx, handle) = crate::store::open(&db).unwrap();
drop(tx);
handle.await.unwrap();
let (mut state, _rx) = challenger_state(50.0, &db.to_string_lossy());
let toml = format!(
"[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
ladder = [\"{INCUMBENT}\"]\ngates = [\"json-valid\"]\n\
[route.challenger]\nladder = [\"{CHALLENGER}\"]\nmode = \"paired\"\n\
max_usd_per_day = 5.0\n"
);
let mut cfg = (*state.config).clone();
cfg.routing = Some(firstpass_core::Config::parse(&toml).expect("paired config parses"));
state.experiments = Arc::new(crate::experiment::ExperimentMonitor::from_config(
cfg.routing.as_ref().unwrap(),
));
state.config = Arc::new(cfg);
let fingerprint = state.experiments.fingerprint(0).unwrap().to_owned();
for _ in 0..500 {
if state.experiments.challenger_blocked("default", 0) {
break;
}
state.experiments.apply_pair_locally(
"default",
0,
&fingerprint,
1.0,
0.0,
0.001,
uuid::Uuid::nil(),
);
}
assert!(
state.experiments.challenger_blocked("default", 0),
"a challenger that always loses halts"
);
let mut outs = HashMap::new();
outs.insert(
INCUMBENT.to_owned(),
Ok(model_resp(INCUMBENT, "{\"ok\": 1}")),
);
outs.insert(
CHALLENGER.to_owned(),
Ok(model_resp(CHALLENGER, "{\"ok\": 1}")),
);
let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
map.insert(
"anthropic".to_owned(),
Arc::new(MockProvider::new("anthropic", outs)),
);
state.providers = ProviderRegistry::from_map(map);
let ledger = state.paired_ledger.clone();
let router = app(state).expect("router");
let req = axum::http::Request::builder()
.method("POST")
.uri("/v1/messages")
.body(Body::from(user_body()))
.unwrap();
let resp = router.oneshot(req).await.unwrap();
assert_eq!(
resp.status(),
axum::http::StatusCode::OK,
"incumbent still serves"
);
tokio::time::sleep(std::time::Duration::from_millis(300)).await;
assert!(
crate::store::load_all_experiment_pairs(&db)
.unwrap()
.is_empty(),
"no pair after the halt"
);
assert!(
ledger.may_spend("default", 0, 1e-9, jiff::Timestamp::now()),
"nothing debited after the halt"
);
let _ = std::fs::remove_file(&db);
}
#[tokio::test]
async fn shadow_reports_no_pass_when_every_rung_fails_its_gate() {
let (mut state, _rx) = challenger_state(50.0, ":memory:");
let mut outs = HashMap::new();
outs.insert(INCUMBENT.to_owned(), Ok(model_resp(INCUMBENT, "not json")));
let mut map: HashMap<String, Arc<dyn Provider>> = HashMap::new();
map.insert(
"anthropic".to_owned(),
Arc::new(MockProvider::new("anthropic", outs)),
);
state.providers = ProviderRegistry::from_map(map);
let cfg = firstpass_core::Config::parse(&format!(
"[[route]]\nmatch = {{}}\nmode = \"enforce\"\n\
ladder = [\"{INCUMBENT}\"]\ngates = [\"json-valid\"]\n"
))
.unwrap();
let shadow = firstpass_core::rollout::Shadow {
sample_rate: 1.0,
max_usd_per_day: 5.0,
};
let signal = evaluate_shadow(
&state,
&cfg.routes[0],
shadow,
0,
&user_body(),
Auth::from_headers(&HeaderMap::new(), firstpass_core::Dialect::Anthropic),
&Features::new(TaskKind::Other),
"t".to_owned(),
"s".to_owned(),
"anthropic.messages",
0.0,
)
.await;
assert_eq!(signal.skipped, None);
assert!(!signal.would_pass, "a best attempt is not a pass");
assert_eq!(signal.would_serve_rung, None);
}
fn ux_get(uri: &str, bearer: Option<&str>) -> axum::http::Request<Body> {
let mut b = axum::http::Request::builder().method("GET").uri(uri);
if let Some(k) = bearer {
b = b.header("authorization", format!("Bearer {k}"));
}
b.body(Body::empty()).unwrap()
}
#[tokio::test]
async fn experiments_endpoint_reports_per_arm_counts_scoped_to_the_tenant() {
use firstpass_core::Arm;
use tower::ServiceExt;
let db = std::env::temp_dir().join(format!("firstpass-ux-{}.db", Uuid::now_v7()));
let mut state = ux_state(&db.to_string_lossy());
let (traces_tx, mut traces_rx) = mpsc::channel(256);
state.traces = traces_tx;
let router = app(state.clone()).expect("router");
let mut sessions = Vec::new();
for arm in [Arm::Incumbent, Arm::Challenger] {
for n in 0..6 {
sessions.push((arm, session_in_arm(&state, arm, n)));
}
}
for (_, session) in &sessions {
let req = axum::http::Request::builder()
.method("POST")
.uri("/v1/messages")
.header("authorization", "Bearer tenant-a.key-a")
.header("x-firstpass-session", session.as_str())
.body(Body::from(user_body()))
.unwrap();
let resp = router.clone().oneshot(req).await.unwrap();
assert_eq!(resp.status(), axum::http::StatusCode::OK);
}
let mut traces = Vec::new();
while let Ok(t) = traces_rx.try_recv() {
traces.push(t);
}
assert_eq!(traces.len(), 12);
let (tx, handle) = crate::store::open(&db).unwrap();
for t in &traces {
tx.try_send(t.clone()).unwrap();
}
drop(tx);
handle.await.unwrap();
let mut challenger_seen = 0;
for t in &traces {
let arm = t.experiment.as_ref().unwrap().arm;
let verdict = if arm == Arm::Challenger {
challenger_seen += 1;
if challenger_seen <= 2 { "fail" } else { "pass" }
} else {
"pass"
};
let req = axum::http::Request::builder()
.method("POST")
.uri("/v1/feedback")
.header("authorization", "Bearer tenant-a.key-a")
.body(Body::from(
serde_json::json!({
"trace_id": t.trace_id, "gate_id": "tests",
"verdict": verdict, "reporter": "ci"
})
.to_string(),
))
.unwrap();
let resp = router.clone().oneshot(req).await.unwrap();
assert_eq!(resp.status(), axum::http::StatusCode::ACCEPTED);
}
let resp = router
.clone()
.oneshot(ux_get("/v1/experiments", Some("tenant-a.key-a")))
.await
.unwrap();
assert_eq!(resp.status(), axum::http::StatusCode::OK);
let a = body_json(resp).await;
let exps = a["experiments"].as_array().unwrap();
assert_eq!(exps.len(), 1, "{a}");
let r = &exps[0]["report"];
assert_eq!(r["status"], "undecided");
assert_eq!(r["pairs"], 6);
assert_eq!(r["incumbent"]["requests"], 6);
assert_eq!(r["incumbent"]["outcomes"], 6);
assert_eq!(r["incumbent"]["outcome_sum"], 6.0);
assert_eq!(r["challenger"]["requests"], 6);
assert_eq!(r["challenger"]["outcomes"], 6);
assert_eq!(r["challenger"]["outcome_sum"], 4.0);
assert_eq!(exps[0]["challenger_ladder"][0], CHALLENGER);
assert_eq!(exps[0]["recommendation"], "keep_running");
assert_eq!(a["configured"][0]["unavailable"], false);
assert_eq!(a["configured"][0]["mode"], "live");
assert!(a["configured"][0]["paired"].is_null());
let resp = router
.clone()
.oneshot(ux_get("/v1/experiments", Some("tenant-b.key-b")))
.await
.unwrap();
assert_eq!(resp.status(), axum::http::StatusCode::OK);
let b = body_json(resp).await;
assert_eq!(b["experiments"].as_array().unwrap().len(), 0, "{b}");
assert!(!b.to_string().contains("tenant-a"));
let resp = router
.clone()
.oneshot(ux_get("/v1/receipts?limit=3", Some("tenant-a.key-a")))
.await
.unwrap();
let ra = body_json(resp).await;
assert_eq!(ra["receipts"].as_array().unwrap().len(), 3);
assert_eq!(ra["summary"]["requests"], 12, "summary spans all receipts");
let first = &ra["receipts"][0];
assert!(first["experiment"]["arm"].is_string());
assert_eq!(first["hash"].as_str().unwrap().len(), 64);
assert!(first["prev_hash"].is_string());
assert!(first["attempts"][0]["gates"][0]["gate_id"].is_string());
let resp = router
.clone()
.oneshot(ux_get("/v1/receipts", Some("tenant-b.key-b")))
.await
.unwrap();
let rb = body_json(resp).await;
assert_eq!(rb["receipts"].as_array().unwrap().len(), 0);
assert_eq!(rb["summary"]["requests"], 0);
for bearer in [None, Some("tenant-a.wrong")] {
let resp = router
.clone()
.oneshot(ux_get("/v1/experiments", bearer))
.await
.unwrap();
assert_eq!(resp.status(), axum::http::StatusCode::UNAUTHORIZED);
}
let _ = std::fs::remove_file(&db);
}
#[test]
fn receipts_summary_json_keeps_losses_signed() {
let s = summary_json(&crate::store::ReceiptsSummary {
requests: 2,
spent_usd: 6.0,
baseline_usd: 6.0,
first_pass: 1,
escalated: 1,
gates: [3, 1, 2],
truncated: 4,
});
assert_eq!(s["requests"], 2);
assert_eq!(s["truncated"], 4);
assert_eq!(s["first_pass"], 1);
assert_eq!(s["escalated"], 1);
assert_eq!(s["gates"]["pass"], 3);
assert_eq!(s["gates"]["fail"], 1);
assert_eq!(s["gates"]["abstain"], 2);
assert!((s["saved_usd"].as_f64().unwrap()).abs() < 1e-9);
let loss = summary_json(&crate::store::ReceiptsSummary {
spent_usd: 5.0,
baseline_usd: 3.0,
..Default::default()
});
assert!(
loss["saved_usd"].as_f64().unwrap() < 0.0,
"loss stays negative"
);
}
#[tokio::test]
async fn panel_is_served_with_a_csp_pinning_its_own_script() {
use sha2::Digest as _;
let resp = panel().await.into_response();
let csp = resp
.headers()
.get(axum::http::header::CONTENT_SECURITY_POLICY)
.expect("panel sets a CSP")
.to_str()
.unwrap()
.to_owned();
let body = PANEL_HTML.split_once("<script>").unwrap().1;
let script = body.split_once("</script>").unwrap().0;
assert!(script.len() > 1000, "extracted the panel script");
let want = format!(
"script-src 'sha256-{}'",
base64_std(&sha2::Sha256::digest(script.as_bytes()))
);
assert!(csp.contains(&want), "{csp}");
for directive in [
"default-src 'none'",
"connect-src 'self'",
"frame-ancestors 'none'",
] {
assert!(csp.contains(directive), "{directive} missing: {csp}");
}
assert!(!csp.contains("unsafe-eval") && !csp.contains("script-src 'unsafe-inline'"));
assert_eq!(
PANEL_HTML.matches("<script").count(),
1,
"one inline script"
);
assert_eq!(
resp.headers()[axum::http::header::X_CONTENT_TYPE_OPTIONS],
"nosniff"
);
}
#[test]
fn base64_std_matches_rfc4648_vectors() {
for (input, want) in [
("", ""),
("f", "Zg=="),
("fo", "Zm8="),
("foo", "Zm9v"),
("foob", "Zm9vYg=="),
("fooba", "Zm9vYmE="),
("foobar", "Zm9vYmFy"),
] {
assert_eq!(base64_std(input.as_bytes()), want, "{input:?}");
}
assert_eq!(base64_std(&[0xfb, 0xff]), "+/8=");
}
#[test]
fn panel_wires_the_experiments_view_and_has_no_external_references() {
assert!(
PANEL_HTML.contains("/v1/experiments"),
"fetches the endpoint"
);
assert!(PANEL_HTML.contains("id=\"view-experiments\""));
assert!(PANEL_HTML.contains("id=\"exp-list\""));
assert!(
PANEL_HTML.contains("[route.challenger]"),
"empty state teaches the config"
);
assert!(
PANEL_HTML.contains("data-copy"),
"adopt snippet has a copy button"
);
assert!(
PANEL_HTML.contains("document.hidden"),
"polling pauses in a hidden tab"
);
assert!(PANEL_HTML.contains("prefers-reduced-motion"));
assert!(PANEL_HTML.contains("prefers-color-scheme: dark"));
for needle in [
"http://",
"https://",
"src=\"//",
"href=\"//",
"@import",
"<link",
] {
assert!(!PANEL_HTML.contains(needle), "external reference: {needle}");
}
for tag in ["<script", "<img", "<iframe", "<source"] {
for (i, _) in PANEL_HTML.match_indices(tag) {
let end = PANEL_HTML[i..]
.find('>')
.map_or(PANEL_HTML.len(), |e| i + e);
assert!(
!PANEL_HTML[i..end].contains("src="),
"{tag} loads an external source: {}",
&PANEL_HTML[i..end]
);
}
}
}
#[test]
fn panel_interpolates_only_escaped_or_numeric_values() {
let start = PANEL_HTML.find("<script>").expect("script") + "<script>".len();
let js = &PANEL_HTML[start..PANEL_HTML.find("</script>").expect("end")];
let bytes = js.as_bytes();
let mut i = 0;
let mut count = 0;
while let Some(off) = js[i..].find("${") {
let begin = i + off + 2;
let (mut depth, mut j) = (1, begin);
while depth > 0 && j < bytes.len() {
match bytes[j] {
b'{' => depth += 1,
b'}' => depth -= 1,
_ => {}
}
j += 1;
}
let expr = js[begin..j - 1].trim();
assert!(
["esc(", "usd(", "num(", "pct(", "cls(", "h_"]
.iter()
.any(|p| expr.starts_with(p)),
"unescaped interpolation in the panel: ${{{expr}}}"
);
count += 1;
i = begin;
}
assert!(
count > 40,
"the scan must actually see the templates ({count})"
);
assert!(js.contains("const esc ="), "the escaper must exist");
}
struct TestStream {
chunks: std::collections::VecDeque<Result<Bytes, reqwest::Error>>,
}
impl TestStream {
fn from_ok(v: Vec<&'static [u8]>) -> Self {
Self {
chunks: v.into_iter().map(|b| Ok(Bytes::from_static(b))).collect(),
}
}
}
impl Stream for TestStream {
type Item = Result<Bytes, reqwest::Error>;
fn poll_next(
mut self: std::pin::Pin<&mut Self>,
_cx: &mut Context<'_>,
) -> Poll<Option<Self::Item>> {
Poll::Ready(self.chunks.pop_front())
}
}
fn anthropic_sse_bytes() -> (&'static [u8], &'static [u8]) {
let c1: &'static [u8] = b"event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-haiku-4-5\",\"usage\":{\"input_tokens\":100,\"cache_crea";
let c2: &'static [u8] = b"tion_input_tokens\":50,\"cache_read_input_tokens\":25}}}\n\nevent: message_delta\ndata: {\"type\":\"message_delta\",\"usage\":{\"output_tokens\":200},\"delta\":{\"stop_reason\":\"max_tokens\"}}\n\n";
(c1, c2)
}
#[tokio::test]
async fn sse_tap_parses_usage_across_split_chunks_and_relays_bytes_unchanged() {
let (c1, c2) = anthropic_sse_bytes();
let all_bytes: Vec<u8> = [c1, c2].concat();
let stream = TestStream::from_ok(vec![c1, c2]);
let (tx, rx) = tokio::sync::oneshot::channel::<SseUsage>();
let tap = SseTap::new(stream, move |u, _reason| {
let _ = tx.send(u);
});
let collected = axum::body::to_bytes(Body::from_stream(tap), usize::MAX)
.await
.unwrap();
assert_eq!(
collected.as_ref(),
all_bytes.as_slice(),
"relayed bytes must equal input bytes exactly"
);
let usage = rx.await.unwrap();
assert_eq!(usage.in_tokens, 100, "input_tokens from message_start");
assert_eq!(
usage.cache_write_tokens, 50,
"cache_creation from message_start"
);
assert_eq!(usage.cache_read_tokens, 25, "cache_read from message_start");
assert_eq!(usage.out_tokens, 200, "output_tokens from message_delta");
assert_eq!(
usage.model.as_deref(),
Some("claude-haiku-4-5"),
"model from message_start"
);
assert_eq!(
usage.stop_reason.as_deref(),
Some("max_tokens"),
"stop_reason from message_delta"
);
let config = test_config();
let req = Bytes::from_static(br#"{"model":"claude-haiku-4-5","messages":[]}"#);
let trace = build_stream_trace_from_usage(
&config,
&req,
10,
None,
usage,
"anthropic",
StreamOutcome::Served,
"anthropic.messages",
None,
);
assert!(
trace.attempts[0].truncated,
"max_tokens stop_reason must set truncated=true"
);
assert!(
trace.attempts[0].cost_usd > 0.0,
"non-zero tokens must produce a non-zero cost"
);
}
#[tokio::test]
async fn observe_records_an_upstream_429_as_an_error_on_both_paths() {
const ERR: &str =
r#"{"type":"error","error":{"type":"rate_limit_error","message":"slow down"}}"#;
let app = axum::Router::new().route(
"/v1/messages",
axum::routing::post(|| async {
(
axum::http::StatusCode::TOO_MANY_REQUESTS,
[(axum::http::header::CONTENT_TYPE, "application/json")],
ERR,
)
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("bind");
let addr = listener.local_addr().expect("addr");
tokio::spawn(async move {
let _ = axum::serve(listener, app).await;
});
let base_url = format!("http://{addr}");
let config = ProxyConfig::from_lookup(|k| match k {
"FIRSTPASS_UPSTREAM_ANTHROPIC" => Some(base_url.clone()),
_ => None,
})
.unwrap();
let (traces, mut rx) = mpsc::channel(4);
let state = AppState {
config: Arc::new(config),
http: reqwest::Client::new(),
providers: ProviderRegistry::from_map(HashMap::new()),
gate_health: Arc::new(GateHealthRegistry::new()),
shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
experiments: Arc::default(),
guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
traces,
adaptive: None,
eprocess: None,
bandit: None,
promoter: None,
verified_cache: None,
predictor: None,
prior: None,
tenant_rate_limiter: None,
spill: None,
};
for body in [
&br#"{"model":"claude-haiku-4-5","messages":[{"role":"user","content":"hi"}]}"#[..],
&br#"{"model":"claude-haiku-4-5","stream":true,"messages":[{"role":"user","content":"hi"}]}"#[..],
] {
let resp = observe_passthrough(
state.clone(),
HeaderMap::new(),
Bytes::from_static(body),
None,
"default".to_owned(),
)
.await;
assert_eq!(resp.status(), axum::http::StatusCode::TOO_MANY_REQUESTS);
let relayed = axum::body::to_bytes(resp.into_body(), 1 << 20).await.unwrap();
assert_eq!(relayed.as_ref(), ERR.as_bytes(), "relayed byte-for-byte");
let trace = tokio::time::timeout(std::time::Duration::from_secs(5), rx.recv())
.await
.expect("trace within 5s")
.expect("a trace");
assert_eq!(
trace.final_.served_from,
ServedFrom::Error,
"stream={}",
String::from_utf8_lossy(body).contains("stream")
);
assert!(trace.attempts.is_empty());
}
}
#[tokio::test]
async fn observe_stream_receipt_has_sse_usage_and_non_zero_cost() {
const SSE: &str = concat!(
"event: message_start\n",
"data: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-haiku-4-5\",",
"\"usage\":{\"input_tokens\":100,\"cache_creation_input_tokens\":0,",
"\"cache_read_input_tokens\":0}}}\n",
"\n",
"event: message_delta\n",
"data: {\"type\":\"message_delta\",\"usage\":{\"output_tokens\":50},",
"\"delta\":{\"stop_reason\":\"max_tokens\"}}\n",
"\n",
);
let app = axum::Router::new().route(
"/v1/messages",
axum::routing::post(|| async {
(
[(
axum::http::header::CONTENT_TYPE,
"text/event-stream; charset=utf-8",
)],
SSE,
)
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("bind");
let addr = listener.local_addr().expect("addr");
tokio::spawn(async move {
let _ = axum::serve(listener, app).await;
});
let base_url = format!("http://{addr}");
let config = ProxyConfig::from_lookup(|k| match k {
"FIRSTPASS_UPSTREAM_ANTHROPIC" => Some(base_url.clone()),
_ => None,
})
.unwrap();
let (traces, mut rx) = mpsc::channel(4);
let state = AppState {
config: Arc::new(config),
http: reqwest::Client::new(),
providers: ProviderRegistry::from_map(HashMap::new()),
gate_health: Arc::new(GateHealthRegistry::new()),
shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
experiments: Arc::default(),
guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
traces,
adaptive: None,
eprocess: None,
bandit: None,
promoter: None,
verified_cache: None,
predictor: None,
prior: None,
tenant_rate_limiter: None,
spill: None,
};
let body = Bytes::from_static(
br#"{"model":"claude-haiku-4-5","stream":true,"messages":[{"role":"user","content":"hi"}]}"#,
);
let resp = observe_stream(state, HeaderMap::new(), body, None, "default".to_owned()).await;
assert_eq!(resp.status(), axum::http::StatusCode::OK);
let _ = axum::body::to_bytes(resp.into_body(), 1 << 20)
.await
.unwrap();
let trace = rx
.recv()
.await
.expect("trace must be sent after stream ends");
assert_eq!(trace.attempts.len(), 1, "one attempt");
let attempt = &trace.attempts[0];
assert_eq!(
attempt.in_tokens, 100,
"input tokens from SSE message_start"
);
assert_eq!(
attempt.out_tokens, 50,
"output tokens from SSE message_delta"
);
assert!(
attempt.cost_usd > 0.0,
"non-zero tokens must produce a non-zero cost"
);
assert!(
attempt.truncated,
"max_tokens stop_reason must set truncated=true in the receipt"
);
assert_eq!(
attempt.model, "claude-haiku-4-5",
"model from SSE message_start"
);
}
#[test]
fn sse_tap_oversize_line_skipped_and_following_valid_event_still_parsed() {
let mut big_chunk = b"data: ".to_vec();
big_chunk.extend(std::iter::repeat_n(b'x', SSE_LINE_BUF_CAP + 1));
big_chunk.push(b'\n');
big_chunk.extend_from_slice(
b"event: message_start\n\
data: {\"type\":\"message_start\",\"message\":{\"model\":\"m\",\
\"usage\":{\"input_tokens\":42,\"cache_creation_input_tokens\":0,\
\"cache_read_input_tokens\":0}}}\n\n",
);
let mut line_buf: Vec<u8> = Vec::new();
let mut overflow = false;
let mut usage = SseUsage::default();
parse_sse_chunk_into(&big_chunk, &mut line_buf, &mut overflow, &mut usage);
assert!(
line_buf.len() <= SSE_LINE_BUF_CAP,
"line buffer must stay bounded: len={}",
line_buf.len()
);
assert_eq!(
usage.in_tokens, 42,
"valid event after oversize line must still be parsed"
);
assert_eq!(
usage.model.as_deref(),
Some("m"),
"model must be extracted from the valid event"
);
}
#[tokio::test]
async fn sse_tap_upstream_error_fires_done_with_upstream_error_reason() {
let err: reqwest::Error = reqwest::Client::new().get("not-a-url").build().unwrap_err();
let ok_chunk: &'static [u8] =
b"event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"m\",\"usage\":{\"input_tokens\":5,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0}}}\n\n";
let stream = TestStream {
chunks: [Ok(Bytes::from_static(ok_chunk)), Err(err)]
.into_iter()
.collect(),
};
let (tx, rx) = tokio::sync::oneshot::channel::<SseEndReason>();
let tap = SseTap::new(stream, move |_u, reason| {
let _ = tx.send(reason);
});
let _ = axum::body::to_bytes(Body::from_stream(tap), usize::MAX).await;
let reason = rx.await.expect("on_done must fire");
assert_eq!(
reason,
SseEndReason::UpstreamError,
"transport error mid-stream must produce UpstreamError, not Eof"
);
}
#[tokio::test]
async fn sse_tap_normal_end_fires_done_with_eof_reason() {
let (c1, c2) = anthropic_sse_bytes();
let stream = TestStream::from_ok(vec![c1, c2]);
let (tx, rx) = tokio::sync::oneshot::channel::<SseEndReason>();
let tap = SseTap::new(stream, move |_u, reason| {
let _ = tx.send(reason);
});
let _ = axum::body::to_bytes(Body::from_stream(tap), usize::MAX)
.await
.unwrap();
let reason = rx.await.expect("on_done must fire");
assert_eq!(
reason,
SseEndReason::Eof,
"normal stream end must produce Eof, not Dropped or UpstreamError"
);
}
#[test]
fn build_trace_dated_snapshot_falls_back_to_request_model_for_pricing() {
let config = test_config();
let req = Bytes::from_static(br#"{"model":"claude-haiku-4-5","messages":[]}"#);
let resp = Bytes::from_static(
br#"{"model":"claude-haiku-4-5-20251001","usage":{"input_tokens":500,"output_tokens":100}}"#,
);
let trace = build_trace(
&config,
&req,
&resp,
10,
None,
"anthropic",
"anthropic.messages",
None,
);
let attempt = &trace.attempts[0];
assert!(
attempt.cost_usd > 0.0,
"dated snapshot model must fall back to request model for pricing; got $0"
);
assert_eq!(
attempt.model, "claude-haiku-4-5-20251001",
"attempt model must be the dated snapshot name, not the canonical request model"
);
}
#[test]
fn build_stream_trace_dated_snapshot_falls_back_to_request_model_for_pricing() {
let config = test_config();
let req = Bytes::from_static(br#"{"model":"claude-haiku-4-5","messages":[]}"#);
let usage = SseUsage {
model: Some("claude-haiku-4-5-20251001".to_owned()),
in_tokens: 500,
out_tokens: 100,
..Default::default()
};
let trace = build_stream_trace_from_usage(
&config,
&req,
10,
None,
usage,
"anthropic",
StreamOutcome::Served,
"anthropic.messages",
None,
);
let attempt = &trace.attempts[0];
assert!(
attempt.cost_usd > 0.0,
"dated snapshot model must fall back to request model for pricing; got $0"
);
assert_eq!(
attempt.model, "claude-haiku-4-5-20251001",
"attempt model must be the dated snapshot name, not the canonical request model"
);
}
#[test]
fn gemini_stream_max_tokens_is_truncated() {
let config = test_config();
let req = Bytes::from_static(br#"{"contents":[]}"#);
let usage = SseUsage {
in_tokens: 10,
out_tokens: 64,
stop_reason: Some("MAX_TOKENS".to_owned()),
completed: true,
..Default::default()
};
let trace = build_stream_trace_from_usage(
&config,
&req,
10,
None,
usage,
"google",
StreamOutcome::Served,
"gemini.generate_content",
Some("gemini-2.5-pro"),
);
assert!(trace.attempts[0].truncated);
}
#[test]
fn price_fallback_does_not_apply_to_unrelated_unpriced_model() {
let config = test_config();
let req = Bytes::from_static(br#"{"model":"claude-haiku-4-5","messages":[]}"#);
let resp = Bytes::from_static(
br#"{"model":"gpt-99-turbo","usage":{"input_tokens":500,"output_tokens":100}}"#,
);
let trace = build_trace(
&config,
&req,
&resp,
10,
None,
"anthropic",
"anthropic.messages",
None,
);
let attempt = &trace.attempts[0];
assert_eq!(
attempt.cost_usd, 0.0,
"an unrelated unpriced response model must NOT fall back to the request model's price"
);
assert_eq!(
attempt.model, "gpt-99-turbo",
"the unpriced response model name must still appear on the attempt"
);
}
#[test]
fn price_fallback_strips_provider_prefix_from_request_model() {
let config = test_config();
let req = Bytes::from_static(br#"{"model":"openai/gpt-4.1-mini","messages":[]}"#);
let usage = SseUsage {
model: Some("gpt-4.1-mini-20250514".to_owned()),
in_tokens: 300,
out_tokens: 50,
..Default::default()
};
let trace = build_stream_trace_from_usage(
&config,
&req,
10,
None,
usage,
"openai",
StreamOutcome::Served,
"openai.chat_completions",
None,
);
let attempt = &trace.attempts[0];
assert!(
attempt.cost_usd > 0.0,
"snapshot of request model must be priced correctly even with provider-prefix in req body"
);
assert_eq!(
attempt.model, "gpt-4.1-mini-20250514",
"attempt model must be the snapshot name from SSE, not the canonical request model"
);
}
fn test_state_with_channel() -> (AppState, tokio::sync::mpsc::Receiver<firstpass_core::Trace>) {
let config = test_config();
let (traces, rx) = mpsc::channel(4);
let state = AppState {
config: Arc::new(config),
http: reqwest::Client::new(),
providers: ProviderRegistry::from_map(HashMap::new()),
gate_health: Arc::new(GateHealthRegistry::new()),
shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
experiments: Arc::default(),
guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
traces,
adaptive: None,
eprocess: None,
bandit: None,
promoter: None,
verified_cache: None,
predictor: None,
prior: None,
tenant_rate_limiter: None,
spill: None,
};
(state, rx)
}
#[tokio::test]
async fn stream_callback_upstream_error_records_billed_usage_as_error() {
let (state, mut rx) = test_state_with_channel();
let req = Bytes::from_static(br#"{"model":"claude-haiku-4-5","messages":[]}"#);
let usage = SseUsage {
model: Some("claude-haiku-4-5".to_owned()),
in_tokens: 200,
out_tokens: 40,
..Default::default()
};
let cb = stream_trace_callback(
&state,
req,
15,
None,
"default".to_owned(),
"anthropic",
true,
"anthropic.messages",
None,
);
cb(usage, SseEndReason::UpstreamError);
let trace = tokio::time::timeout(std::time::Duration::from_secs(5), rx.recv())
.await
.expect("trace within 5s")
.expect("a trace");
assert_eq!(
trace.final_.served_from,
ServedFrom::Error,
"UpstreamError must produce ServedFrom::Error"
);
assert_eq!(trace.attempts.len(), 1, "one attempt with billed usage");
let attempt = &trace.attempts[0];
assert_eq!(
attempt.verdict,
Verdict::Abstain,
"UpstreamError attempt must be Abstain, not Pass"
);
assert!(
attempt.cost_usd > 0.0,
"billed tokens must produce non-zero cost in the error trace"
);
assert_eq!(attempt.in_tokens, 200);
}
async fn dropped_receipt(usage: SseUsage) -> Trace {
let (state, mut rx) = test_state_with_channel();
let req = Bytes::from_static(br#"{"model":"claude-haiku-4-5","messages":[]}"#);
let cb = stream_trace_callback(
&state,
req,
5,
None,
"default".to_owned(),
"anthropic",
true,
"anthropic.messages",
None,
);
cb(usage, SseEndReason::Dropped);
tokio::time::timeout(std::time::Duration::from_secs(5), rx.recv())
.await
.expect("trace within 5s")
.expect("a trace")
}
#[tokio::test]
async fn stream_callback_dropped_before_completion_is_a_partial_not_a_pass() {
for usage in [
SseUsage::default(),
SseUsage {
model: Some("claude-haiku-4-5".to_owned()),
in_tokens: 100,
out_tokens: 30,
..Default::default()
},
] {
let had_tokens = usage.in_tokens;
let trace = dropped_receipt(usage).await;
assert_eq!(trace.final_.served_from, ServedFrom::Attempt);
assert_eq!(trace.attempts.len(), 1);
assert_eq!(trace.attempts[0].verdict, Verdict::Abstain, "never a pass");
assert_eq!(trace.attempts[0].in_tokens, had_tokens, "billed usage kept");
}
}
#[tokio::test]
async fn stream_callback_dropped_after_completion_is_a_served_pass() {
let trace = dropped_receipt(SseUsage {
model: Some("claude-haiku-4-5".to_owned()),
in_tokens: 100,
out_tokens: 30,
stop_reason: Some("end_turn".to_owned()),
completed: true,
..Default::default()
})
.await;
assert_eq!(trace.final_.served_from, ServedFrom::Attempt);
assert_eq!(trace.attempts[0].verdict, Verdict::Pass);
}
#[test]
fn exact_provider_qualified_model_is_priced() {
let config = test_config();
let req = Bytes::from_static(br#"{"model":"openai/gpt-4.1-mini","messages":[]}"#);
let resp = Bytes::from_static(br#"{"usage":{"prompt_tokens":100,"completion_tokens":10}}"#);
let trace = build_trace(
&config,
&req,
&resp,
1,
None,
"openai",
"openai.chat_completions",
None,
);
assert!(trace.attempts[0].cost_usd > 0.0, "{:?}", trace.attempts[0]);
}
#[tokio::test]
async fn stream_callback_eof_before_completion_is_a_partial() {
let (state, mut rx) = test_state_with_channel();
let req = Bytes::from_static(br#"{"model":"claude-haiku-4-5","messages":[]}"#);
for (completed, want) in [(false, Verdict::Abstain), (true, Verdict::Pass)] {
let cb = stream_trace_callback(
&state,
req.clone(),
5,
None,
"default".to_owned(),
"anthropic",
true,
"anthropic.messages",
None,
);
cb(
SseUsage {
model: Some("claude-haiku-4-5".to_owned()),
in_tokens: 50,
completed,
..Default::default()
},
SseEndReason::Eof,
);
let trace = tokio::time::timeout(std::time::Duration::from_secs(5), rx.recv())
.await
.expect("trace within 5s")
.expect("a trace");
assert_eq!(trace.attempts[0].verdict, want, "completed={completed}");
}
}
#[test]
fn sse_parser_tolerates_crlf() {
let mut u = SseUsage::default();
let (mut buf, mut over) = (Vec::new(), false);
parse_sse_chunk_into(
b"data: {\"type\":\"message_start\",\"message\":{\"model\":\"m\",\"usage\":{\"input_tokens\":7}}}\r\ndata: [DONE]\r\n",
&mut buf,
&mut over,
&mut u,
);
assert_eq!(u.in_tokens, 7);
assert!(u.completed);
}
#[test]
fn sse_completion_is_set_only_by_terminal_events() {
let parse = |lines: &[&str]| {
let mut u = SseUsage::default();
let (mut buf, mut over) = (Vec::new(), false);
for l in lines {
parse_sse_chunk_into(format!("{l}\n").as_bytes(), &mut buf, &mut over, &mut u);
}
u.completed
};
assert!(!parse(&[
r#"data: {"type":"message_start","message":{"model":"m","usage":{"input_tokens":5}}}"#
]));
assert!(parse(&[
r#"data: {"type":"message_delta","usage":{"output_tokens":3},"delta":{"stop_reason":"end_turn"}}"#
]));
assert!(parse(&[r#"data: {"type":"message_stop"}"#]));
assert!(!parse(&[
r#"data: {"model":"gpt","choices":[{"delta":{"content":"hi"},"finish_reason":null}]}"#
]));
assert!(parse(&[
r#"data: {"model":"gpt","choices":[{"finish_reason":"stop"}]}"#
]));
assert!(parse(&["data: [DONE]"]));
}
#[test]
fn sse_tap_drop_before_exhaustion_fires_dropped() {
use std::sync::{Arc, Mutex};
let (c1, _c2) = anthropic_sse_bytes();
let stream = TestStream::from_ok(vec![c1, b"more-data"]);
let captured = Arc::new(Mutex::new(None::<SseEndReason>));
let cap2 = captured.clone();
let tap = SseTap::new(stream, move |_u, reason| {
*cap2.lock().unwrap() = Some(reason);
});
drop(tap);
let reason = captured.lock().unwrap().expect("on_done must fire on drop");
assert_eq!(
reason,
SseEndReason::Dropped,
"tap dropped before stream ends must fire Dropped"
);
}
#[test]
fn error_trace_carries_upstream_provider() {
let config = test_config();
let req = Bytes::from_static(br#"{"model":"claude-haiku-4-5","messages":[]}"#);
let trace = build_error_trace(&config, &req, 10, None, "anthropic", "anthropic.messages");
assert_eq!(
trace.upstream_provider, "anthropic",
"error trace must record the provider for metrics even without an attempt"
);
}
#[tokio::test]
async fn observe_records_upstream_429_openai_both_paths() {
const ERR: &str = r#"{"error":{"message":"Rate limit reached","type":"rate_limit_error","code":"rate_limit_exceeded"}}"#;
let app = axum::Router::new().route(
"/v1/chat/completions",
axum::routing::post(|| async {
(
axum::http::StatusCode::TOO_MANY_REQUESTS,
[(axum::http::header::CONTENT_TYPE, "application/json")],
ERR,
)
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("bind");
let addr = listener.local_addr().expect("addr");
tokio::spawn(async move {
let _ = axum::serve(listener, app).await;
});
let base_url = format!("http://{addr}");
let config = ProxyConfig::from_lookup(|k| match k {
"FIRSTPASS_UPSTREAM_OPENAI" => Some(base_url.clone()),
_ => None,
})
.unwrap();
let (traces, mut rx) = mpsc::channel(4);
let state = AppState {
config: Arc::new(config),
http: reqwest::Client::new(),
providers: ProviderRegistry::from_map(HashMap::new()),
gate_health: Arc::new(GateHealthRegistry::new()),
shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
experiments: Arc::default(),
guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
traces,
adaptive: None,
eprocess: None,
bandit: None,
promoter: None,
verified_cache: None,
predictor: None,
prior: None,
tenant_rate_limiter: None,
spill: None,
};
for body in [
&br#"{"model":"gpt-4.1-mini","messages":[{"role":"user","content":"hi"}]}"#[..],
&br#"{"model":"gpt-4.1-mini","stream":true,"messages":[{"role":"user","content":"hi"}]}"#[..],
] {
let resp = observe_passthrough_openai_path(
state.clone(),
HeaderMap::new(),
Bytes::from_static(body),
None,
"default".to_owned(),
crate::upstream::OPENAI_CHAT_PATH,
)
.await;
assert_eq!(
resp.status(),
axum::http::StatusCode::TOO_MANY_REQUESTS,
"upstream 429 must be relayed, stream={}",
String::from_utf8_lossy(body).contains("stream")
);
let relayed = axum::body::to_bytes(resp.into_body(), 1 << 20)
.await
.unwrap();
assert_eq!(relayed.as_ref(), ERR.as_bytes(), "body relayed byte-for-byte");
let trace = tokio::time::timeout(std::time::Duration::from_secs(5), rx.recv())
.await
.expect("trace within 5s")
.expect("a trace");
assert_eq!(
trace.final_.served_from,
ServedFrom::Error,
"OpenAI 429 must be ServedFrom::Error, stream={}",
String::from_utf8_lossy(body).contains("stream")
);
assert!(trace.attempts.is_empty(), "no attempts on 429");
}
}
fn thinking_tool_resp() -> ModelResponse {
ModelResponse {
model: "anthropic/claude-3-7-sonnet-20250219".to_owned(),
text: String::new(),
in_tokens: 20,
cache_write_tokens: 100,
cache_read_tokens: 50,
out_tokens: 30,
raw: serde_json::json!({
"stop_reason": "tool_use",
"stop_sequence": null,
"content": [
{ "type": "thinking", "thinking": "I need to call the tool.", "signature": "sig_abc123" },
{ "type": "tool_use", "id": "tu_1", "name": "search", "input": { "q": "test" } }
]
}),
}
}
#[test]
fn anthropic_response_json_carries_stop_reason_and_cache_tokens() {
let resp = thinking_tool_resp();
let json = anthropic_response_json(&resp);
assert_eq!(
json["stop_reason"], "tool_use",
"stop_reason must be forwarded"
);
assert_eq!(json["stop_sequence"], Value::Null);
assert_eq!(json["usage"]["cache_creation_input_tokens"], 100);
assert_eq!(json["usage"]["cache_read_input_tokens"], 50);
assert_eq!(json["usage"]["input_tokens"], 20);
assert_eq!(json["usage"]["output_tokens"], 30);
let content = json["content"].as_array().unwrap();
assert_eq!(content[0]["type"], "thinking");
assert_eq!(content[1]["type"], "tool_use");
}
#[test]
fn anthropic_response_json_omits_cache_tokens_when_zero() {
let resp = ModelResponse {
cache_write_tokens: 0,
cache_read_tokens: 0,
..thinking_tool_resp()
};
let json = anthropic_response_json(&resp);
assert!(
json["usage"]["cache_creation_input_tokens"].is_null(),
"zero cache_write omitted"
);
assert!(
json["usage"]["cache_read_input_tokens"].is_null(),
"zero cache_read omitted"
);
}
#[test]
fn anthropic_response_json_maps_openai_finish_reason_to_anthropic_stop_reason() {
let cases = [
("length", "max_tokens"),
("stop", "end_turn"),
("content_filter", "end_turn"),
];
for (finish_reason, expected_stop_reason) in cases {
let resp = ModelResponse {
model: "gpt-4o".to_owned(),
text: String::new(),
in_tokens: 5,
cache_write_tokens: 0,
cache_read_tokens: 0,
out_tokens: 3,
raw: serde_json::json!({
"choices": [{ "finish_reason": finish_reason, "message": { "content": null } }]
}),
};
let json = anthropic_response_json(&resp);
assert_eq!(
json["stop_reason"].as_str(),
Some(expected_stop_reason),
"finish_reason={finish_reason} → stop_reason={expected_stop_reason}"
);
}
let resp_with_calls = ModelResponse {
model: "gpt-4o".to_owned(),
text: String::new(),
in_tokens: 5,
cache_write_tokens: 0,
cache_read_tokens: 0,
out_tokens: 3,
raw: serde_json::json!({
"choices": [{ "finish_reason": "tool_calls", "message": {
"content": null,
"tool_calls": [{"id":"c1","type":"function",
"function":{"name":"f","arguments":"{}"}}]
}}]
}),
};
assert_eq!(
anthropic_response_json(&resp_with_calls)["stop_reason"].as_str(),
Some("tool_use"),
"finish_reason=tool_calls with tool_call in message → stop_reason=tool_use"
);
}
#[test]
fn anthropic_sse_from_message_uses_real_stop_reason_and_thinking() {
let resp = thinking_tool_resp();
let message = anthropic_response_json(&resp);
let sse = anthropic_sse_from_message(&message);
let frames: Vec<Value> = sse
.lines()
.filter_map(|l| {
if l.starts_with("event: ") {
return None;
}
l.strip_prefix("data: ")
})
.map(|d| serde_json::from_str::<Value>(d).expect("valid JSON"))
.collect();
let msg_delta = frames
.iter()
.find(|f| f["type"] == "message_delta")
.expect("message_delta frame");
assert_eq!(
msg_delta["delta"]["stop_reason"], "tool_use",
"stop_reason must not be hardcoded to end_turn"
);
assert!(
frames
.iter()
.any(|f| f["content_block"]["type"] == "thinking"),
"thinking block_start must appear"
);
assert!(
frames.iter().any(|f| f["delta"]["type"] == "thinking_delta"
&& f["delta"]["thinking"] == "I need to call the tool."),
"thinking_delta must carry the thinking text"
);
assert!(
frames
.iter()
.any(|f| f["delta"]["type"] == "signature_delta"
&& f["delta"]["signature"] == "sig_abc123"),
"signature_delta must carry the signature"
);
assert!(
frames
.iter()
.any(|f| f["content_block"]["type"] == "tool_use"
&& f["content_block"]["name"] == "search"),
"tool_use block must still appear"
);
}
#[test]
fn anthropic_sse_from_message_handles_redacted_thinking() {
let resp = ModelResponse {
raw: serde_json::json!({
"stop_reason": "end_turn",
"content": [
{ "type": "redacted_thinking", "data": "base64encodeddata==" }
]
}),
..thinking_tool_resp()
};
let message = anthropic_response_json(&resp);
let sse = anthropic_sse_from_message(&message);
let frames: Vec<Value> = sse
.lines()
.filter_map(|l| {
if l.starts_with("event: ") {
return None;
}
l.strip_prefix("data: ")
})
.map(|d| serde_json::from_str::<Value>(d).expect("valid JSON"))
.collect();
assert!(
frames.iter().any(|f| {
f["type"] == "content_block_start"
&& f["content_block"]["type"] == "redacted_thinking"
&& f["content_block"]["data"] == "base64encodeddata=="
}),
"content_block_start must carry the full redacted_thinking block with data"
);
assert!(
!frames
.iter()
.any(|f| f["delta"]["type"] == "redacted_thinking_delta"),
"no redacted_thinking_delta must appear"
);
}
#[test]
fn openai_response_json_sets_finish_reason_length_when_truncated() {
let resp = ModelResponse {
model: "gpt-4o".to_owned(),
text: "partial".to_owned(),
in_tokens: 5,
cache_write_tokens: 0,
cache_read_tokens: 0,
out_tokens: 100,
raw: serde_json::json!({ "stop_reason": "max_tokens", "content": [{ "type": "text", "text": "partial" }] }),
};
let json = openai_response_json(&resp);
assert_eq!(json["choices"][0]["finish_reason"], "length");
}
#[test]
fn openai_sse_from_message_tool_calls_have_index() {
let resp = ModelResponse {
model: "gpt-4o".to_owned(),
text: String::new(),
in_tokens: 10,
cache_write_tokens: 0,
cache_read_tokens: 0,
out_tokens: 8,
raw: serde_json::json!({
"content": [
{ "type": "tool_use", "id": "c1", "name": "alpha", "input": { "x": 1 } },
{ "type": "tool_use", "id": "c2", "name": "beta", "input": { "y": 2 } }
]
}),
};
let sse = openai_sse_from_message(&openai_response_json(&resp), false);
let frames: Vec<Value> = sse
.lines()
.filter_map(|l| l.strip_prefix("data: "))
.filter(|f| *f != "[DONE]")
.map(|d| serde_json::from_str::<Value>(d).expect("valid JSON"))
.collect();
let tc_frame = frames
.iter()
.find(|f| !f["choices"][0]["delta"]["tool_calls"].is_null())
.expect("tool_calls delta frame");
let tcs = tc_frame["choices"][0]["delta"]["tool_calls"]
.as_array()
.expect("tool_calls array");
assert_eq!(tcs.len(), 2, "both tool calls present");
assert_eq!(tcs[0]["index"], 0, "first call has index 0");
assert_eq!(tcs[1]["index"], 1, "second call has index 1");
}
#[test]
fn openai_sse_from_message_emits_usage_chunk() {
let resp = ModelResponse {
model: "gpt-4o".to_owned(),
text: "hi".to_owned(),
in_tokens: 3,
cache_write_tokens: 0,
cache_read_tokens: 0,
out_tokens: 2,
raw: serde_json::json!({ "content": [{ "type": "text", "text": "hi" }] }),
};
let sse = openai_sse_from_message(&openai_response_json(&resp), true);
let frames: Vec<Value> = sse
.lines()
.filter_map(|l| l.strip_prefix("data: "))
.filter(|f| *f != "[DONE]")
.map(|d| serde_json::from_str::<Value>(d).expect("valid JSON"))
.collect();
let usage_frame = frames
.iter()
.find(|f| f["choices"].as_array().is_some_and(|a| a.is_empty()))
.expect("usage chunk with choices:[] must appear before [DONE]");
assert_eq!(usage_frame["usage"]["prompt_tokens"], 3);
assert_eq!(usage_frame["usage"]["completion_tokens"], 2);
assert_eq!(
sse.lines()
.filter_map(|l| l.strip_prefix("data: "))
.next_back()
.unwrap(),
"[DONE]"
);
}
#[test]
fn openai_sse_from_message_no_usage_chunk_without_include_usage() {
let resp = ModelResponse {
model: "gpt-4o".to_owned(),
text: "hi".to_owned(),
in_tokens: 3,
cache_write_tokens: 0,
cache_read_tokens: 0,
out_tokens: 2,
raw: serde_json::json!({ "content": [{ "type": "text", "text": "hi" }] }),
};
let sse = openai_sse_from_message(&openai_response_json(&resp), false);
let has_empty_choices_chunk = sse
.lines()
.filter_map(|l| l.strip_prefix("data: "))
.filter(|f| *f != "[DONE]")
.any(|d| {
serde_json::from_str::<Value>(d)
.ok()
.and_then(|v| v.get("choices").cloned())
.and_then(|c| c.as_array().map(|a| a.is_empty()))
.unwrap_or(false)
});
assert!(
!has_empty_choices_chunk,
"usage chunk must not appear when include_usage is false"
);
}
#[test]
fn stream_options_include_usage_parses_correctly() {
assert!(stream_options_include_usage(
br#"{"stream": true, "stream_options": {"include_usage": true}}"#
));
assert!(!stream_options_include_usage(
br#"{"stream": true, "stream_options": {"include_usage": false}}"#
));
assert!(!stream_options_include_usage(br#"{"stream": true}"#));
assert!(!stream_options_include_usage(b"not json"));
}
#[test]
fn translated_tool_use_input_is_always_an_object() {
for args in ["null", "[1]", "\"s\""] {
let resp = ModelResponse {
model: "gpt-4o".to_owned(),
text: String::new(),
in_tokens: 1,
cache_write_tokens: 0,
cache_read_tokens: 0,
out_tokens: 1,
raw: serde_json::json!({"choices": [{"finish_reason": "tool_calls", "message": {
"role": "assistant", "content": null,
"tool_calls": [{"id": "c", "type": "function",
"function": {"name": "f", "arguments": args}}]}}]}),
};
let json = anthropic_response_json(&resp);
assert!(
json["content"][0]["input"].is_object(),
"args={args}: {json}"
);
}
}
#[test]
fn anthropic_response_json_translates_openai_tool_calls_to_tool_use_blocks() {
let resp = ModelResponse {
model: "gpt-4o".to_owned(),
text: String::new(),
in_tokens: 5,
cache_write_tokens: 0,
cache_read_tokens: 0,
out_tokens: 3,
raw: serde_json::json!({
"choices": [{
"finish_reason": "tool_calls",
"message": {
"role": "assistant",
"content": null,
"tool_calls": [{
"id": "call_abc123",
"type": "function",
"function": {
"name": "get_weather",
"arguments": "{\"location\":\"London\"}"
}
}]
}
}],
"usage": {"prompt_tokens": 5, "completion_tokens": 3}
}),
};
let json = anthropic_response_json(&resp);
let content = json["content"].as_array().expect("content must be array");
assert_eq!(content.len(), 1, "exactly one tool_use block");
assert_eq!(content[0]["type"], "tool_use");
assert_eq!(content[0]["id"], "call_abc123");
assert_eq!(content[0]["name"], "get_weather");
assert_eq!(content[0]["input"]["location"], "London");
assert_eq!(json["stop_reason"], "tool_use");
}
#[test]
fn anthropic_response_json_openai_tool_calls_with_text_emits_both_blocks() {
let resp = ModelResponse {
model: "gpt-4o".to_owned(),
text: "Here is the weather call.".to_owned(),
in_tokens: 5,
cache_write_tokens: 0,
cache_read_tokens: 0,
out_tokens: 10,
raw: serde_json::json!({
"choices": [{
"finish_reason": "tool_calls",
"message": {
"content": "Here is the weather call.",
"tool_calls": [{
"id": "call_xyz",
"type": "function",
"function": { "name": "search", "arguments": "{}" }
}]
}
}]
}),
};
let json = anthropic_response_json(&resp);
let content = json["content"].as_array().expect("content array");
assert!(
content.iter().any(|b| b["type"] == "text"),
"text block must appear when message content is non-empty"
);
assert!(
content.iter().any(|b| b["type"] == "tool_use"),
"tool_use block must appear"
);
}
#[test]
fn anthropic_response_json_no_tool_use_blocks_never_emits_tool_use_stop_reason() {
let resp = ModelResponse {
model: "gpt-4o".to_owned(),
text: "some text".to_owned(),
in_tokens: 3,
cache_write_tokens: 0,
cache_read_tokens: 0,
out_tokens: 2,
raw: serde_json::json!({
"choices": [{
"finish_reason": "tool_calls",
"message": {
"content": "some text",
"tool_calls": [] }
}]
}),
};
let json = anthropic_response_json(&resp);
assert_eq!(
json["stop_reason"], "end_turn",
"stop_reason must be end_turn when content has no tool_use blocks"
);
}
#[test]
fn anthropic_sse_from_message_tool_use_block_produces_input_json_delta() {
let resp = ModelResponse {
model: "gpt-4o".to_owned(),
text: String::new(),
in_tokens: 4,
cache_write_tokens: 0,
cache_read_tokens: 0,
out_tokens: 2,
raw: serde_json::json!({
"choices": [{
"finish_reason": "tool_calls",
"message": {
"content": null,
"tool_calls": [{
"id": "call_1",
"type": "function",
"function": { "name": "lookup", "arguments": "{\"q\":\"test\"}" }
}]
}
}]
}),
};
let message = anthropic_response_json(&resp);
let sse = anthropic_sse_from_message(&message);
let frames: Vec<Value> = sse
.lines()
.filter_map(|l| l.strip_prefix("data: "))
.map(|d| serde_json::from_str::<Value>(d).expect("valid JSON"))
.collect();
assert!(
frames
.iter()
.any(|f| f["content_block"]["type"] == "tool_use"
&& f["content_block"]["name"] == "lookup"),
"content_block_start with tool_use block must appear"
);
assert!(
frames
.iter()
.any(|f| f["delta"]["type"] == "input_json_delta"),
"input_json_delta must be emitted for the tool call"
);
}
#[test]
fn anthropic_sse_message_start_has_null_stop_reason() {
let resp = thinking_tool_resp();
let message = anthropic_response_json(&resp);
let sse = anthropic_sse_from_message(&message);
let frames: Vec<Value> = sse
.lines()
.filter_map(|l| l.strip_prefix("data: "))
.map(|d| serde_json::from_str::<Value>(d).expect("valid JSON"))
.collect();
let start = frames
.iter()
.find(|f| f["type"] == "message_start")
.expect("message_start must appear");
assert!(
start["message"]["stop_reason"].is_null(),
"message_start.message.stop_reason must be null, got: {}",
start["message"]["stop_reason"]
);
assert!(
start["message"]["stop_sequence"].is_null(),
"message_start.message.stop_sequence must be null"
);
let delta = frames
.iter()
.find(|f| f["type"] == "message_delta")
.expect("message_delta must appear");
assert!(
!delta["delta"]["stop_reason"].is_null(),
"message_delta.delta.stop_reason must be non-null"
);
}
#[test]
fn anthropic_sse_message_start_has_zero_output_tokens() {
let resp = thinking_tool_resp(); let message = anthropic_response_json(&resp);
assert!(
message["usage"]["output_tokens"].as_u64().unwrap_or(0) > 0,
"precondition: source message must have non-zero output_tokens"
);
let sse = anthropic_sse_from_message(&message);
let frames: Vec<serde_json::Value> = sse
.lines()
.filter_map(|l| l.strip_prefix("data: "))
.map(|d| serde_json::from_str::<serde_json::Value>(d).expect("valid JSON"))
.collect();
let start = frames
.iter()
.find(|f| f["type"] == "message_start")
.expect("message_start must appear");
assert_eq!(
start["message"]["usage"]["output_tokens"].as_u64(),
Some(0),
"message_start.message.usage.output_tokens must be 0, got: {}",
start["message"]["usage"]["output_tokens"]
);
}
#[test]
fn anthropic_response_json_maps_gemini_finish_reason() {
for (gemini_reason, expected) in [("MAX_TOKENS", "max_tokens"), ("STOP", "end_turn")] {
let resp = ModelResponse {
model: "google/gemini-3.5-flash-lite".to_owned(),
text: "some text".to_owned(),
in_tokens: 5,
cache_write_tokens: 0,
cache_read_tokens: 0,
out_tokens: 3,
raw: serde_json::json!({
"candidates": [{"finishReason": gemini_reason,
"content": {"parts": [{"text": "some text"}]}}]
}),
};
let json = anthropic_response_json(&resp);
assert_eq!(
json["stop_reason"].as_str(),
Some(expected),
"Gemini finishReason={gemini_reason} → stop_reason={expected}"
);
}
}
#[tokio::test]
async fn count_tokens_passthrough_relays_upstream_response() {
let mock = axum::Router::new().route(
"/v1/messages/count_tokens",
axum::routing::post(|| async { axum::Json(serde_json::json!({"input_tokens": 42})) }),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move {
axum::serve(listener, mock).await.unwrap();
});
let upstream = format!("http://{addr}");
let config = ProxyConfig::from_lookup(move |k| match k {
"FIRSTPASS_UPSTREAM_ANTHROPIC" => Some(upstream.clone()),
_ => None,
})
.unwrap();
let (traces, mut rx) = mpsc::channel(64);
let state = AppState {
config: Arc::new(config),
http: reqwest::Client::new(),
providers: ProviderRegistry::new("http://127.0.0.1:1", "http://127.0.0.1:1"),
gate_health: Arc::new(GateHealthRegistry::new()),
shadow_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
paired_ledger: Arc::new(crate::shadow::ShadowLedger::new()),
experiments: Arc::default(),
guardrails: Arc::new(crate::guard::GuardrailRegistry::new()),
traces,
adaptive: None,
eprocess: None,
bandit: None,
promoter: None,
verified_cache: None,
predictor: None,
prior: None,
tenant_rate_limiter: None,
spill: None,
};
let body = Bytes::from_static(br#"{"model":"claude-3-haiku-20240307","messages":[]}"#);
let resp = count_tokens_passthrough(
State(state),
Extension(TenantId("default".to_owned())),
HeaderMap::new(),
body,
)
.await;
assert_eq!(resp.status(), axum::http::StatusCode::OK);
let json = body_json(resp).await;
assert_eq!(json["input_tokens"], 42, "token count must be relayed");
assert!(
rx.try_recv().is_err(),
"count_tokens must not enqueue a trace"
);
}
#[test]
fn percent_decode_ascii_handles_common_cases() {
assert_eq!(percent_decode_ascii("AIza-key"), "AIza-key");
assert_eq!(percent_decode_ascii("AIza%3Dkey"), "AIza=key"); assert_eq!(percent_decode_ascii("key%2Bplus"), "key+plus"); assert_eq!(percent_decode_ascii("%41%42%43"), "ABC"); assert_eq!(percent_decode_ascii("%"), "%"); assert_eq!(percent_decode_ascii("%XY"), "%XY"); }
#[test]
fn parse_gemini_query_decodes_key() {
let (key, scrubbed) = parse_gemini_query(Some("key=AIza%3Dsecret&alt=sse"));
assert_eq!(key.as_deref(), Some("AIza=secret"), "key must be decoded");
assert_eq!(scrubbed, "alt=sse", "key must be scrubbed from query");
}
#[test]
fn gemini_sse_tokens_and_finish_reason_parsed() {
let mut u = SseUsage::default();
let (mut buf, mut over) = (Vec::new(), false);
parse_sse_chunk_into(
b"data: {\"candidates\":[{\"finishReason\":\"STOP\"}],\"usageMetadata\":{\"promptTokenCount\":10,\"candidatesTokenCount\":5,\"thoughtsTokenCount\":2}}\n",
&mut buf,
&mut over,
&mut u,
);
assert_eq!(u.in_tokens, 10, "promptTokenCount");
assert_eq!(u.out_tokens, 7, "candidatesTokenCount + thoughtsTokenCount");
assert!(u.completed, "STOP sets completed");
}
#[test]
fn gemini_sse_finish_reason_and_usage_in_separate_chunks_still_complete() {
let mut u = SseUsage::default();
let (mut buf, mut over) = (Vec::new(), false);
parse_sse_chunk_into(
b"data: {\"candidates\":[{\"content\":{\"parts\":[{\"text\":\"hi\"}]},\"finishReason\":\"STOP\"}]}\n",
&mut buf,
&mut over,
&mut u,
);
parse_sse_chunk_into(
b"data: {\"usageMetadata\":{\"promptTokenCount\":100,\"cachedContentTokenCount\":60,\"candidatesTokenCount\":5}}\n",
&mut buf,
&mut over,
&mut u,
);
assert!(
u.completed,
"finishReason in its own chunk completes the stream"
);
assert_eq!(u.stop_reason.as_deref(), Some("STOP"));
assert_eq!(
u.in_tokens, 40,
"cached content is split out of promptTokenCount"
);
assert_eq!(u.cache_read_tokens, 60);
assert_eq!(u.out_tokens, 5);
}
#[test]
fn gemini_buffered_usage_splits_cached_content() {
let body = br#"{"candidates":[],"usageMetadata":{"promptTokenCount":100,"cachedContentTokenCount":60,"candidatesTokenCount":5,"thoughtsTokenCount":2}}"#;
let u = response_usage(body);
assert_eq!(
(u.in_tokens, u.cache_read_tokens, u.out_tokens),
(40, 60, 7)
);
}
#[test]
fn gemini_sse_intermediate_chunk_updates_tokens_without_completing() {
let mut u = SseUsage::default();
let (mut buf, mut over) = (Vec::new(), false);
parse_sse_chunk_into(
b"data: {\"candidates\":[{\"content\":{\"parts\":[{\"text\":\"hi\"}]}}],\"usageMetadata\":{\"promptTokenCount\":8,\"candidatesTokenCount\":3}}\n",
&mut buf,
&mut over,
&mut u,
);
assert_eq!(u.in_tokens, 8);
assert_eq!(u.out_tokens, 3);
assert!(!u.completed, "no finishReason → not completed");
}
#[test]
fn response_usage_parses_gemini_usage_metadata() {
let body = serde_json::json!({
"candidates": [{ "content": { "role": "model", "parts": [{ "text": "hi" }] } }],
"usageMetadata": {
"promptTokenCount": 20,
"candidatesTokenCount": 5,
"thoughtsTokenCount": 3,
"totalTokenCount": 28,
},
});
let usage = response_usage(body.to_string().as_bytes());
assert_eq!(usage.in_tokens, 20, "promptTokenCount");
assert_eq!(
usage.out_tokens, 8,
"candidatesTokenCount + thoughtsTokenCount"
);
assert_eq!(
usage.model, None,
"Gemini response has no top-level model field"
);
}
}