Skip to main content

firstpass_proxy/
prior.rs

1//! Verified predictive routing: the I/O side. Calls a pre-generation decision-model (TypeSafe's
2//! Jev today) with the request text and turns its answer into a `decision_prior` — a per-rung
3//! `P(pass | rung)` fed into the existing start-rung expected-cost argmin
4//! ([`firstpass_core::cumulative_pass`], [`crate::bandit`]).
5//!
6//! **This is a prior, not a verifier.** The enforce gate still runs on every served attempt
7//! exactly as before; nothing here can cause a failing output to be served. A timeout, a
8//! transport error, an unexpected status, or a response this module can't parse all fail open:
9//! `None`, a `firstpass_prior_errors_total` metric bump, and a `tracing::warn!` — never a panic,
10//! never added serving latency beyond the configured `timeout_ms`.
11
12use std::time::Duration;
13
14use firstpass_core::PriorConfig;
15use serde_json::Value;
16
17use crate::provider::ModelRequest;
18
19/// The single choice question sent to the decision-model. Its options are the ladder rungs
20/// (`r0..rN`, in ladder order); its `probabilities` map is the per-rung "this is the least
21/// capable tier that suffices" distribution.
22const QUESTION_NAME: &str = "rung";
23
24/// A configured verified-predictive-routing client: the shared HTTP client, the resolved config,
25/// and the API key read once at startup (never logged, never put on the trace).
26#[derive(Debug)]
27pub struct PriorClient {
28    http: reqwest::Client,
29    cfg: PriorConfig,
30    api_key: String,
31}
32
33impl PriorClient {
34    /// Build a client from a resolved config and API key.
35    #[must_use]
36    pub fn new(http: reqwest::Client, cfg: PriorConfig, api_key: String) -> Self {
37        Self { http, cfg, api_key }
38    }
39
40    /// Ladder rung labels, in ladder order (the choice question's criteria).
41    #[must_use]
42    pub fn rungs(&self) -> &[String] {
43        &self.cfg.rungs
44    }
45
46    /// Beta pseudo-count weight for the bandit blend (`[escalation.prior] strength`).
47    #[must_use]
48    pub fn strength(&self) -> f64 {
49        self.cfg.strength
50    }
51
52    /// Fetch the cumulative gate-pass prior for `query_text`, or `None` on any error/timeout/
53    /// malformed response (fail open — never blocks or corrupts serving).
54    pub async fn fetch(&self, query_text: &str) -> Option<Vec<f64>> {
55        fetch_prior(&self.http, &self.cfg, &self.api_key, query_text).await
56    }
57}
58
59/// Build the request text a prior should evaluate: the system prompt (if any) plus every
60/// message's text projection, newline-joined. Pure and unit-testable independent of the network
61/// call.
62#[must_use]
63pub fn query_text(req: &ModelRequest) -> String {
64    let mut parts: Vec<String> = Vec::with_capacity(req.messages.len() + 1);
65    if let Some(system) = req.system.as_deref() {
66        parts.push(system.to_owned());
67    }
68    parts.extend(
69        req.messages
70            .iter()
71            .map(crate::provider::ChatMessage::text_view),
72    );
73    parts.join("\n")
74}
75
76/// Build the Jev `/v1/systemone` request body: one choice question whose criteria are the
77/// configured ladder rung labels (`r0..rN`).
78fn build_request(cfg: &PriorConfig, query_text: &str) -> Value {
79    let mut criteria = serde_json::Map::new();
80    for (i, rung) in cfg.rungs.iter().enumerate() {
81        criteria.insert(format!("r{i}"), Value::String(rung.clone()));
82    }
83    serde_json::json!({
84        "model": cfg.model,
85        "state": query_text,
86        "questions": {
87            QUESTION_NAME: {
88                "type": "choice",
89                "instructions": "Which is the least capable model tier that will fully and \
90                    correctly handle this request?",
91                "criteria": criteria,
92            }
93        }
94    })
95}
96
97/// Extract the `rung` question's `probabilities` map from a Jev response, in ladder order
98/// (`r0..rN`; a missing option maps to `0.0`). Accepts either `{"answers":{"rung":{...}}}` or
99/// `{"rung":{...}}` at top level — anything else, or a missing/non-object `probabilities`,
100/// yields `None`.
101fn extract_probabilities(json: &Value, cfg: &PriorConfig) -> Option<Vec<f64>> {
102    let answer = json
103        .get("answers")
104        .and_then(|a| a.get(QUESTION_NAME))
105        .or_else(|| json.get(QUESTION_NAME))?;
106    let probabilities = answer.get("probabilities")?.as_object()?;
107    Some(
108        (0..cfg.rungs.len())
109            .map(|i| {
110                probabilities
111                    .get(&format!("r{i}"))
112                    .and_then(Value::as_f64)
113                    .unwrap_or(0.0)
114            })
115            .collect(),
116    )
117}
118
119/// Call the decision-model, map its answer to a cumulative `P(pass | rung)` prior, and fail open
120/// on anything that isn't a clean, well-formed, in-time answer.
121///
122/// Never panics: every fallible step (transport, timeout, status, decode, shape, math) returns
123/// `None` rather than propagating an error, because a wrong or missing prior can only cost money
124/// (a suboptimal start rung) — this call is never allowed to block or corrupt serving.
125pub async fn fetch_prior(
126    client: &reqwest::Client,
127    cfg: &PriorConfig,
128    api_key: &str,
129    query_text: &str,
130) -> Option<Vec<f64>> {
131    let body = build_request(cfg, query_text);
132    let url = format!("{}/v1/systemone", cfg.base_url.trim_end_matches('/'));
133    let call = client.post(url).bearer_auth(api_key).json(&body).send();
134
135    let resp = match tokio::time::timeout(Duration::from_millis(cfg.timeout_ms), call).await {
136        Ok(Ok(resp)) => resp,
137        Ok(Err(e)) => {
138            metrics::counter!("firstpass_prior_errors_total").increment(1);
139            tracing::warn!(error = %e, "verified predictive routing: transport error, prior disabled for this request");
140            return None;
141        }
142        Err(_) => {
143            metrics::counter!("firstpass_prior_errors_total").increment(1);
144            tracing::warn!(
145                timeout_ms = cfg.timeout_ms,
146                "verified predictive routing: prior call timed out, prior disabled for this request"
147            );
148            return None;
149        }
150    };
151
152    if !resp.status().is_success() {
153        metrics::counter!("firstpass_prior_errors_total").increment(1);
154        tracing::warn!(
155            status = resp.status().as_u16(),
156            "verified predictive routing: non-2xx response, prior disabled for this request"
157        );
158        return None;
159    }
160
161    let json: Value = match resp.json().await {
162        Ok(j) => j,
163        Err(e) => {
164            metrics::counter!("firstpass_prior_errors_total").increment(1);
165            tracing::warn!(error = %e, "verified predictive routing: undecodable response, prior disabled for this request");
166            return None;
167        }
168    };
169
170    let Some(probs) = extract_probabilities(&json, cfg) else {
171        metrics::counter!("firstpass_prior_errors_total").increment(1);
172        tracing::warn!(
173            "verified predictive routing: response missing the expected answer shape, prior \
174             disabled for this request"
175        );
176        return None;
177    };
178
179    let prior = firstpass_core::cumulative_pass(&probs);
180    if prior.is_none() {
181        metrics::counter!("firstpass_prior_errors_total").increment(1);
182        tracing::warn!(
183            "verified predictive routing: probabilities could not be normalized, prior disabled \
184             for this request"
185        );
186    }
187    prior
188}
189
190#[cfg(test)]
191mod tests {
192    use axum::response::IntoResponse;
193
194    use super::*;
195
196    fn test_cfg(rungs: &[&str]) -> PriorConfig {
197        PriorConfig {
198            provider: "typesafe".to_owned(),
199            model: "jev-latest".to_owned(),
200            api_key_env: "TYPESAFE_API_KEY".to_owned(),
201            base_url: "https://api.typesafe.ai".to_owned(),
202            timeout_ms: 150,
203            strength: 10.0,
204            rungs: rungs.iter().map(|s| (*s).to_owned()).collect(),
205        }
206    }
207
208    /// Live wire check against a real `/v1/systemone` server (e.g. a local OpenJev). Opt-in:
209    /// `OPENJEV_URL=http://127.0.0.1:8080 cargo test -p firstpass-proxy -- --ignored live_`.
210    #[tokio::test]
211    #[ignore = "needs a live /v1/systemone server in OPENJEV_URL"]
212    async fn live_prior_reads_a_real_choice_answer() {
213        let Ok(url) = std::env::var("OPENJEV_URL") else {
214            return;
215        };
216        let mut cfg = test_cfg(&[
217            "a small fast model fully handles this",
218            "only a frontier model fully handles this",
219        ]);
220        cfg.base_url = url;
221        cfg.timeout_ms = 60_000;
222        let client = PriorClient::new(reqwest::Client::new(), cfg, "local".to_owned());
223        let easy = client
224            .fetch("Write a python function to reverse a string.")
225            .await;
226        let hard = client
227            .fetch("Prove termination and implement a linear-time parser for nested balanced expressions with escapes and error recovery.")
228            .await;
229        let (easy, hard) = (easy.expect("easy prior"), hard.expect("hard prior"));
230        assert_eq!(easy.len(), 2);
231        assert!(
232            (easy[1] - 1.0).abs() < 1e-9,
233            "cumulative ends at 1: {easy:?}"
234        );
235        assert!(
236            easy[0] > hard[0],
237            "easy {easy:?} should favour rung 0 over hard {hard:?}"
238        );
239    }
240
241    #[test]
242    fn query_text_joins_system_and_messages() {
243        let req = ModelRequest {
244            model: "anthropic/claude-haiku-4-5".to_owned(),
245            system: Some("be terse".to_owned()),
246            messages: vec![
247                crate::provider::ChatMessage::text("user", "fix the bug"),
248                crate::provider::ChatMessage::text("assistant", "on it"),
249            ],
250            max_tokens: 64,
251            tools: Value::Null,
252            raw: Value::Null,
253            cache_prefix: false,
254        };
255        assert_eq!(query_text(&req), "be terse\nfix the bug\non it");
256    }
257
258    #[test]
259    fn query_text_without_system_skips_it() {
260        let req = ModelRequest {
261            model: "m".to_owned(),
262            system: None,
263            messages: vec![crate::provider::ChatMessage::text("user", "hi")],
264            max_tokens: 1,
265            tools: Value::Null,
266            raw: Value::Null,
267            cache_prefix: false,
268        };
269        assert_eq!(query_text(&req), "hi");
270    }
271
272    #[test]
273    fn build_request_shape() {
274        let cfg = test_cfg(&["cheap", "strong"]);
275        let body = build_request(&cfg, "hello");
276        assert_eq!(body["model"], "jev-latest");
277        assert_eq!(body["state"], "hello");
278        assert_eq!(body["questions"]["rung"]["type"], "choice");
279        assert_eq!(body["questions"]["rung"]["criteria"]["r0"], "cheap");
280        assert_eq!(body["questions"]["rung"]["criteria"]["r1"], "strong");
281    }
282
283    #[test]
284    fn extract_probabilities_from_nested_answers_map() {
285        let cfg = test_cfg(&["cheap", "strong"]);
286        let json = serde_json::json!({
287            "answers": { "rung": { "choice": "r1", "confidence": 0.8,
288                "probabilities": { "r0": 0.2, "r1": 0.8 } } }
289        });
290        assert_eq!(extract_probabilities(&json, &cfg), Some(vec![0.2, 0.8]));
291    }
292
293    #[test]
294    fn extract_probabilities_from_flat_top_level() {
295        let cfg = test_cfg(&["cheap", "strong"]);
296        let json = serde_json::json!({
297            "rung": { "choice": "r0", "confidence": 0.6, "probabilities": { "r0": 0.6, "r1": 0.4 } }
298        });
299        assert_eq!(extract_probabilities(&json, &cfg), Some(vec![0.6, 0.4]));
300    }
301
302    #[test]
303    fn extract_probabilities_missing_option_defaults_to_zero() {
304        let cfg = test_cfg(&["cheap", "strong", "top"]);
305        let json = serde_json::json!({
306            "rung": { "probabilities": { "r0": 0.5, "r2": 0.5 } }
307        });
308        assert_eq!(
309            extract_probabilities(&json, &cfg),
310            Some(vec![0.5, 0.0, 0.5])
311        );
312    }
313
314    #[test]
315    fn extract_probabilities_rejects_unexpected_shape() {
316        let cfg = test_cfg(&["cheap", "strong"]);
317        assert_eq!(extract_probabilities(&serde_json::json!({}), &cfg), None);
318        assert_eq!(
319            extract_probabilities(&serde_json::json!({"rung": {}}), &cfg),
320            None
321        );
322        assert_eq!(
323            extract_probabilities(
324                &serde_json::json!({"rung": {"probabilities": "nope"}}),
325                &cfg
326            ),
327            None
328        );
329    }
330
331    // ---- fetch_prior against a fake axum `/v1/systemone` server ----
332
333    async fn spawn_fake_server(
334        respond: impl Fn() -> axum::response::Response + Send + Sync + 'static,
335    ) -> String {
336        use axum::routing::post;
337        let respond = std::sync::Arc::new(respond);
338        let app = axum::Router::new().route(
339            "/v1/systemone",
340            post(move || {
341                let respond = respond.clone();
342                async move { (*respond)() }
343            }),
344        );
345        let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
346            .await
347            .expect("bind");
348        let addr = listener.local_addr().expect("addr");
349        tokio::spawn(async move {
350            let _ = axum::serve(listener, app).await;
351        });
352        format!("http://{addr}")
353    }
354
355    #[tokio::test]
356    async fn fetch_prior_maps_a_confident_answer() {
357        let base_url = spawn_fake_server(|| {
358            axum::Json(serde_json::json!({
359                "answers": { "rung": { "choice": "r2", "confidence": 0.9,
360                    "probabilities": { "r0": 0.05, "r1": 0.1, "r2": 0.85 } } }
361            }))
362            .into_response()
363        })
364        .await;
365        let mut cfg = test_cfg(&["cheap", "mid", "top"]);
366        cfg.base_url = base_url;
367        let client = reqwest::Client::new();
368        let prior = fetch_prior(&client, &cfg, "test-key", "hard request")
369            .await
370            .expect("a well-formed answer must produce a prior");
371        assert_eq!(prior.len(), 3);
372        assert!(prior[0] < prior[1] && prior[1] < prior[2]);
373        assert_eq!(*prior.last().unwrap(), 1.0);
374    }
375
376    #[tokio::test]
377    async fn fetch_prior_fails_open_on_500() {
378        let base_url = spawn_fake_server(|| {
379            (axum::http::StatusCode::INTERNAL_SERVER_ERROR, "boom").into_response()
380        })
381        .await;
382        let mut cfg = test_cfg(&["cheap", "top"]);
383        cfg.base_url = base_url;
384        let client = reqwest::Client::new();
385        assert_eq!(
386            fetch_prior(&client, &cfg, "test-key", "anything").await,
387            None
388        );
389    }
390
391    #[tokio::test]
392    async fn fetch_prior_fails_open_on_malformed_body() {
393        let base_url = spawn_fake_server(|| {
394            axum::Json(serde_json::json!({"unexpected": true})).into_response()
395        })
396        .await;
397        let mut cfg = test_cfg(&["cheap", "top"]);
398        cfg.base_url = base_url;
399        let client = reqwest::Client::new();
400        assert_eq!(
401            fetch_prior(&client, &cfg, "test-key", "anything").await,
402            None
403        );
404    }
405
406    #[tokio::test]
407    async fn fetch_prior_fails_open_on_timeout() {
408        let base_url = spawn_fake_server(|| {
409            // The handler never returns before the client's short timeout fires. axum server
410            // itself has no timeout, but a slow handler models what a stalled upstream does.
411            axum::Json(serde_json::json!({})).into_response()
412        })
413        .await;
414        let mut cfg = test_cfg(&["cheap", "top"]);
415        cfg.base_url = base_url;
416        cfg.timeout_ms = 0; // instantly elapsed
417        let client = reqwest::Client::new();
418        // timeout_ms = 0 would fail config validation in production, but the fetch path itself
419        // must still handle a zero/near-zero duration without panicking.
420        assert_eq!(
421            fetch_prior(&client, &cfg, "test-key", "anything").await,
422            None
423        );
424    }
425}