1use std::time::Duration;
13
14use firstpass_core::PriorConfig;
15use serde_json::Value;
16
17use crate::provider::ModelRequest;
18
19const QUESTION_NAME: &str = "rung";
23
24#[derive(Debug)]
27pub struct PriorClient {
28 http: reqwest::Client,
29 cfg: PriorConfig,
30 api_key: String,
31}
32
33impl PriorClient {
34 #[must_use]
36 pub fn new(http: reqwest::Client, cfg: PriorConfig, api_key: String) -> Self {
37 Self { http, cfg, api_key }
38 }
39
40 #[must_use]
42 pub fn rungs(&self) -> &[String] {
43 &self.cfg.rungs
44 }
45
46 #[must_use]
48 pub fn strength(&self) -> f64 {
49 self.cfg.strength
50 }
51
52 pub async fn fetch(&self, query_text: &str) -> Option<Vec<f64>> {
55 fetch_prior(&self.http, &self.cfg, &self.api_key, query_text).await
56 }
57}
58
59#[must_use]
63pub fn query_text(req: &ModelRequest) -> String {
64 let mut parts: Vec<String> = Vec::with_capacity(req.messages.len() + 1);
65 if let Some(system) = req.system.as_deref() {
66 parts.push(system.to_owned());
67 }
68 parts.extend(
69 req.messages
70 .iter()
71 .map(crate::provider::ChatMessage::text_view),
72 );
73 parts.join("\n")
74}
75
76fn build_request(cfg: &PriorConfig, query_text: &str) -> Value {
79 let mut criteria = serde_json::Map::new();
80 for (i, rung) in cfg.rungs.iter().enumerate() {
81 criteria.insert(format!("r{i}"), Value::String(rung.clone()));
82 }
83 serde_json::json!({
84 "model": cfg.model,
85 "state": query_text,
86 "questions": {
87 QUESTION_NAME: {
88 "type": "choice",
89 "instructions": "Which is the least capable model tier that will fully and \
90 correctly handle this request?",
91 "criteria": criteria,
92 }
93 }
94 })
95}
96
97fn extract_probabilities(json: &Value, cfg: &PriorConfig) -> Option<Vec<f64>> {
102 let answer = json
103 .get("answers")
104 .and_then(|a| a.get(QUESTION_NAME))
105 .or_else(|| json.get(QUESTION_NAME))?;
106 let probabilities = answer.get("probabilities")?.as_object()?;
107 Some(
108 (0..cfg.rungs.len())
109 .map(|i| {
110 probabilities
111 .get(&format!("r{i}"))
112 .and_then(Value::as_f64)
113 .unwrap_or(0.0)
114 })
115 .collect(),
116 )
117}
118
119pub async fn fetch_prior(
126 client: &reqwest::Client,
127 cfg: &PriorConfig,
128 api_key: &str,
129 query_text: &str,
130) -> Option<Vec<f64>> {
131 let body = build_request(cfg, query_text);
132 let url = format!("{}/v1/systemone", cfg.base_url.trim_end_matches('/'));
133 let call = client.post(url).bearer_auth(api_key).json(&body).send();
134
135 let resp = match tokio::time::timeout(Duration::from_millis(cfg.timeout_ms), call).await {
136 Ok(Ok(resp)) => resp,
137 Ok(Err(e)) => {
138 metrics::counter!("firstpass_prior_errors_total").increment(1);
139 tracing::warn!(error = %e, "verified predictive routing: transport error, prior disabled for this request");
140 return None;
141 }
142 Err(_) => {
143 metrics::counter!("firstpass_prior_errors_total").increment(1);
144 tracing::warn!(
145 timeout_ms = cfg.timeout_ms,
146 "verified predictive routing: prior call timed out, prior disabled for this request"
147 );
148 return None;
149 }
150 };
151
152 if !resp.status().is_success() {
153 metrics::counter!("firstpass_prior_errors_total").increment(1);
154 tracing::warn!(
155 status = resp.status().as_u16(),
156 "verified predictive routing: non-2xx response, prior disabled for this request"
157 );
158 return None;
159 }
160
161 let json: Value = match resp.json().await {
162 Ok(j) => j,
163 Err(e) => {
164 metrics::counter!("firstpass_prior_errors_total").increment(1);
165 tracing::warn!(error = %e, "verified predictive routing: undecodable response, prior disabled for this request");
166 return None;
167 }
168 };
169
170 let Some(probs) = extract_probabilities(&json, cfg) else {
171 metrics::counter!("firstpass_prior_errors_total").increment(1);
172 tracing::warn!(
173 "verified predictive routing: response missing the expected answer shape, prior \
174 disabled for this request"
175 );
176 return None;
177 };
178
179 let prior = firstpass_core::cumulative_pass(&probs);
180 if prior.is_none() {
181 metrics::counter!("firstpass_prior_errors_total").increment(1);
182 tracing::warn!(
183 "verified predictive routing: probabilities could not be normalized, prior disabled \
184 for this request"
185 );
186 }
187 prior
188}
189
190#[cfg(test)]
191mod tests {
192 use axum::response::IntoResponse;
193
194 use super::*;
195
196 fn test_cfg(rungs: &[&str]) -> PriorConfig {
197 PriorConfig {
198 provider: "typesafe".to_owned(),
199 model: "jev-latest".to_owned(),
200 api_key_env: "TYPESAFE_API_KEY".to_owned(),
201 base_url: "https://api.typesafe.ai".to_owned(),
202 timeout_ms: 150,
203 strength: 10.0,
204 rungs: rungs.iter().map(|s| (*s).to_owned()).collect(),
205 }
206 }
207
208 #[tokio::test]
211 #[ignore = "needs a live /v1/systemone server in OPENJEV_URL"]
212 async fn live_prior_reads_a_real_choice_answer() {
213 let Ok(url) = std::env::var("OPENJEV_URL") else {
214 return;
215 };
216 let mut cfg = test_cfg(&[
217 "a small fast model fully handles this",
218 "only a frontier model fully handles this",
219 ]);
220 cfg.base_url = url;
221 cfg.timeout_ms = 60_000;
222 let client = PriorClient::new(reqwest::Client::new(), cfg, "local".to_owned());
223 let easy = client
224 .fetch("Write a python function to reverse a string.")
225 .await;
226 let hard = client
227 .fetch("Prove termination and implement a linear-time parser for nested balanced expressions with escapes and error recovery.")
228 .await;
229 let (easy, hard) = (easy.expect("easy prior"), hard.expect("hard prior"));
230 assert_eq!(easy.len(), 2);
231 assert!(
232 (easy[1] - 1.0).abs() < 1e-9,
233 "cumulative ends at 1: {easy:?}"
234 );
235 assert!(
236 easy[0] > hard[0],
237 "easy {easy:?} should favour rung 0 over hard {hard:?}"
238 );
239 }
240
241 #[test]
242 fn query_text_joins_system_and_messages() {
243 let req = ModelRequest {
244 model: "anthropic/claude-haiku-4-5".to_owned(),
245 system: Some("be terse".to_owned()),
246 messages: vec![
247 crate::provider::ChatMessage::text("user", "fix the bug"),
248 crate::provider::ChatMessage::text("assistant", "on it"),
249 ],
250 max_tokens: 64,
251 tools: Value::Null,
252 raw: Value::Null,
253 cache_prefix: false,
254 };
255 assert_eq!(query_text(&req), "be terse\nfix the bug\non it");
256 }
257
258 #[test]
259 fn query_text_without_system_skips_it() {
260 let req = ModelRequest {
261 model: "m".to_owned(),
262 system: None,
263 messages: vec![crate::provider::ChatMessage::text("user", "hi")],
264 max_tokens: 1,
265 tools: Value::Null,
266 raw: Value::Null,
267 cache_prefix: false,
268 };
269 assert_eq!(query_text(&req), "hi");
270 }
271
272 #[test]
273 fn build_request_shape() {
274 let cfg = test_cfg(&["cheap", "strong"]);
275 let body = build_request(&cfg, "hello");
276 assert_eq!(body["model"], "jev-latest");
277 assert_eq!(body["state"], "hello");
278 assert_eq!(body["questions"]["rung"]["type"], "choice");
279 assert_eq!(body["questions"]["rung"]["criteria"]["r0"], "cheap");
280 assert_eq!(body["questions"]["rung"]["criteria"]["r1"], "strong");
281 }
282
283 #[test]
284 fn extract_probabilities_from_nested_answers_map() {
285 let cfg = test_cfg(&["cheap", "strong"]);
286 let json = serde_json::json!({
287 "answers": { "rung": { "choice": "r1", "confidence": 0.8,
288 "probabilities": { "r0": 0.2, "r1": 0.8 } } }
289 });
290 assert_eq!(extract_probabilities(&json, &cfg), Some(vec![0.2, 0.8]));
291 }
292
293 #[test]
294 fn extract_probabilities_from_flat_top_level() {
295 let cfg = test_cfg(&["cheap", "strong"]);
296 let json = serde_json::json!({
297 "rung": { "choice": "r0", "confidence": 0.6, "probabilities": { "r0": 0.6, "r1": 0.4 } }
298 });
299 assert_eq!(extract_probabilities(&json, &cfg), Some(vec![0.6, 0.4]));
300 }
301
302 #[test]
303 fn extract_probabilities_missing_option_defaults_to_zero() {
304 let cfg = test_cfg(&["cheap", "strong", "top"]);
305 let json = serde_json::json!({
306 "rung": { "probabilities": { "r0": 0.5, "r2": 0.5 } }
307 });
308 assert_eq!(
309 extract_probabilities(&json, &cfg),
310 Some(vec![0.5, 0.0, 0.5])
311 );
312 }
313
314 #[test]
315 fn extract_probabilities_rejects_unexpected_shape() {
316 let cfg = test_cfg(&["cheap", "strong"]);
317 assert_eq!(extract_probabilities(&serde_json::json!({}), &cfg), None);
318 assert_eq!(
319 extract_probabilities(&serde_json::json!({"rung": {}}), &cfg),
320 None
321 );
322 assert_eq!(
323 extract_probabilities(
324 &serde_json::json!({"rung": {"probabilities": "nope"}}),
325 &cfg
326 ),
327 None
328 );
329 }
330
331 async fn spawn_fake_server(
334 respond: impl Fn() -> axum::response::Response + Send + Sync + 'static,
335 ) -> String {
336 use axum::routing::post;
337 let respond = std::sync::Arc::new(respond);
338 let app = axum::Router::new().route(
339 "/v1/systemone",
340 post(move || {
341 let respond = respond.clone();
342 async move { (*respond)() }
343 }),
344 );
345 let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
346 .await
347 .expect("bind");
348 let addr = listener.local_addr().expect("addr");
349 tokio::spawn(async move {
350 let _ = axum::serve(listener, app).await;
351 });
352 format!("http://{addr}")
353 }
354
355 #[tokio::test]
356 async fn fetch_prior_maps_a_confident_answer() {
357 let base_url = spawn_fake_server(|| {
358 axum::Json(serde_json::json!({
359 "answers": { "rung": { "choice": "r2", "confidence": 0.9,
360 "probabilities": { "r0": 0.05, "r1": 0.1, "r2": 0.85 } } }
361 }))
362 .into_response()
363 })
364 .await;
365 let mut cfg = test_cfg(&["cheap", "mid", "top"]);
366 cfg.base_url = base_url;
367 let client = reqwest::Client::new();
368 let prior = fetch_prior(&client, &cfg, "test-key", "hard request")
369 .await
370 .expect("a well-formed answer must produce a prior");
371 assert_eq!(prior.len(), 3);
372 assert!(prior[0] < prior[1] && prior[1] < prior[2]);
373 assert_eq!(*prior.last().unwrap(), 1.0);
374 }
375
376 #[tokio::test]
377 async fn fetch_prior_fails_open_on_500() {
378 let base_url = spawn_fake_server(|| {
379 (axum::http::StatusCode::INTERNAL_SERVER_ERROR, "boom").into_response()
380 })
381 .await;
382 let mut cfg = test_cfg(&["cheap", "top"]);
383 cfg.base_url = base_url;
384 let client = reqwest::Client::new();
385 assert_eq!(
386 fetch_prior(&client, &cfg, "test-key", "anything").await,
387 None
388 );
389 }
390
391 #[tokio::test]
392 async fn fetch_prior_fails_open_on_malformed_body() {
393 let base_url = spawn_fake_server(|| {
394 axum::Json(serde_json::json!({"unexpected": true})).into_response()
395 })
396 .await;
397 let mut cfg = test_cfg(&["cheap", "top"]);
398 cfg.base_url = base_url;
399 let client = reqwest::Client::new();
400 assert_eq!(
401 fetch_prior(&client, &cfg, "test-key", "anything").await,
402 None
403 );
404 }
405
406 #[tokio::test]
407 async fn fetch_prior_fails_open_on_timeout() {
408 let base_url = spawn_fake_server(|| {
409 axum::Json(serde_json::json!({})).into_response()
412 })
413 .await;
414 let mut cfg = test_cfg(&["cheap", "top"]);
415 cfg.base_url = base_url;
416 cfg.timeout_ms = 0; let client = reqwest::Client::new();
418 assert_eq!(
421 fetch_prior(&client, &cfg, "test-key", "anything").await,
422 None
423 );
424 }
425}