score_rows/
score_rows.rs

1//! Score a handful of rows against a running inference server.
2//!
3//! The upstream is any OpenAI-compatible service that returns next-token log
4//! probabilities — llama.cpp, vLLM, or another runtime the probe recognises.
5//!
6//! ```bash
7//! export JEV_BRIDGE_UPSTREAM_URL=http://127.0.0.1:8080/v1
8//! export JEV_BRIDGE_UPSTREAM_KEY=...          # only if the server requires it
9//! cargo run --example score_rows
10//! ```
11//!
12//! Every decision costs one prompt evaluation and zero output tokens.
13
14use std::time::Instant;
15
16use anyhow::{Context, Result};
17use serde_json::json;
18
19use jev_bridge::server::{Bridge, BridgeConfig};
20use jev_bridge::wire::{Row, RowOption};
21
22fn option(id: &str, description: &str) -> RowOption {
23    RowOption {
24        id: id.to_string(),
25        description: description.to_string(),
26    }
27}
28
29#[tokio::main]
30async fn main() -> Result<()> {
31    let openai_base = std::env::var("JEV_BRIDGE_UPSTREAM_URL")
32        .unwrap_or_else(|_| "http://127.0.0.1:8080/v1".to_string());
33    let native_base = std::env::var("JEV_BRIDGE_NATIVE_URL")
34        .unwrap_or_else(|_| openai_base.trim_end_matches("/v1").to_string());
35    let upstream_key = std::env::var("JEV_BRIDGE_UPSTREAM_KEY").ok();
36    let upstream_model =
37        std::env::var("JEV_BRIDGE_UPSTREAM_MODEL").unwrap_or_else(|_| "local-model".to_string());
38
39    let started = Instant::now();
40    let bridge = Bridge::connect(
41        reqwest::Client::new(),
42        BridgeConfig {
43            openai_base,
44            native_base,
45            upstream_model,
46            upstream_key,
47            served_model: "example-bridge".to_string(),
48            description: "example bridge".to_string(),
49            release_date: "2026-09-22".to_string(),
50            chat_template_kwargs: json!({"enable_thinking": false}),
51            max_input_tokens: None,
52            local: Default::default(),
53            readout: Default::default(),
54        },
55    )
56    .await
57    .context("connecting to the upstream service failed")?;
58
59    println!(
60        "connected in {:.2?}: transport={} endpoint={} renders_locally={} tokenizes_locally={}",
61        started.elapsed(),
62        bridge.probe().name(),
63        bridge.probe().endpoint(),
64        bridge.renders_locally(),
65        bridge.tokenizes_locally(),
66    );
67
68    let rows = vec![
69        Row {
70            id: "refund".to_string(),
71            state: json!("I was charged twice and need a refund today."),
72            question: "Which queue should handle this request?".to_string(),
73            options: vec![
74                option("access", "Account access and authentication."),
75                option("billing", "Billing, payments, and refunds."),
76                option("sales", "Pricing and new contracts."),
77            ],
78        },
79        Row {
80            id: "outage".to_string(),
81            state: json!("The API returns 502 for every request since this morning."),
82            question: "Which queue should handle this request?".to_string(),
83            options: vec![
84                option("access", "Account access and authentication."),
85                option("billing", "Billing, payments, and refunds."),
86                option("sales", "Pricing and new contracts."),
87            ],
88        },
89    ];
90
91    let started = Instant::now();
92    let scored = bridge.score_rows(&rows).await?;
93    println!("scored {} rows in {:.2?}\n", scored.len(), started.elapsed());
94
95    for row in &scored {
96        let winner = row
97            .answer
98            .probabilities
99            .iter()
100            .enumerate()
101            .max_by(|a, b| a.1.partial_cmp(b.1).unwrap())
102            .map(|(index, _)| row.answer.option_ids[index].as_str())
103            .unwrap();
104        println!("{} -> {winner}", row.answer.id);
105        for (id, probability) in row.answer.option_ids.iter().zip(&row.answer.probabilities) {
106            println!("    {id:<8} {probability:.4}");
107        }
108        println!(
109            "    input_tokens={} prompt_sha256={}",
110            row.answer.input_tokens,
111            &row.prompt_sha256[..16]
112        );
113    }
114
115    // Probabilities are conditional on the supplied options and uncalibrated.
116    println!("\nThese scores are conditional option probabilities, not calibrated confidence.");
117    Ok(())
118}
score_rows/score_rows.rs

score_rows/
score_rows.rs
