score_rows/
score_rows.rs

1//! Score a handful of rows against a running inference server.
2//!
3//! The upstream is any OpenAI-compatible service that returns next-token log
4//! probabilities — llama.cpp, vLLM, or another runtime the probe recognises.
5//!
6//! ```bash
7//! export JEV_BRIDGE_UPSTREAM_URL=http://127.0.0.1:8080/v1
8//! export JEV_BRIDGE_UPSTREAM_KEY=...          # only if the server requires it
9//! cargo run --example score_rows
10//! ```
11//!
12//! Every decision costs one prompt evaluation and zero output tokens.
13
14use std::time::Instant;
15
16use anyhow::{Context, Result};
17use serde_json::json;
18
19use jev_bridge::server::{Bridge, BridgeConfig};
20use jev_bridge::wire::{Row, RowOption};
21
22fn option(id: &str, description: &str) -> RowOption {
23    RowOption {
24        id: id.to_string(),
25        description: description.to_string(),
26    }
27}
28
29#[tokio::main]
30async fn main() -> Result<()> {
31    let openai_base = std::env::var("JEV_BRIDGE_UPSTREAM_URL")
32        .unwrap_or_else(|_| "http://127.0.0.1:8080/v1".to_string());
33    let native_base = std::env::var("JEV_BRIDGE_NATIVE_URL")
34        .unwrap_or_else(|_| openai_base.trim_end_matches("/v1").to_string());
35    let upstream_key = std::env::var("JEV_BRIDGE_UPSTREAM_KEY").ok();
36    let upstream_model =
37        std::env::var("JEV_BRIDGE_UPSTREAM_MODEL").unwrap_or_else(|_| "local-model".to_string());
38
39    let started = Instant::now();
40    let bridge = Bridge::connect(
41        reqwest::Client::new(),
42        BridgeConfig {
43            openai_base,
44            native_base,
45            upstream_model,
46            upstream_key,
47            served_model: "example-bridge".to_string(),
48            description: "example bridge".to_string(),
49            release_date: "2026-09-22".to_string(),
50            chat_template_kwargs: json!({"enable_thinking": false}),
51            max_input_tokens: None,
52            local: Default::default(),
53        },
54    )
55    .await
56    .context("connecting to the upstream service failed")?;
57
58    println!(
59        "connected in {:.2?}: transport={} endpoint={} renders_locally={} tokenizes_locally={}",
60        started.elapsed(),
61        bridge.probe().name(),
62        bridge.probe().endpoint(),
63        bridge.renders_locally(),
64        bridge.tokenizes_locally(),
65    );
66
67    let rows = vec![
68        Row {
69            id: "refund".to_string(),
70            state: json!("I was charged twice and need a refund today."),
71            question: "Which queue should handle this request?".to_string(),
72            options: vec![
73                option("access", "Account access and authentication."),
74                option("billing", "Billing, payments, and refunds."),
75                option("sales", "Pricing and new contracts."),
76            ],
77        },
78        Row {
79            id: "outage".to_string(),
80            state: json!("The API returns 502 for every request since this morning."),
81            question: "Which queue should handle this request?".to_string(),
82            options: vec![
83                option("access", "Account access and authentication."),
84                option("billing", "Billing, payments, and refunds."),
85                option("sales", "Pricing and new contracts."),
86            ],
87        },
88    ];
89
90    let started = Instant::now();
91    let scored = bridge.score_rows(&rows).await?;
92    println!("scored {} rows in {:.2?}\n", scored.len(), started.elapsed());
93
94    for row in &scored {
95        let winner = row
96            .answer
97            .probabilities
98            .iter()
99            .enumerate()
100            .max_by(|a, b| a.1.partial_cmp(b.1).unwrap())
101            .map(|(index, _)| row.answer.option_ids[index].as_str())
102            .unwrap();
103        println!("{} -> {winner}", row.answer.id);
104        for (id, probability) in row.answer.option_ids.iter().zip(&row.answer.probabilities) {
105            println!("    {id:<8} {probability:.4}");
106        }
107        println!(
108            "    input_tokens={} prompt_sha256={}",
109            row.answer.input_tokens,
110            &row.prompt_sha256[..16]
111        );
112    }
113
114    // Probabilities are conditional on the supplied options and uncalibrated.
115    println!("\nThese scores are conditional option probabilities, not calibrated confidence.");
116    Ok(())
117}
score_rows/score_rows.rs

score_rows/
score_rows.rs
