score_rows/
score_rows.rs

1//! Score a handful of rows against a running inference server.
2//!
3//! The upstream is any OpenAI-compatible service that returns next-token log
4//! probabilities — llama.cpp, vLLM, or another runtime the probe recognises.
5//!
6//! ```bash
7//! export JEV_BRIDGE_UPSTREAM_URL=http://127.0.0.1:8080/v1
8//! export JEV_BRIDGE_UPSTREAM_KEY=...          # only if the server requires it
9//! cargo run --example score_rows
10//! ```
11//!
12//! Every decision costs one prompt evaluation and zero output tokens.
13
14use std::time::Instant;
15
16use anyhow::{Context, Result};
17use serde_json::json;
18
19use jev_bridge::server::{Bridge, BridgeConfig};
20use jev_bridge::wire::{Row, RowOption};
21
22fn option(id: &str, description: &str) -> RowOption {
23    RowOption {
24        id: id.to_string(),
25        description: description.to_string(),
26    }
27}
28
29#[tokio::main]
30async fn main() -> Result<()> {
31    let openai_base = std::env::var("JEV_BRIDGE_UPSTREAM_URL")
32        .unwrap_or_else(|_| "http://127.0.0.1:8080/v1".to_string());
33    let native_base = std::env::var("JEV_BRIDGE_NATIVE_URL")
34        .unwrap_or_else(|_| openai_base.trim_end_matches("/v1").to_string());
35    let upstream_key = std::env::var("JEV_BRIDGE_UPSTREAM_KEY").ok();
36    let upstream_model =
37        std::env::var("JEV_BRIDGE_UPSTREAM_MODEL").unwrap_or_else(|_| "local-model".to_string());
38
39    let started = Instant::now();
40    let bridge = Bridge::connect(
41        reqwest::Client::new(),
42        BridgeConfig {
43            openai_base,
44            native_base,
45            upstream_model,
46            upstream_key,
47            served_model: "example-bridge".to_string(),
48            description: "example bridge".to_string(),
49            release_date: "2026-09-22".to_string(),
50            chat_template_kwargs: json!({"enable_thinking": false}),
51            max_input_tokens: None,
52            local: Default::default(),
53            accept_any_model: false,
54            readout: Default::default(),
55        },
56    )
57    .await
58    .context("connecting to the upstream service failed")?;
59
60    println!(
61        "connected in {:.2?}: transport={} endpoint={} renders_locally={} tokenizes_locally={}",
62        started.elapsed(),
63        bridge.probe().name(),
64        bridge.probe().endpoint(),
65        bridge.renders_locally(),
66        bridge.tokenizes_locally(),
67    );
68
69    let rows = vec![
70        Row {
71            id: "refund".to_string(),
72            state: json!("I was charged twice and need a refund today."),
73            question: "Which queue should handle this request?".to_string(),
74            options: vec![
75                option("access", "Account access and authentication."),
76                option("billing", "Billing, payments, and refunds."),
77                option("sales", "Pricing and new contracts."),
78            ],
79        },
80        Row {
81            id: "outage".to_string(),
82            state: json!("The API returns 502 for every request since this morning."),
83            question: "Which queue should handle this request?".to_string(),
84            options: vec![
85                option("access", "Account access and authentication."),
86                option("billing", "Billing, payments, and refunds."),
87                option("sales", "Pricing and new contracts."),
88            ],
89        },
90    ];
91
92    let started = Instant::now();
93    // Independent decisions, each carrying its own state: one readout per row.
94    let mut scored = Vec::with_capacity(rows.len());
95    for row in &rows {
96        scored.push(bridge.score_row(row).await?);
97    }
98    println!("scored {} rows in {:.2?}\n", scored.len(), started.elapsed());
99
100    for row in &scored {
101        let winner = row
102            .answer
103            .probabilities
104            .iter()
105            .enumerate()
106            .max_by(|a, b| a.1.partial_cmp(b.1).unwrap())
107            .map(|(index, _)| row.answer.option_ids[index].as_str())
108            .unwrap();
109        println!("{} -> {winner}", row.answer.id);
110        for (id, probability) in row.answer.option_ids.iter().zip(&row.answer.probabilities) {
111            println!("    {id:<8} {probability:.4}");
112        }
113        println!(
114            "    input_tokens={} prompt_sha256={}",
115            row.answer.input_tokens,
116            &row.prompt_sha256[..16]
117        );
118    }
119
120    // Probabilities are conditional on the supplied options and uncalibrated.
121    println!("\nThese scores are conditional option probabilities, not calibrated confidence.");
122    Ok(())
123}
score_rows/score_rows.rs

score_rows/
score_rows.rs
