Enum Probe 

pub enum Probe {
    LogprobTokenIds,
    AllowedTokenIdsTopK,
    CompletionsTopK,
    LlamaCppNProbs,
}

How the target service exposes option-token log probabilities.

Variants

LogprobTokenIds

vLLM: restrict the distribution to the option tokens and ask for their exact token ids. This is the same contract fastjev’s own vLLM backend uses through SamplingParams.

AllowedTokenIdsTopK

vLLM without logprob_token_ids: restrict the distribution and read the top-k, which must then contain every option token.

CompletionsTopK

OpenAI-compatible completions: read the top-k of the unrestricted next-token distribution.

LlamaCppNProbs

llama.cpp native /completion with n_probs.

Implementations

impl Probe

pub fn name(self) -> &'static str

Stable identifier reported by GET /health.

Examples found in repository

examples/serve.rs (line 60)

async fn main() -> Result<()> {
    let openai_base = std::env::var("JEV_BRIDGE_UPSTREAM_URL")
        .unwrap_or_else(|_| "http://127.0.0.1:8080/v1".to_string());
    let native_base = std::env::var("JEV_BRIDGE_NATIVE_URL")
        .unwrap_or_else(|_| openai_base.trim_end_matches("/v1").to_string());
    let upstream_model =
        std::env::var("JEV_BRIDGE_UPSTREAM_MODEL").unwrap_or_else(|_| "local-model".to_string());

    let bridge = Bridge::connect(
        reqwest::Client::new(),
        BridgeConfig {
            openai_base,
            native_base,
            upstream_model,
            upstream_key: std::env::var("JEV_BRIDGE_UPSTREAM_KEY").ok(),
            served_model: "example-bridge".to_string(),
            description: "example bridge over a generic inference API".to_string(),
            release_date: "2026-09-22".to_string(),
            chat_template_kwargs: json!({"enable_thinking": false}),
            max_input_tokens: None,
            local: Default::default(),
            accept_any_model: false,
            scoring: Default::default(),
            confidence: Default::default(),
            readout: Default::default(),
        },
    )
    .await
    .context("connecting to the upstream service failed")?;

    println!("transport: {} via {}", bridge.probe().name(), bridge.probe().endpoint());

    let state = Arc::new(AppState {
        bridge,
        api_key: std::env::var("JEV_BRIDGE_API_KEY").ok(),
        readout_check: None,
    });
    let listener = tokio::net::TcpListener::bind(ADDRESS)
        .await
        .with_context(|| format!("binding {ADDRESS} failed"))?;
    println!("listening on http://{ADDRESS}");
    axum::serve(listener, router(state))
        .with_graceful_shutdown(async {
            let _ = tokio::signal::ctrl_c().await;
        })
        .await
        .context("the HTTP server stopped unexpectedly")
}

examples/score_rows.rs (line 65)

async fn main() -> Result<()> {
    let openai_base = std::env::var("JEV_BRIDGE_UPSTREAM_URL")
        .unwrap_or_else(|_| "http://127.0.0.1:8080/v1".to_string());
    let native_base = std::env::var("JEV_BRIDGE_NATIVE_URL")
        .unwrap_or_else(|_| openai_base.trim_end_matches("/v1").to_string());
    let upstream_key = std::env::var("JEV_BRIDGE_UPSTREAM_KEY").ok();
    let upstream_model =
        std::env::var("JEV_BRIDGE_UPSTREAM_MODEL").unwrap_or_else(|_| "local-model".to_string());

    let started = Instant::now();
    let bridge = Bridge::connect(
        reqwest::Client::new(),
        BridgeConfig {
            openai_base,
            native_base,
            upstream_model,
            upstream_key,
            served_model: "example-bridge".to_string(),
            description: "example bridge".to_string(),
            release_date: "2026-09-22".to_string(),
            chat_template_kwargs: json!({"enable_thinking": false}),
            max_input_tokens: None,
            local: Default::default(),
            accept_any_model: false,
            scoring: Default::default(),
            confidence: Default::default(),
            readout: Default::default(),
        },
    )
    .await
    .context("connecting to the upstream service failed")?;

    println!(
        "connected in {:.2?}: transport={} endpoint={} renders_locally={} tokenizes_locally={}",
        started.elapsed(),
        bridge.probe().name(),
        bridge.probe().endpoint(),
        bridge.renders_locally(),
        bridge.tokenizes_locally(),
    );

    let rows = vec![
        Row {
            id: "refund".to_string(),
            state: json!("I was charged twice and need a refund today."),
            question: "Which queue should handle this request?".to_string(),
            options: vec![
                option("access", "Account access and authentication."),
                option("billing", "Billing, payments, and refunds."),
                option("sales", "Pricing and new contracts."),
            ],
        },
        Row {
            id: "outage".to_string(),
            state: json!("The API returns 502 for every request since this morning."),
            question: "Which queue should handle this request?".to_string(),
            options: vec![
                option("access", "Account access and authentication."),
                option("billing", "Billing, payments, and refunds."),
                option("sales", "Pricing and new contracts."),
            ],
        },
    ];

    let started = Instant::now();
    // Independent decisions, each carrying its own state: one readout per row.
    let mut scored = Vec::with_capacity(rows.len());
    for row in &rows {
        scored.push(bridge.score_row(row).await?);
    }
    println!("scored {} rows in {:.2?}\n", scored.len(), started.elapsed());

    for row in &scored {
        let winner = row
            .answer
            .probabilities
            .iter()
            .enumerate()
            .max_by(|a, b| a.1.partial_cmp(b.1).unwrap())
            .map(|(index, _)| row.answer.option_ids[index].as_str())
            .unwrap();
        println!("{} -> {winner}", row.answer.id);
        for (id, probability) in row.answer.option_ids.iter().zip(&row.answer.probabilities) {
            println!("    {id:<8} {probability:.4}");
        }
        println!(
            "    input_tokens={} prompt_sha256={}",
            row.answer.input_tokens,
            &row.prompt_sha256[..16]
        );
    }

    // Probabilities are conditional on the supplied options and uncalibrated.
    println!("\nThese scores are conditional option probabilities, not calibrated confidence.");
    Ok(())
}

pub fn note(self) -> &'static str

One line describing what this transport relies on.

pub fn restricts_distribution(self) -> bool

Whether the request is expected to restrict the distribution to the answer tokens. A probe that claims restriction but returns foreign candidates is rejected rather than silently trusted.

pub fn supports_batch(self) -> bool

Whether this transport can read several prompts in one request.

The OpenAI-shaped transports accept an array prompt, which a prefix-caching server prefills together. llama.cpp’s native /completion answers an array with a bare JSON array of response objects instead of one object, so it stays one prompt per request.

pub fn endpoint(self) -> &'static str

The path this transport posts to.

Examples found in repository

examples/serve.rs (line 60)

async fn main() -> Result<()> {
    let openai_base = std::env::var("JEV_BRIDGE_UPSTREAM_URL")
        .unwrap_or_else(|_| "http://127.0.0.1:8080/v1".to_string());
    let native_base = std::env::var("JEV_BRIDGE_NATIVE_URL")
        .unwrap_or_else(|_| openai_base.trim_end_matches("/v1").to_string());
    let upstream_model =
        std::env::var("JEV_BRIDGE_UPSTREAM_MODEL").unwrap_or_else(|_| "local-model".to_string());

    let bridge = Bridge::connect(
        reqwest::Client::new(),
        BridgeConfig {
            openai_base,
            native_base,
            upstream_model,
            upstream_key: std::env::var("JEV_BRIDGE_UPSTREAM_KEY").ok(),
            served_model: "example-bridge".to_string(),
            description: "example bridge over a generic inference API".to_string(),
            release_date: "2026-09-22".to_string(),
            chat_template_kwargs: json!({"enable_thinking": false}),
            max_input_tokens: None,
            local: Default::default(),
            accept_any_model: false,
            scoring: Default::default(),
            confidence: Default::default(),
            readout: Default::default(),
        },
    )
    .await
    .context("connecting to the upstream service failed")?;

    println!("transport: {} via {}", bridge.probe().name(), bridge.probe().endpoint());

    let state = Arc::new(AppState {
        bridge,
        api_key: std::env::var("JEV_BRIDGE_API_KEY").ok(),
        readout_check: None,
    });
    let listener = tokio::net::TcpListener::bind(ADDRESS)
        .await
        .with_context(|| format!("binding {ADDRESS} failed"))?;
    println!("listening on http://{ADDRESS}");
    axum::serve(listener, router(state))
        .with_graceful_shutdown(async {
            let _ = tokio::signal::ctrl_c().await;
        })
        .await
        .context("the HTTP server stopped unexpectedly")
}

examples/score_rows.rs (line 66)

async fn main() -> Result<()> {
    let openai_base = std::env::var("JEV_BRIDGE_UPSTREAM_URL")
        .unwrap_or_else(|_| "http://127.0.0.1:8080/v1".to_string());
    let native_base = std::env::var("JEV_BRIDGE_NATIVE_URL")
        .unwrap_or_else(|_| openai_base.trim_end_matches("/v1").to_string());
    let upstream_key = std::env::var("JEV_BRIDGE_UPSTREAM_KEY").ok();
    let upstream_model =
        std::env::var("JEV_BRIDGE_UPSTREAM_MODEL").unwrap_or_else(|_| "local-model".to_string());

    let started = Instant::now();
    let bridge = Bridge::connect(
        reqwest::Client::new(),
        BridgeConfig {
            openai_base,
            native_base,
            upstream_model,
            upstream_key,
            served_model: "example-bridge".to_string(),
            description: "example bridge".to_string(),
            release_date: "2026-09-22".to_string(),
            chat_template_kwargs: json!({"enable_thinking": false}),
            max_input_tokens: None,
            local: Default::default(),
            accept_any_model: false,
            scoring: Default::default(),
            confidence: Default::default(),
            readout: Default::default(),
        },
    )
    .await
    .context("connecting to the upstream service failed")?;

    println!(
        "connected in {:.2?}: transport={} endpoint={} renders_locally={} tokenizes_locally={}",
        started.elapsed(),
        bridge.probe().name(),
        bridge.probe().endpoint(),
        bridge.renders_locally(),
        bridge.tokenizes_locally(),
    );

    let rows = vec![
        Row {
            id: "refund".to_string(),
            state: json!("I was charged twice and need a refund today."),
            question: "Which queue should handle this request?".to_string(),
            options: vec![
                option("access", "Account access and authentication."),
                option("billing", "Billing, payments, and refunds."),
                option("sales", "Pricing and new contracts."),
            ],
        },
        Row {
            id: "outage".to_string(),
            state: json!("The API returns 502 for every request since this morning."),
            question: "Which queue should handle this request?".to_string(),
            options: vec![
                option("access", "Account access and authentication."),
                option("billing", "Billing, payments, and refunds."),
                option("sales", "Pricing and new contracts."),
            ],
        },
    ];

    let started = Instant::now();
    // Independent decisions, each carrying its own state: one readout per row.
    let mut scored = Vec::with_capacity(rows.len());
    for row in &rows {
        scored.push(bridge.score_row(row).await?);
    }
    println!("scored {} rows in {:.2?}\n", scored.len(), started.elapsed());

    for row in &scored {
        let winner = row
            .answer
            .probabilities
            .iter()
            .enumerate()
            .max_by(|a, b| a.1.partial_cmp(b.1).unwrap())
            .map(|(index, _)| row.answer.option_ids[index].as_str())
            .unwrap();
        println!("{} -> {winner}", row.answer.id);
        for (id, probability) in row.answer.option_ids.iter().zip(&row.answer.probabilities) {
            println!("    {id:<8} {probability:.4}");
        }
        println!(
            "    input_tokens={} prompt_sha256={}",
            row.answer.input_tokens,
            &row.prompt_sha256[..16]
        );
    }

    // Probabilities are conditional on the supplied options and uncalibrated.
    println!("\nThese scores are conditional option probabilities, not calibrated confidence.");
    Ok(())
}

Trait Implementations

impl Clone for Probe

fn clone(&self) -> Self

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Copy for Probe

impl Debug for Probe

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Eq for Probe

impl PartialEq for Probe

fn eq(&self, other: &Self) -> bool

Equality operator ==.

fn ne(&self, other: &Rhs) -> bool

Inequality operator !=.

impl StructuralPartialEq for Probe