Struct Bridge

Source

pub struct Bridge { /* private fields */ }

Expand description

One configured bridge over a single upstream model.

Implementations§

Source §

impl Bridge

Source

pub async fn connect(client: Client, config: BridgeConfig) -> Result<Self>

Resolve the prompt contract and pick a transport, or fail with reasons.

Examples found in repository ?

examples/serve.rs (lines 38-53)

30async fn main() -> Result<()> {
31    let openai_base = std::env::var("JEV_BRIDGE_UPSTREAM_URL")
32        .unwrap_or_else(|_| "http://127.0.0.1:8080/v1".to_string());
33    let native_base = std::env::var("JEV_BRIDGE_NATIVE_URL")
34        .unwrap_or_else(|_| openai_base.trim_end_matches("/v1").to_string());
35    let upstream_model =
36        std::env::var("JEV_BRIDGE_UPSTREAM_MODEL").unwrap_or_else(|_| "local-model".to_string());
37
38    let bridge = Bridge::connect(
39        reqwest::Client::new(),
40        BridgeConfig {
41            openai_base,
42            native_base,
43            upstream_model,
44            upstream_key: std::env::var("JEV_BRIDGE_UPSTREAM_KEY").ok(),
45            served_model: "example-bridge".to_string(),
46            description: "example bridge over a generic inference API".to_string(),
47            release_date: "2026-09-22".to_string(),
48            chat_template_kwargs: json!({"enable_thinking": false}),
49            max_input_tokens: None,
50            local: Default::default(),
51            readout: Default::default(),
52        },
53    )
54    .await
55    .context("connecting to the upstream service failed")?;
56
57    println!("transport: {} via {}", bridge.probe().name(), bridge.probe().endpoint());
58
59    let state = Arc::new(AppState {
60        bridge,
61        api_key: std::env::var("JEV_BRIDGE_API_KEY").ok(),
62    });
63    let listener = tokio::net::TcpListener::bind(ADDRESS)
64        .await
65        .with_context(|| format!("binding {ADDRESS} failed"))?;
66    println!("listening on http://{ADDRESS}");
67    axum::serve(listener, router(state))
68        .with_graceful_shutdown(async {
69            let _ = tokio::signal::ctrl_c().await;
70        })
71        .await
72        .context("the HTTP server stopped unexpectedly")
73}

More examples

Hide additional examples

examples/score_rows.rs (lines 40-55)

30async fn main() -> Result<()> {
31    let openai_base = std::env::var("JEV_BRIDGE_UPSTREAM_URL")
32        .unwrap_or_else(|_| "http://127.0.0.1:8080/v1".to_string());
33    let native_base = std::env::var("JEV_BRIDGE_NATIVE_URL")
34        .unwrap_or_else(|_| openai_base.trim_end_matches("/v1").to_string());
35    let upstream_key = std::env::var("JEV_BRIDGE_UPSTREAM_KEY").ok();
36    let upstream_model =
37        std::env::var("JEV_BRIDGE_UPSTREAM_MODEL").unwrap_or_else(|_| "local-model".to_string());
38
39    let started = Instant::now();
40    let bridge = Bridge::connect(
41        reqwest::Client::new(),
42        BridgeConfig {
43            openai_base,
44            native_base,
45            upstream_model,
46            upstream_key,
47            served_model: "example-bridge".to_string(),
48            description: "example bridge".to_string(),
49            release_date: "2026-09-22".to_string(),
50            chat_template_kwargs: json!({"enable_thinking": false}),
51            max_input_tokens: None,
52            local: Default::default(),
53            readout: Default::default(),
54        },
55    )
56    .await
57    .context("connecting to the upstream service failed")?;
58
59    println!(
60        "connected in {:.2?}: transport={} endpoint={} renders_locally={} tokenizes_locally={}",
61        started.elapsed(),
62        bridge.probe().name(),
63        bridge.probe().endpoint(),
64        bridge.renders_locally(),
65        bridge.tokenizes_locally(),
66    );
67
68    let rows = vec![
69        Row {
70            id: "refund".to_string(),
71            state: json!("I was charged twice and need a refund today."),
72            question: "Which queue should handle this request?".to_string(),
73            options: vec![
74                option("access", "Account access and authentication."),
75                option("billing", "Billing, payments, and refunds."),
76                option("sales", "Pricing and new contracts."),
77            ],
78        },
79        Row {
80            id: "outage".to_string(),
81            state: json!("The API returns 502 for every request since this morning."),
82            question: "Which queue should handle this request?".to_string(),
83            options: vec![
84                option("access", "Account access and authentication."),
85                option("billing", "Billing, payments, and refunds."),
86                option("sales", "Pricing and new contracts."),
87            ],
88        },
89    ];
90
91    let started = Instant::now();
92    let scored = bridge.score_rows(&rows).await?;
93    println!("scored {} rows in {:.2?}\n", scored.len(), started.elapsed());
94
95    for row in &scored {
96        let winner = row
97            .answer
98            .probabilities
99            .iter()
100            .enumerate()
101            .max_by(|a, b| a.1.partial_cmp(b.1).unwrap())
102            .map(|(index, _)| row.answer.option_ids[index].as_str())
103            .unwrap();
104        println!("{} -> {winner}", row.answer.id);
105        for (id, probability) in row.answer.option_ids.iter().zip(&row.answer.probabilities) {
106            println!("    {id:<8} {probability:.4}");
107        }
108        println!(
109            "    input_tokens={} prompt_sha256={}",
110            row.answer.input_tokens,
111            &row.prompt_sha256[..16]
112        );
113    }
114
115    // Probabilities are conditional on the supplied options and uncalibrated.
116    println!("\nThese scores are conditional option probabilities, not calibrated confidence.");
117    Ok(())
118}

Source

pub fn probe(&self) -> Probe

The transport negotiated at startup.

Examples found in repository ?

examples/serve.rs (line 57)

30async fn main() -> Result<()> {
31    let openai_base = std::env::var("JEV_BRIDGE_UPSTREAM_URL")
32        .unwrap_or_else(|_| "http://127.0.0.1:8080/v1".to_string());
33    let native_base = std::env::var("JEV_BRIDGE_NATIVE_URL")
34        .unwrap_or_else(|_| openai_base.trim_end_matches("/v1").to_string());
35    let upstream_model =
36        std::env::var("JEV_BRIDGE_UPSTREAM_MODEL").unwrap_or_else(|_| "local-model".to_string());
37
38    let bridge = Bridge::connect(
39        reqwest::Client::new(),
40        BridgeConfig {
41            openai_base,
42            native_base,
43            upstream_model,
44            upstream_key: std::env::var("JEV_BRIDGE_UPSTREAM_KEY").ok(),
45            served_model: "example-bridge".to_string(),
46            description: "example bridge over a generic inference API".to_string(),
47            release_date: "2026-09-22".to_string(),
48            chat_template_kwargs: json!({"enable_thinking": false}),
49            max_input_tokens: None,
50            local: Default::default(),
51            readout: Default::default(),
52        },
53    )
54    .await
55    .context("connecting to the upstream service failed")?;
56
57    println!("transport: {} via {}", bridge.probe().name(), bridge.probe().endpoint());
58
59    let state = Arc::new(AppState {
60        bridge,
61        api_key: std::env::var("JEV_BRIDGE_API_KEY").ok(),
62    });
63    let listener = tokio::net::TcpListener::bind(ADDRESS)
64        .await
65        .with_context(|| format!("binding {ADDRESS} failed"))?;
66    println!("listening on http://{ADDRESS}");
67    axum::serve(listener, router(state))
68        .with_graceful_shutdown(async {
69            let _ = tokio::signal::ctrl_c().await;
70        })
71        .await
72        .context("the HTTP server stopped unexpectedly")
73}

More examples

Hide additional examples

examples/score_rows.rs (line 62)

30async fn main() -> Result<()> {
31    let openai_base = std::env::var("JEV_BRIDGE_UPSTREAM_URL")
32        .unwrap_or_else(|_| "http://127.0.0.1:8080/v1".to_string());
33    let native_base = std::env::var("JEV_BRIDGE_NATIVE_URL")
34        .unwrap_or_else(|_| openai_base.trim_end_matches("/v1").to_string());
35    let upstream_key = std::env::var("JEV_BRIDGE_UPSTREAM_KEY").ok();
36    let upstream_model =
37        std::env::var("JEV_BRIDGE_UPSTREAM_MODEL").unwrap_or_else(|_| "local-model".to_string());
38
39    let started = Instant::now();
40    let bridge = Bridge::connect(
41        reqwest::Client::new(),
42        BridgeConfig {
43            openai_base,
44            native_base,
45            upstream_model,
46            upstream_key,
47            served_model: "example-bridge".to_string(),
48            description: "example bridge".to_string(),
49            release_date: "2026-09-22".to_string(),
50            chat_template_kwargs: json!({"enable_thinking": false}),
51            max_input_tokens: None,
52            local: Default::default(),
53            readout: Default::default(),
54        },
55    )
56    .await
57    .context("connecting to the upstream service failed")?;
58
59    println!(
60        "connected in {:.2?}: transport={} endpoint={} renders_locally={} tokenizes_locally={}",
61        started.elapsed(),
62        bridge.probe().name(),
63        bridge.probe().endpoint(),
64        bridge.renders_locally(),
65        bridge.tokenizes_locally(),
66    );
67
68    let rows = vec![
69        Row {
70            id: "refund".to_string(),
71            state: json!("I was charged twice and need a refund today."),
72            question: "Which queue should handle this request?".to_string(),
73            options: vec![
74                option("access", "Account access and authentication."),
75                option("billing", "Billing, payments, and refunds."),
76                option("sales", "Pricing and new contracts."),
77            ],
78        },
79        Row {
80            id: "outage".to_string(),
81            state: json!("The API returns 502 for every request since this morning."),
82            question: "Which queue should handle this request?".to_string(),
83            options: vec![
84                option("access", "Account access and authentication."),
85                option("billing", "Billing, payments, and refunds."),
86                option("sales", "Pricing and new contracts."),
87            ],
88        },
89    ];
90
91    let started = Instant::now();
92    let scored = bridge.score_rows(&rows).await?;
93    println!("scored {} rows in {:.2?}\n", scored.len(), started.elapsed());
94
95    for row in &scored {
96        let winner = row
97            .answer
98            .probabilities
99            .iter()
100            .enumerate()
101            .max_by(|a, b| a.1.partial_cmp(b.1).unwrap())
102            .map(|(index, _)| row.answer.option_ids[index].as_str())
103            .unwrap();
104        println!("{} -> {winner}", row.answer.id);
105        for (id, probability) in row.answer.option_ids.iter().zip(&row.answer.probabilities) {
106            println!("    {id:<8} {probability:.4}");
107        }
108        println!(
109            "    input_tokens={} prompt_sha256={}",
110            row.answer.input_tokens,
111            &row.prompt_sha256[..16]
112        );
113    }
114
115    // Probabilities are conditional on the supplied options and uncalibrated.
116    println!("\nThese scores are conditional option probabilities, not calibrated confidence.");
117    Ok(())
118}

Source

pub fn readout(&self) -> &ReadoutStatus

What the deployment declared about the readout channel on this model.

Source

pub fn slots(&self) -> &[u32]

Resolved token id for each of the sixteen answer letters.

Source

pub fn upstream_model(&self) -> &str

The model name this bridge sends upstream.

Source

pub fn chat_template_kwargs(&self) -> Value

Template variables forwarded on every render.

Source

pub fn renders_locally(&self) -> bool

Whether the prompt is rendered in-process rather than by the runtime.

Examples found in repository ?

examples/score_rows.rs (line 64)

30async fn main() -> Result<()> {
31    let openai_base = std::env::var("JEV_BRIDGE_UPSTREAM_URL")
32        .unwrap_or_else(|_| "http://127.0.0.1:8080/v1".to_string());
33    let native_base = std::env::var("JEV_BRIDGE_NATIVE_URL")
34        .unwrap_or_else(|_| openai_base.trim_end_matches("/v1").to_string());
35    let upstream_key = std::env::var("JEV_BRIDGE_UPSTREAM_KEY").ok();
36    let upstream_model =
37        std::env::var("JEV_BRIDGE_UPSTREAM_MODEL").unwrap_or_else(|_| "local-model".to_string());
38
39    let started = Instant::now();
40    let bridge = Bridge::connect(
41        reqwest::Client::new(),
42        BridgeConfig {
43            openai_base,
44            native_base,
45            upstream_model,
46            upstream_key,
47            served_model: "example-bridge".to_string(),
48            description: "example bridge".to_string(),
49            release_date: "2026-09-22".to_string(),
50            chat_template_kwargs: json!({"enable_thinking": false}),
51            max_input_tokens: None,
52            local: Default::default(),
53            readout: Default::default(),
54        },
55    )
56    .await
57    .context("connecting to the upstream service failed")?;
58
59    println!(
60        "connected in {:.2?}: transport={} endpoint={} renders_locally={} tokenizes_locally={}",
61        started.elapsed(),
62        bridge.probe().name(),
63        bridge.probe().endpoint(),
64        bridge.renders_locally(),
65        bridge.tokenizes_locally(),
66    );
67
68    let rows = vec![
69        Row {
70            id: "refund".to_string(),
71            state: json!("I was charged twice and need a refund today."),
72            question: "Which queue should handle this request?".to_string(),
73            options: vec![
74                option("access", "Account access and authentication."),
75                option("billing", "Billing, payments, and refunds."),
76                option("sales", "Pricing and new contracts."),
77            ],
78        },
79        Row {
80            id: "outage".to_string(),
81            state: json!("The API returns 502 for every request since this morning."),
82            question: "Which queue should handle this request?".to_string(),
83            options: vec![
84                option("access", "Account access and authentication."),
85                option("billing", "Billing, payments, and refunds."),
86                option("sales", "Pricing and new contracts."),
87            ],
88        },
89    ];
90
91    let started = Instant::now();
92    let scored = bridge.score_rows(&rows).await?;
93    println!("scored {} rows in {:.2?}\n", scored.len(), started.elapsed());
94
95    for row in &scored {
96        let winner = row
97            .answer
98            .probabilities
99            .iter()
100            .enumerate()
101            .max_by(|a, b| a.1.partial_cmp(b.1).unwrap())
102            .map(|(index, _)| row.answer.option_ids[index].as_str())
103            .unwrap();
104        println!("{} -> {winner}", row.answer.id);
105        for (id, probability) in row.answer.option_ids.iter().zip(&row.answer.probabilities) {
106            println!("    {id:<8} {probability:.4}");
107        }
108        println!(
109            "    input_tokens={} prompt_sha256={}",
110            row.answer.input_tokens,
111            &row.prompt_sha256[..16]
112        );
113    }
114
115    // Probabilities are conditional on the supplied options and uncalibrated.
116    println!("\nThese scores are conditional option probabilities, not calibrated confidence.");
117    Ok(())
118}

Source

pub fn tokenizes_locally(&self) -> bool

Whether tokenization happens in-process rather than by the runtime.

Examples found in repository ?

examples/score_rows.rs (line 65)

30async fn main() -> Result<()> {
31    let openai_base = std::env::var("JEV_BRIDGE_UPSTREAM_URL")
32        .unwrap_or_else(|_| "http://127.0.0.1:8080/v1".to_string());
33    let native_base = std::env::var("JEV_BRIDGE_NATIVE_URL")
34        .unwrap_or_else(|_| openai_base.trim_end_matches("/v1").to_string());
35    let upstream_key = std::env::var("JEV_BRIDGE_UPSTREAM_KEY").ok();
36    let upstream_model =
37        std::env::var("JEV_BRIDGE_UPSTREAM_MODEL").unwrap_or_else(|_| "local-model".to_string());
38
39    let started = Instant::now();
40    let bridge = Bridge::connect(
41        reqwest::Client::new(),
42        BridgeConfig {
43            openai_base,
44            native_base,
45            upstream_model,
46            upstream_key,
47            served_model: "example-bridge".to_string(),
48            description: "example bridge".to_string(),
49            release_date: "2026-09-22".to_string(),
50            chat_template_kwargs: json!({"enable_thinking": false}),
51            max_input_tokens: None,
52            local: Default::default(),
53            readout: Default::default(),
54        },
55    )
56    .await
57    .context("connecting to the upstream service failed")?;
58
59    println!(
60        "connected in {:.2?}: transport={} endpoint={} renders_locally={} tokenizes_locally={}",
61        started.elapsed(),
62        bridge.probe().name(),
63        bridge.probe().endpoint(),
64        bridge.renders_locally(),
65        bridge.tokenizes_locally(),
66    );
67
68    let rows = vec![
69        Row {
70            id: "refund".to_string(),
71            state: json!("I was charged twice and need a refund today."),
72            question: "Which queue should handle this request?".to_string(),
73            options: vec![
74                option("access", "Account access and authentication."),
75                option("billing", "Billing, payments, and refunds."),
76                option("sales", "Pricing and new contracts."),
77            ],
78        },
79        Row {
80            id: "outage".to_string(),
81            state: json!("The API returns 502 for every request since this morning."),
82            question: "Which queue should handle this request?".to_string(),
83            options: vec![
84                option("access", "Account access and authentication."),
85                option("billing", "Billing, payments, and refunds."),
86                option("sales", "Pricing and new contracts."),
87            ],
88        },
89    ];
90
91    let started = Instant::now();
92    let scored = bridge.score_rows(&rows).await?;
93    println!("scored {} rows in {:.2?}\n", scored.len(), started.elapsed());
94
95    for row in &scored {
96        let winner = row
97            .answer
98            .probabilities
99            .iter()
100            .enumerate()
101            .max_by(|a, b| a.1.partial_cmp(b.1).unwrap())
102            .map(|(index, _)| row.answer.option_ids[index].as_str())
103            .unwrap();
104        println!("{} -> {winner}", row.answer.id);
105        for (id, probability) in row.answer.option_ids.iter().zip(&row.answer.probabilities) {
106            println!("    {id:<8} {probability:.4}");
107        }
108        println!(
109            "    input_tokens={} prompt_sha256={}",
110            row.answer.input_tokens,
111            &row.prompt_sha256[..16]
112        );
113    }
114
115    // Probabilities are conditional on the supplied options and uncalibrated.
116    println!("\nThese scores are conditional option probabilities, not calibrated confidence.");
117    Ok(())
118}

Source

pub async fn score_rows(&self, rows: &[Row]) -> Result<Vec<DetailedScore>>

Score every row of one request against the same upstream model.

Examples found in repository ?

examples/score_rows.rs (line 92)

30async fn main() -> Result<()> {
31    let openai_base = std::env::var("JEV_BRIDGE_UPSTREAM_URL")
32        .unwrap_or_else(|_| "http://127.0.0.1:8080/v1".to_string());
33    let native_base = std::env::var("JEV_BRIDGE_NATIVE_URL")
34        .unwrap_or_else(|_| openai_base.trim_end_matches("/v1").to_string());
35    let upstream_key = std::env::var("JEV_BRIDGE_UPSTREAM_KEY").ok();
36    let upstream_model =
37        std::env::var("JEV_BRIDGE_UPSTREAM_MODEL").unwrap_or_else(|_| "local-model".to_string());
38
39    let started = Instant::now();
40    let bridge = Bridge::connect(
41        reqwest::Client::new(),
42        BridgeConfig {
43            openai_base,
44            native_base,
45            upstream_model,
46            upstream_key,
47            served_model: "example-bridge".to_string(),
48            description: "example bridge".to_string(),
49            release_date: "2026-09-22".to_string(),
50            chat_template_kwargs: json!({"enable_thinking": false}),
51            max_input_tokens: None,
52            local: Default::default(),
53            readout: Default::default(),
54        },
55    )
56    .await
57    .context("connecting to the upstream service failed")?;
58
59    println!(
60        "connected in {:.2?}: transport={} endpoint={} renders_locally={} tokenizes_locally={}",
61        started.elapsed(),
62        bridge.probe().name(),
63        bridge.probe().endpoint(),
64        bridge.renders_locally(),
65        bridge.tokenizes_locally(),
66    );
67
68    let rows = vec![
69        Row {
70            id: "refund".to_string(),
71            state: json!("I was charged twice and need a refund today."),
72            question: "Which queue should handle this request?".to_string(),
73            options: vec![
74                option("access", "Account access and authentication."),
75                option("billing", "Billing, payments, and refunds."),
76                option("sales", "Pricing and new contracts."),
77            ],
78        },
79        Row {
80            id: "outage".to_string(),
81            state: json!("The API returns 502 for every request since this morning."),
82            question: "Which queue should handle this request?".to_string(),
83            options: vec![
84                option("access", "Account access and authentication."),
85                option("billing", "Billing, payments, and refunds."),
86                option("sales", "Pricing and new contracts."),
87            ],
88        },
89    ];
90
91    let started = Instant::now();
92    let scored = bridge.score_rows(&rows).await?;
93    println!("scored {} rows in {:.2?}\n", scored.len(), started.elapsed());
94
95    for row in &scored {
96        let winner = row
97            .answer
98            .probabilities
99            .iter()
100            .enumerate()
101            .max_by(|a, b| a.1.partial_cmp(b.1).unwrap())
102            .map(|(index, _)| row.answer.option_ids[index].as_str())
103            .unwrap();
104        println!("{} -> {winner}", row.answer.id);
105        for (id, probability) in row.answer.option_ids.iter().zip(&row.answer.probabilities) {
106            println!("    {id:<8} {probability:.4}");
107        }
108        println!(
109            "    input_tokens={} prompt_sha256={}",
110            row.answer.input_tokens,
111            &row.prompt_sha256[..16]
112        );
113    }
114
115    // Probabilities are conditional on the supplied options and uncalibrated.
116    println!("\nThese scores are conditional option probabilities, not calibrated confidence.");
117    Ok(())
118}

Auto Trait Implementations§

§

impl UnsafeUnpin for Bridge

Blanket Implementations§

Source §

impl<T> Any for T
where T: 'static + ?Sized,

Source §

fn type_id(&self) -> TypeId

Gets the TypeId of self. Read more

Source §

impl<T> Borrow<T> for T
where T: ?Sized,

Source §

fn borrow(&self) -> &T

Immutably borrows from an owned value. Read more

Source §

impl<T> BorrowMut<T> for T
where T: ?Sized,

Source §

fn borrow_mut(&mut self) -> &mut T

Mutably borrows from an owned value. Read more

Source §

impl<ST, DT> CastableFrom<ST, Initialized, Initialized> for DT
where ST: ?Sized, DT: ?Sized,

Source §

impl<ST, DT> CastableFrom<ST, Uninit, Uninit> for DT
where ST: ?Sized, DT: ?Sized,

Source §

impl<T> From<T> for T

Source §

fn from(t: T) -> T

Returns the argument unchanged.

Source §

impl<A, B, T> HttpServerConnExec<A, B> for T
where B: Body,

Source §

impl<T> Instrument for T

Source §

fn instrument(self, span: Span) -> Instrumented<Self> ⓘ

Instruments this type with the provided Span, returning an Instrumented wrapper. Read more

Source §

fn in_current_span(self) -> Instrumented<Self> ⓘ

Instruments this type with the current Span, returning an Instrumented wrapper. Read more

Source §

impl<T, U> Into for T
where U: From<T>,

Source §

fn into(self) -> U

Calls U::from(self).

That is, this conversion is whatever the implementation of From<T> for U chooses to do.

Source §

impl<T> IntoEither for T

Source §

fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ

Converts self into a Left variant of Either<Self, Self> if into_left is true. Converts self into a Right variant of Either<Self, Self> otherwise. Read more

Source §

fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
where F: FnOnce(&Self) -> bool,

Converts self into a Left variant of Either<Self, Self> if into_left(&self) returns true. Converts self into a Right variant of Either<Self, Self> otherwise. Read more

Source §