Struct Bridge

Source

pub struct Bridge { /* private fields */ }

Expand description

One configured bridge over a single upstream model.

Implementations§

Source §

impl Bridge

Source

pub async fn connect(client: Client, config: BridgeConfig) -> Result<Self>

Resolve the prompt contract and pick a transport, or fail with reasons.

Examples found in repository ?

examples/serve.rs (lines 38-52)

30async fn main() -> Result<()> {
31    let openai_base = std::env::var("JEV_BRIDGE_UPSTREAM_URL")
32        .unwrap_or_else(|_| "http://127.0.0.1:8080/v1".to_string());
33    let native_base = std::env::var("JEV_BRIDGE_NATIVE_URL")
34        .unwrap_or_else(|_| openai_base.trim_end_matches("/v1").to_string());
35    let upstream_model =
36        std::env::var("JEV_BRIDGE_UPSTREAM_MODEL").unwrap_or_else(|_| "local-model".to_string());
37
38    let bridge = Bridge::connect(
39        reqwest::Client::new(),
40        BridgeConfig {
41            openai_base,
42            native_base,
43            upstream_model,
44            upstream_key: std::env::var("JEV_BRIDGE_UPSTREAM_KEY").ok(),
45            served_model: "example-bridge".to_string(),
46            description: "example bridge over a generic inference API".to_string(),
47            release_date: "2026-09-22".to_string(),
48            chat_template_kwargs: json!({"enable_thinking": false}),
49            max_input_tokens: None,
50            local: Default::default(),
51        },
52    )
53    .await
54    .context("connecting to the upstream service failed")?;
55
56    println!("transport: {} via {}", bridge.probe().name(), bridge.probe().endpoint());
57
58    let state = Arc::new(AppState {
59        bridge,
60        api_key: std::env::var("JEV_BRIDGE_API_KEY").ok(),
61    });
62    let listener = tokio::net::TcpListener::bind(ADDRESS)
63        .await
64        .with_context(|| format!("binding {ADDRESS} failed"))?;
65    println!("listening on http://{ADDRESS}");
66    axum::serve(listener, router(state))
67        .with_graceful_shutdown(async {
68            let _ = tokio::signal::ctrl_c().await;
69        })
70        .await
71        .context("the HTTP server stopped unexpectedly")
72}

More examples

Hide additional examples

examples/score_rows.rs (lines 40-54)

30async fn main() -> Result<()> {
31    let openai_base = std::env::var("JEV_BRIDGE_UPSTREAM_URL")
32        .unwrap_or_else(|_| "http://127.0.0.1:8080/v1".to_string());
33    let native_base = std::env::var("JEV_BRIDGE_NATIVE_URL")
34        .unwrap_or_else(|_| openai_base.trim_end_matches("/v1").to_string());
35    let upstream_key = std::env::var("JEV_BRIDGE_UPSTREAM_KEY").ok();
36    let upstream_model =
37        std::env::var("JEV_BRIDGE_UPSTREAM_MODEL").unwrap_or_else(|_| "local-model".to_string());
38
39    let started = Instant::now();
40    let bridge = Bridge::connect(
41        reqwest::Client::new(),
42        BridgeConfig {
43            openai_base,
44            native_base,
45            upstream_model,
46            upstream_key,
47            served_model: "example-bridge".to_string(),
48            description: "example bridge".to_string(),
49            release_date: "2026-09-22".to_string(),
50            chat_template_kwargs: json!({"enable_thinking": false}),
51            max_input_tokens: None,
52            local: Default::default(),
53        },
54    )
55    .await
56    .context("connecting to the upstream service failed")?;
57
58    println!(
59        "connected in {:.2?}: transport={} endpoint={} renders_locally={} tokenizes_locally={}",
60        started.elapsed(),
61        bridge.probe().name(),
62        bridge.probe().endpoint(),
63        bridge.renders_locally(),
64        bridge.tokenizes_locally(),
65    );
66
67    let rows = vec![
68        Row {
69            id: "refund".to_string(),
70            state: json!("I was charged twice and need a refund today."),
71            question: "Which queue should handle this request?".to_string(),
72            options: vec![
73                option("access", "Account access and authentication."),
74                option("billing", "Billing, payments, and refunds."),
75                option("sales", "Pricing and new contracts."),
76            ],
77        },
78        Row {
79            id: "outage".to_string(),
80            state: json!("The API returns 502 for every request since this morning."),
81            question: "Which queue should handle this request?".to_string(),
82            options: vec![
83                option("access", "Account access and authentication."),
84                option("billing", "Billing, payments, and refunds."),
85                option("sales", "Pricing and new contracts."),
86            ],
87        },
88    ];
89
90    let started = Instant::now();
91    let scored = bridge.score_rows(&rows).await?;
92    println!("scored {} rows in {:.2?}\n", scored.len(), started.elapsed());
93
94    for row in &scored {
95        let winner = row
96            .answer
97            .probabilities
98            .iter()
99            .enumerate()
100            .max_by(|a, b| a.1.partial_cmp(b.1).unwrap())
101            .map(|(index, _)| row.answer.option_ids[index].as_str())
102            .unwrap();
103        println!("{} -> {winner}", row.answer.id);
104        for (id, probability) in row.answer.option_ids.iter().zip(&row.answer.probabilities) {
105            println!("    {id:<8} {probability:.4}");
106        }
107        println!(
108            "    input_tokens={} prompt_sha256={}",
109            row.answer.input_tokens,
110            &row.prompt_sha256[..16]
111        );
112    }
113
114    // Probabilities are conditional on the supplied options and uncalibrated.
115    println!("\nThese scores are conditional option probabilities, not calibrated confidence.");
116    Ok(())
117}

Source

pub fn probe(&self) -> Probe

The transport negotiated at startup.

Examples found in repository ?

examples/serve.rs (line 56)

30async fn main() -> Result<()> {
31    let openai_base = std::env::var("JEV_BRIDGE_UPSTREAM_URL")
32        .unwrap_or_else(|_| "http://127.0.0.1:8080/v1".to_string());
33    let native_base = std::env::var("JEV_BRIDGE_NATIVE_URL")
34        .unwrap_or_else(|_| openai_base.trim_end_matches("/v1").to_string());
35    let upstream_model =
36        std::env::var("JEV_BRIDGE_UPSTREAM_MODEL").unwrap_or_else(|_| "local-model".to_string());
37
38    let bridge = Bridge::connect(
39        reqwest::Client::new(),
40        BridgeConfig {
41            openai_base,
42            native_base,
43            upstream_model,
44            upstream_key: std::env::var("JEV_BRIDGE_UPSTREAM_KEY").ok(),
45            served_model: "example-bridge".to_string(),
46            description: "example bridge over a generic inference API".to_string(),
47            release_date: "2026-09-22".to_string(),
48            chat_template_kwargs: json!({"enable_thinking": false}),
49            max_input_tokens: None,
50            local: Default::default(),
51        },
52    )
53    .await
54    .context("connecting to the upstream service failed")?;
55
56    println!("transport: {} via {}", bridge.probe().name(), bridge.probe().endpoint());
57
58    let state = Arc::new(AppState {
59        bridge,
60        api_key: std::env::var("JEV_BRIDGE_API_KEY").ok(),
61    });
62    let listener = tokio::net::TcpListener::bind(ADDRESS)
63        .await
64        .with_context(|| format!("binding {ADDRESS} failed"))?;
65    println!("listening on http://{ADDRESS}");
66    axum::serve(listener, router(state))
67        .with_graceful_shutdown(async {
68            let _ = tokio::signal::ctrl_c().await;
69        })
70        .await
71        .context("the HTTP server stopped unexpectedly")
72}

More examples

Hide additional examples

examples/score_rows.rs (line 61)

30async fn main() -> Result<()> {
31    let openai_base = std::env::var("JEV_BRIDGE_UPSTREAM_URL")
32        .unwrap_or_else(|_| "http://127.0.0.1:8080/v1".to_string());
33    let native_base = std::env::var("JEV_BRIDGE_NATIVE_URL")
34        .unwrap_or_else(|_| openai_base.trim_end_matches("/v1").to_string());
35    let upstream_key = std::env::var("JEV_BRIDGE_UPSTREAM_KEY").ok();
36    let upstream_model =
37        std::env::var("JEV_BRIDGE_UPSTREAM_MODEL").unwrap_or_else(|_| "local-model".to_string());
38
39    let started = Instant::now();
40    let bridge = Bridge::connect(
41        reqwest::Client::new(),
42        BridgeConfig {
43            openai_base,
44            native_base,
45            upstream_model,
46            upstream_key,
47            served_model: "example-bridge".to_string(),
48            description: "example bridge".to_string(),
49            release_date: "2026-09-22".to_string(),
50            chat_template_kwargs: json!({"enable_thinking": false}),
51            max_input_tokens: None,
52            local: Default::default(),
53        },
54    )
55    .await
56    .context("connecting to the upstream service failed")?;
57
58    println!(
59        "connected in {:.2?}: transport={} endpoint={} renders_locally={} tokenizes_locally={}",
60        started.elapsed(),
61        bridge.probe().name(),
62        bridge.probe().endpoint(),
63        bridge.renders_locally(),
64        bridge.tokenizes_locally(),
65    );
66
67    let rows = vec![
68        Row {
69            id: "refund".to_string(),
70            state: json!("I was charged twice and need a refund today."),
71            question: "Which queue should handle this request?".to_string(),
72            options: vec![
73                option("access", "Account access and authentication."),
74                option("billing", "Billing, payments, and refunds."),
75                option("sales", "Pricing and new contracts."),
76            ],
77        },
78        Row {
79            id: "outage".to_string(),
80            state: json!("The API returns 502 for every request since this morning."),
81            question: "Which queue should handle this request?".to_string(),
82            options: vec![
83                option("access", "Account access and authentication."),
84                option("billing", "Billing, payments, and refunds."),
85                option("sales", "Pricing and new contracts."),
86            ],
87        },
88    ];
89
90    let started = Instant::now();
91    let scored = bridge.score_rows(&rows).await?;
92    println!("scored {} rows in {:.2?}\n", scored.len(), started.elapsed());
93
94    for row in &scored {
95        let winner = row
96            .answer
97            .probabilities
98            .iter()
99            .enumerate()
100            .max_by(|a, b| a.1.partial_cmp(b.1).unwrap())
101            .map(|(index, _)| row.answer.option_ids[index].as_str())
102            .unwrap();
103        println!("{} -> {winner}", row.answer.id);
104        for (id, probability) in row.answer.option_ids.iter().zip(&row.answer.probabilities) {
105            println!("    {id:<8} {probability:.4}");
106        }
107        println!(
108            "    input_tokens={} prompt_sha256={}",
109            row.answer.input_tokens,
110            &row.prompt_sha256[..16]
111        );
112    }
113
114    // Probabilities are conditional on the supplied options and uncalibrated.
115    println!("\nThese scores are conditional option probabilities, not calibrated confidence.");
116    Ok(())
117}

Source

pub fn slots(&self) -> &[u32]

Resolved token id for each of the sixteen answer letters.

Source

pub fn upstream_model(&self) -> &str

The model name this bridge sends upstream.

Source

pub fn chat_template_kwargs(&self) -> Value

Template variables forwarded on every render.

Source

pub fn renders_locally(&self) -> bool

Whether the prompt is rendered in-process rather than by the runtime.

Examples found in repository ?

examples/score_rows.rs (line 63)

30async fn main() -> Result<()> {
31    let openai_base = std::env::var("JEV_BRIDGE_UPSTREAM_URL")
32        .unwrap_or_else(|_| "http://127.0.0.1:8080/v1".to_string());
33    let native_base = std::env::var("JEV_BRIDGE_NATIVE_URL")
34        .unwrap_or_else(|_| openai_base.trim_end_matches("/v1").to_string());
35    let upstream_key = std::env::var("JEV_BRIDGE_UPSTREAM_KEY").ok();
36    let upstream_model =
37        std::env::var("JEV_BRIDGE_UPSTREAM_MODEL").unwrap_or_else(|_| "local-model".to_string());
38
39    let started = Instant::now();
40    let bridge = Bridge::connect(
41        reqwest::Client::new(),
42        BridgeConfig {
43            openai_base,
44            native_base,
45            upstream_model,
46            upstream_key,
47            served_model: "example-bridge".to_string(),
48            description: "example bridge".to_string(),
49            release_date: "2026-09-22".to_string(),
50            chat_template_kwargs: json!({"enable_thinking": false}),
51            max_input_tokens: None,
52            local: Default::default(),
53        },
54    )
55    .await
56    .context("connecting to the upstream service failed")?;
57
58    println!(
59        "connected in {:.2?}: transport={} endpoint={} renders_locally={} tokenizes_locally={}",
60        started.elapsed(),
61        bridge.probe().name(),
62        bridge.probe().endpoint(),
63        bridge.renders_locally(),
64        bridge.tokenizes_locally(),
65    );
66
67    let rows = vec![
68        Row {
69            id: "refund".to_string(),
70            state: json!("I was charged twice and need a refund today."),
71            question: "Which queue should handle this request?".to_string(),
72            options: vec![
73                option("access", "Account access and authentication."),
74                option("billing", "Billing, payments, and refunds."),
75                option("sales", "Pricing and new contracts."),
76            ],
77        },
78        Row {
79            id: "outage".to_string(),
80            state: json!("The API returns 502 for every request since this morning."),
81            question: "Which queue should handle this request?".to_string(),
82            options: vec![
83                option("access", "Account access and authentication."),
84                option("billing", "Billing, payments, and refunds."),
85                option("sales", "Pricing and new contracts."),
86            ],
87        },
88    ];
89
90    let started = Instant::now();
91    let scored = bridge.score_rows(&rows).await?;
92    println!("scored {} rows in {:.2?}\n", scored.len(), started.elapsed());
93
94    for row in &scored {
95        let winner = row
96            .answer
97            .probabilities
98            .iter()
99            .enumerate()
100            .max_by(|a, b| a.1.partial_cmp(b.1).unwrap())
101            .map(|(index, _)| row.answer.option_ids[index].as_str())
102            .unwrap();
103        println!("{} -> {winner}", row.answer.id);
104        for (id, probability) in row.answer.option_ids.iter().zip(&row.answer.probabilities) {
105            println!("    {id:<8} {probability:.4}");
106        }
107        println!(
108            "    input_tokens={} prompt_sha256={}",
109            row.answer.input_tokens,
110            &row.prompt_sha256[..16]
111        );
112    }
113
114    // Probabilities are conditional on the supplied options and uncalibrated.
115    println!("\nThese scores are conditional option probabilities, not calibrated confidence.");
116    Ok(())
117}

Source

pub fn tokenizes_locally(&self) -> bool

Whether tokenization happens in-process rather than by the runtime.

Examples found in repository ?

examples/score_rows.rs (line 64)

30async fn main() -> Result<()> {
31    let openai_base = std::env::var("JEV_BRIDGE_UPSTREAM_URL")
32        .unwrap_or_else(|_| "http://127.0.0.1:8080/v1".to_string());
33    let native_base = std::env::var("JEV_BRIDGE_NATIVE_URL")
34        .unwrap_or_else(|_| openai_base.trim_end_matches("/v1").to_string());
35    let upstream_key = std::env::var("JEV_BRIDGE_UPSTREAM_KEY").ok();
36    let upstream_model =
37        std::env::var("JEV_BRIDGE_UPSTREAM_MODEL").unwrap_or_else(|_| "local-model".to_string());
38
39    let started = Instant::now();
40    let bridge = Bridge::connect(
41        reqwest::Client::new(),
42        BridgeConfig {
43            openai_base,
44            native_base,
45            upstream_model,
46            upstream_key,
47            served_model: "example-bridge".to_string(),
48            description: "example bridge".to_string(),
49            release_date: "2026-09-22".to_string(),
50            chat_template_kwargs: json!({"enable_thinking": false}),
51            max_input_tokens: None,
52            local: Default::default(),
53        },
54    )
55    .await
56    .context("connecting to the upstream service failed")?;
57
58    println!(
59        "connected in {:.2?}: transport={} endpoint={} renders_locally={} tokenizes_locally={}",
60        started.elapsed(),
61        bridge.probe().name(),
62        bridge.probe().endpoint(),
63        bridge.renders_locally(),
64        bridge.tokenizes_locally(),
65    );
66
67    let rows = vec![
68        Row {
69            id: "refund".to_string(),
70            state: json!("I was charged twice and need a refund today."),
71            question: "Which queue should handle this request?".to_string(),
72            options: vec![
73                option("access", "Account access and authentication."),
74                option("billing", "Billing, payments, and refunds."),
75                option("sales", "Pricing and new contracts."),
76            ],
77        },
78        Row {
79            id: "outage".to_string(),
80            state: json!("The API returns 502 for every request since this morning."),
81            question: "Which queue should handle this request?".to_string(),
82            options: vec![
83                option("access", "Account access and authentication."),
84                option("billing", "Billing, payments, and refunds."),
85                option("sales", "Pricing and new contracts."),
86            ],
87        },
88    ];
89
90    let started = Instant::now();
91    let scored = bridge.score_rows(&rows).await?;
92    println!("scored {} rows in {:.2?}\n", scored.len(), started.elapsed());
93
94    for row in &scored {
95        let winner = row
96            .answer
97            .probabilities
98            .iter()
99            .enumerate()
100            .max_by(|a, b| a.1.partial_cmp(b.1).unwrap())
101            .map(|(index, _)| row.answer.option_ids[index].as_str())
102            .unwrap();
103        println!("{} -> {winner}", row.answer.id);
104        for (id, probability) in row.answer.option_ids.iter().zip(&row.answer.probabilities) {
105            println!("    {id:<8} {probability:.4}");
106        }
107        println!(
108            "    input_tokens={} prompt_sha256={}",
109            row.answer.input_tokens,
110            &row.prompt_sha256[..16]
111        );
112    }
113
114    // Probabilities are conditional on the supplied options and uncalibrated.
115    println!("\nThese scores are conditional option probabilities, not calibrated confidence.");
116    Ok(())
117}

Source

pub async fn score_rows(&self, rows: &[Row]) -> Result<Vec<DetailedScore>>

Score every row of one request against the same upstream model.

Examples found in repository ?

examples/score_rows.rs (line 91)

30async fn main() -> Result<()> {
31    let openai_base = std::env::var("JEV_BRIDGE_UPSTREAM_URL")
32        .unwrap_or_else(|_| "http://127.0.0.1:8080/v1".to_string());
33    let native_base = std::env::var("JEV_BRIDGE_NATIVE_URL")
34        .unwrap_or_else(|_| openai_base.trim_end_matches("/v1").to_string());
35    let upstream_key = std::env::var("JEV_BRIDGE_UPSTREAM_KEY").ok();
36    let upstream_model =
37        std::env::var("JEV_BRIDGE_UPSTREAM_MODEL").unwrap_or_else(|_| "local-model".to_string());
38
39    let started = Instant::now();
40    let bridge = Bridge::connect(
41        reqwest::Client::new(),
42        BridgeConfig {
43            openai_base,
44            native_base,
45            upstream_model,
46            upstream_key,
47            served_model: "example-bridge".to_string(),
48            description: "example bridge".to_string(),
49            release_date: "2026-09-22".to_string(),
50            chat_template_kwargs: json!({"enable_thinking": false}),
51            max_input_tokens: None,
52            local: Default::default(),
53        },
54    )
55    .await
56    .context("connecting to the upstream service failed")?;
57
58    println!(
59        "connected in {:.2?}: transport={} endpoint={} renders_locally={} tokenizes_locally={}",
60        started.elapsed(),
61        bridge.probe().name(),
62        bridge.probe().endpoint(),
63        bridge.renders_locally(),
64        bridge.tokenizes_locally(),
65    );
66
67    let rows = vec![
68        Row {
69            id: "refund".to_string(),
70            state: json!("I was charged twice and need a refund today."),
71            question: "Which queue should handle this request?".to_string(),
72            options: vec![
73                option("access", "Account access and authentication."),
74                option("billing", "Billing, payments, and refunds."),
75                option("sales", "Pricing and new contracts."),
76            ],
77        },
78        Row {
79            id: "outage".to_string(),
80            state: json!("The API returns 502 for every request since this morning."),
81            question: "Which queue should handle this request?".to_string(),
82            options: vec![
83                option("access", "Account access and authentication."),
84                option("billing", "Billing, payments, and refunds."),
85                option("sales", "Pricing and new contracts."),
86            ],
87        },
88    ];
89
90    let started = Instant::now();
91    let scored = bridge.score_rows(&rows).await?;
92    println!("scored {} rows in {:.2?}\n", scored.len(), started.elapsed());
93
94    for row in &scored {
95        let winner = row
96            .answer
97            .probabilities
98            .iter()
99            .enumerate()
100            .max_by(|a, b| a.1.partial_cmp(b.1).unwrap())
101            .map(|(index, _)| row.answer.option_ids[index].as_str())
102            .unwrap();
103        println!("{} -> {winner}", row.answer.id);
104        for (id, probability) in row.answer.option_ids.iter().zip(&row.answer.probabilities) {
105            println!("    {id:<8} {probability:.4}");
106        }
107        println!(
108            "    input_tokens={} prompt_sha256={}",
109            row.answer.input_tokens,
110            &row.prompt_sha256[..16]
111        );
112    }
113
114    // Probabilities are conditional on the supplied options and uncalibrated.
115    println!("\nThese scores are conditional option probabilities, not calibrated confidence.");
116    Ok(())
117}

Auto Trait Implementations§

§

impl UnsafeUnpin for Bridge

Blanket Implementations§

Source §

impl<T> Any for T
where T: 'static + ?Sized,

Source §

fn type_id(&self) -> TypeId

Gets the TypeId of self. Read more

Source §

impl<T> Borrow<T> for T
where T: ?Sized,

Source §

fn borrow(&self) -> &T

Immutably borrows from an owned value. Read more

Source §

impl<T> BorrowMut<T> for T
where T: ?Sized,

Source §

fn borrow_mut(&mut self) -> &mut T

Mutably borrows from an owned value. Read more

Source §

impl<ST, DT> CastableFrom<ST, Initialized, Initialized> for DT
where ST: ?Sized, DT: ?Sized,

Source §

impl<ST, DT> CastableFrom<ST, Uninit, Uninit> for DT
where ST: ?Sized, DT: ?Sized,

Source §

impl<T> From<T> for T

Source §

fn from(t: T) -> T

Returns the argument unchanged.

Source §

impl<A, B, T> HttpServerConnExec<A, B> for T
where B: Body,

Source §

impl<T> Instrument for T

Source §

fn instrument(self, span: Span) -> Instrumented<Self> ⓘ

Instruments this type with the provided Span, returning an Instrumented wrapper. Read more

Source §

fn in_current_span(self) -> Instrumented<Self> ⓘ

Instruments this type with the current Span, returning an Instrumented wrapper. Read more

Source §

impl<T, U> Into for T
where U: From<T>,

Source §

fn into(self) -> U

Calls U::from(self).

That is, this conversion is whatever the implementation of From<T> for U chooses to do.

Source §

impl<T> IntoEither for T

Source §

fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ

Converts self into a Left variant of Either<Self, Self> if into_left is true. Converts self into a Right variant of Either<Self, Self> otherwise. Read more

Source §

fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
where F: FnOnce(&Self) -> bool,

Converts self into a Left variant of Either<Self, Self> if into_left(&self) returns true. Converts self into a Right variant of Either<Self, Self> otherwise. Read more

Source §