local_template/
local_template.rs

1//! Render the chat template in-process instead of asking the runtime.
2//!
3//! Runtimes that expose `/apply-template` (llama.cpp) render the prompt for
4//! you, which guarantees the prompt is exactly what that runtime feeds the
5//! model. Runtimes that do not — vLLM, for instance — need the template here.
6//!
7//! Take the model's own Jinja template, not a hand-written one. For a llama.cpp
8//! server it is served by `/props`:
9//!
10//! ```bash
11//! curl -s "$JEV_BRIDGE_NATIVE_URL/props?model=$JEV_BRIDGE_UPSTREAM_MODEL" \
12//!   | jq -r .chat_template > template.jinja
13//! ```
14//!
15//! Then:
16//!
17//! ```bash
18//! export JEV_BRIDGE_CHAT_TEMPLATE=template.jinja
19//! export JEV_BRIDGE_CHAT_CONTEXT='{"bos_token": "<s>", "eos_token": "</s>"}'
20//! cargo run --example local_template
21//! ```
22//!
23//! The context carries the variables transformers supplies besides the
24//! messages. A template that begins with `{{- bos_token }}` needs `bos_token`
25//! here, or the rendered prompt will silently lack its BOS token.
26
27use anyhow::{Context, Result};
28use serde_json::{json, Value};
29
30use jev_bridge::prompt::direct_messages;
31use jev_bridge::render::LocalRenderer;
32use jev_bridge::wire::RowOption;
33
34#[tokio::main]
35async fn main() -> Result<()> {
36    let path = std::env::var("JEV_BRIDGE_CHAT_TEMPLATE").unwrap_or_else(|_| "template.jinja".to_string());
37    let source = std::fs::read_to_string(&path)
38        .with_context(|| format!("reading {path} failed; see the module docs for how to obtain it"))?;
39
40    let context: Value = serde_json::from_str(
41        &std::env::var("JEV_BRIDGE_CHAT_CONTEXT").unwrap_or_else(|_| "{}".to_string()),
42    )
43    .context("JEV_BRIDGE_CHAT_CONTEXT must be a JSON object")?;
44    let context = context
45        .as_object()
46        .cloned()
47        .ok_or_else(|| anyhow::anyhow!("JEV_BRIDGE_CHAT_CONTEXT must be a JSON object"))?;
48
49    let renderer = LocalRenderer::new(source, context).context("compiling the chat template failed")?;
50
51    let messages = direct_messages(
52        &json!("I was charged twice and need a refund today."),
53        "Which queue should handle this request?",
54        &[
55            RowOption {
56                id: "access".to_string(),
57                description: "Account access and authentication.".to_string(),
58            },
59            RowOption {
60                id: "billing".to_string(),
61                description: "Billing, payments, and refunds.".to_string(),
62            },
63            RowOption {
64                id: "sales".to_string(),
65                description: "Pricing and new contracts.".to_string(),
66            },
67        ],
68    );
69
70    let prompt = renderer.render(&messages, &json!({"enable_thinking": false}))?;
71    println!("rendered {} bytes:\n", prompt.len());
72    println!("{prompt}");
73    println!("\n--- debug ---");
74    println!("{:?}", prompt);
75    println!(
76        "\nThis is the exact text the upstream model must be scored on. Point the bridge at it \
77         with --chat-template-file so the prompt contract stays byte-identical."
78    );
79    Ok(())
80}
local_template/local_template.rs

local_template/
local_template.rs
