Struct LocalRenderer 

pub struct LocalRenderer { /* private fields */ }

A chat template rendered in-process.

Implementations

impl LocalRenderer

pub fn new(template: String, context: Map<String, Value>) -> Result<Self>

Compile template, adding context to every render.

The context carries the variables transformers supplies besides the messages, such as bos_token and eos_token; a template that starts with {{- bos_token }} needs them.

Examples found in repository

examples/local_template.rs (line 49)

async fn main() -> Result<()> {
    let path = std::env::var("JEV_BRIDGE_CHAT_TEMPLATE").unwrap_or_else(|_| "template.jinja".to_string());
    let source = std::fs::read_to_string(&path)
        .with_context(|| format!("reading {path} failed; see the module docs for how to obtain it"))?;

    let context: Value = serde_json::from_str(
        &std::env::var("JEV_BRIDGE_CHAT_CONTEXT").unwrap_or_else(|_| "{}".to_string()),
    )
    .context("JEV_BRIDGE_CHAT_CONTEXT must be a JSON object")?;
    let context = context
        .as_object()
        .cloned()
        .ok_or_else(|| anyhow::anyhow!("JEV_BRIDGE_CHAT_CONTEXT must be a JSON object"))?;

    let renderer = LocalRenderer::new(source, context).context("compiling the chat template failed")?;

    let messages = direct_messages(
        &json!("I was charged twice and need a refund today."),
        "Which queue should handle this request?",
        &[
            RowOption {
                id: "access".to_string(),
                description: "Account access and authentication.".to_string(),
            },
            RowOption {
                id: "billing".to_string(),
                description: "Billing, payments, and refunds.".to_string(),
            },
            RowOption {
                id: "sales".to_string(),
                description: "Pricing and new contracts.".to_string(),
            },
        ],
    );

    let prompt = renderer.render(&messages, &json!({"enable_thinking": false}))?;
    println!("rendered {} bytes:\n", prompt.len());
    println!("{prompt}");
    println!("\n--- debug ---");
    println!("{:?}", prompt);
    println!(
        "\nThis is the exact text the upstream model must be scored on. Point the bridge at it \
         with --chat-template-file so the prompt contract stays byte-identical."
    );
    Ok(())
}

pub fn render<M: Serialize + ?Sized>( &self, messages: &M, chat_template_kwargs: &Value, ) -> Result<String>

Render messages, with chat_template_kwargs taking precedence over the configured context.

messages is generic over Serialize so this module stays independent of the prompt layer: any chat-shaped value renders.

Examples found in repository

examples/local_template.rs (line 70)

async fn main() -> Result<()> {
    let path = std::env::var("JEV_BRIDGE_CHAT_TEMPLATE").unwrap_or_else(|_| "template.jinja".to_string());
    let source = std::fs::read_to_string(&path)
        .with_context(|| format!("reading {path} failed; see the module docs for how to obtain it"))?;

    let context: Value = serde_json::from_str(
        &std::env::var("JEV_BRIDGE_CHAT_CONTEXT").unwrap_or_else(|_| "{}".to_string()),
    )
    .context("JEV_BRIDGE_CHAT_CONTEXT must be a JSON object")?;
    let context = context
        .as_object()
        .cloned()
        .ok_or_else(|| anyhow::anyhow!("JEV_BRIDGE_CHAT_CONTEXT must be a JSON object"))?;

    let renderer = LocalRenderer::new(source, context).context("compiling the chat template failed")?;

    let messages = direct_messages(
        &json!("I was charged twice and need a refund today."),
        "Which queue should handle this request?",
        &[
            RowOption {
                id: "access".to_string(),
                description: "Account access and authentication.".to_string(),
            },
            RowOption {
                id: "billing".to_string(),
                description: "Billing, payments, and refunds.".to_string(),
            },
            RowOption {
                id: "sales".to_string(),
                description: "Pricing and new contracts.".to_string(),
            },
        ],
    );

    let prompt = renderer.render(&messages, &json!({"enable_thinking": false}))?;
    println!("rendered {} bytes:\n", prompt.len());
    println!("{prompt}");
    println!("\n--- debug ---");
    println!("{:?}", prompt);
    println!(
        "\nThis is the exact text the upstream model must be scored on. Point the bridge at it \
         with --chat-template-file so the prompt contract stays byte-identical."
    );
    Ok(())
}