Skip to main content

dynamo_renderer/template/
tokcfg.rs

1// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//based on: https://github.com/EricLBuehler/mistral.rs/blob/d970bb5feb863acf8e8ec90de97e18221fb959f1/mistralrs-core/src/pipeline/chat_template.rs
5
6use std::collections::HashMap;
7
8use chrono::{DateTime, Local};
9use either::Either;
10use std::borrow::Cow;
11
12use minijinja::value::{Kwargs, StringInput, from_args};
13use minijinja::{Error, ErrorKind, Output, State, Value, escape_formatter};
14use serde::{Deserialize, Serialize};
15
16use crate::python::{
17    PyFloats, PyJsonFormatter, is_python_space, python_float_repr, python_split_whitespace,
18    python_splitlines,
19};
20
21#[allow(dead_code)]
22#[derive(Debug, Deserialize)]
23pub struct AddedTokensDecoder {
24    __type: Option<String>,
25    pub content: String,
26    lstrip: bool,
27    normalized: bool,
28    rstrip: bool,
29    single_word: bool,
30    special: Option<bool>,
31}
32
33pub fn raise_exception(msg: String) -> Result<String, minijinja::Error> {
34    Err(minijinja::Error::new(ErrorKind::InvalidOperation, msg))
35}
36
37#[derive(Debug, Deserialize)]
38pub struct BeginEndUnkTok(
39    #[serde(with = "either::serde_untagged")] pub Either<String, AddedTokensDecoder>,
40);
41
42/// Support older tool use patterns where the tool use template was separate from the default/chat template.
43/// Modern patterns use a single template with a `tool_use` key, e.g.
44///
45/// ```jinja
46/// {%- if tools is not none and tool_choice is not none %}
47/// ```
48#[derive(Debug, Deserialize)]
49pub struct ChatTemplateValue(
50    #[serde(with = "either::serde_untagged")] pub Either<String, Vec<HashMap<String, String>>>,
51);
52
53/// If present, pad_token is usually a single value. Deepseek R1 and it's distill's use a map.
54#[allow(dead_code)]
55#[derive(Debug, Deserialize)]
56pub struct PadTokenValue(
57    #[serde(with = "either::serde_untagged")] pub Either<String, AddedTokensDecoder>,
58);
59
60#[allow(dead_code)]
61#[derive(Debug, Deserialize, Default)]
62/// Template for chat models including bos/eos/unk as well as the chat template.
63pub struct ChatTemplate {
64    pub bos_token: Option<BeginEndUnkTok>,
65    pub eos_token: Option<BeginEndUnkTok>,
66    pub unk_token: Option<BeginEndUnkTok>,
67
68    /// Jinja format [chat templating] for chat completion.
69    ///
70    /// [chat templating]: https://huggingface.co/docs/transformers/chat_templating
71    pub chat_template: Option<ChatTemplateValue>,
72
73    // future
74    add_bos_token: Option<bool>,
75    add_eos_token: Option<bool>,
76    added_tokens_decoder: Option<HashMap<String, AddedTokensDecoder>>,
77    additional_special_tokens: Option<Vec<String>>,
78    clean_up_tokenization_spaces: Option<bool>,
79    device_map: Option<String>,
80    legacy: Option<bool>,
81    model_max_length: Option<f64>,
82    pad_token: Option<PadTokenValue>,
83    sp_model_kwargs: Option<HashMap<String, String>>,
84    spaces_between_special_tokens: Option<bool>,
85    tokenizer_class: Option<String>,
86    truncation_size: Option<String>,
87    use_default_system_prompt: Option<bool>,
88}
89
90impl ChatTemplate {
91    pub fn eos_tok(&self) -> Option<String> {
92        match self.eos_token.as_ref()?.0 {
93            Either::Left(ref lit) => Some(lit.clone()),
94            Either::Right(ref added) => Some(added.content.clone()),
95        }
96    }
97
98    pub fn bos_tok(&self) -> Option<String> {
99        match self.bos_token.as_ref()?.0 {
100            Either::Left(ref lit) => Some(lit.clone()),
101            Either::Right(ref added) => Some(added.content.clone()),
102        }
103    }
104
105    pub fn unk_tok(&self) -> Option<String> {
106        match self.unk_token.as_ref()?.0 {
107            Either::Left(ref lit) => Some(lit.clone()),
108            Either::Right(ref added) => Some(added.content.clone()),
109        }
110    }
111}
112
113#[allow(dead_code)]
114#[derive(Debug, Deserialize)]
115pub struct GenerationConfig {
116    #[serde(with = "either::serde_untagged")]
117    bos_token_id: Either<u32, Vec<u32>>,
118    #[serde(with = "either::serde_untagged")]
119    eos_token_id: Either<u32, Vec<u32>>,
120}
121
122/// Python `str` of a non-integer float (`1e-06`, `1e+16`, `nan`); minijinja never uses
123/// an exponent (`0.000001`, `10000000000000000.0`) and prints `NaN`. `None` for every
124/// other value.
125fn python_float_str(value: &Value) -> Option<String> {
126    if !value.is_number() || value.is_integer() {
127        return None;
128    }
129    f64::try_from(value.clone()).ok().map(python_float_repr)
130}
131
132/// Output formatter for `{{ x }}`. HF renders through Python, which prints a float
133/// with `str`; see [`python_float_str`]. Other values are unchanged. `~` concatenation
134/// is a VM op that can't be overridden, and filters that stringify values themselves
135/// (`join`, `replace`, ...) use minijinja's spelling, so `'x' ~ 1e-6` and
136/// `[1e-6] | join` still render `0.000001`.
137pub fn python_formatter(out: &mut Output, state: &State, value: &Value) -> Result<(), Error> {
138    match python_float_str(value) {
139        Some(text) => out.write_str(&text).map_err(Error::from),
140        None => escape_formatter(out, state, value),
141    }
142}
143
144/// The `string` filter with Python `str` floats, matching [`python_formatter`];
145/// other values go to the builtin.
146pub fn python_string(state: &State, value: &Value) -> Result<Value, Error> {
147    match python_float_str(value) {
148        Some(text) => Ok(Value::from(text)),
149        None => minijinja::filters::string(state, value),
150    }
151}
152
153/// minijinja-contrib's pycompat string methods, with Python's whitespace where
154/// Rust's differs: `strip`, `lstrip`, `rstrip` and `split()` also treat U+001C..U+001F
155/// as whitespace, `isspace` is false for `""`, and `splitlines` ends lines at every
156/// Python line break. HF renders templates through Python, so `m.content.strip()` must
157/// strip what Python strips.
158pub fn python_methods(
159    state: &State,
160    value: &Value,
161    method: &str,
162    args: &[Value],
163) -> Result<Value, Error> {
164    if let Some(s) = value.as_str() {
165        match method {
166            "strip" | "lstrip" | "rstrip" => {
167                let (chars,): (Option<&str>,) = from_args(args)?;
168                if chars.is_none() {
169                    let trimmed = match method {
170                        "strip" => s.trim_matches(is_python_space),
171                        "lstrip" => s.trim_start_matches(is_python_space),
172                        _ => s.trim_end_matches(is_python_space),
173                    };
174                    return Ok(StringInput::new(state, value)?.preserve_safety(trimmed.to_owned()));
175                }
176            }
177            "split" => {
178                let (sep, maxsplit): (Option<&str>, Option<i64>) = from_args(args)?;
179                if sep.is_none() {
180                    let maxsplit = maxsplit.and_then(|n| usize::try_from(n).ok());
181                    return Ok(python_split_whitespace(s, maxsplit)
182                        .into_iter()
183                        .map(|part| {
184                            if value.is_safe() {
185                                Value::from_safe_string(part.to_owned())
186                            } else {
187                                Value::from(part)
188                            }
189                        })
190                        .collect());
191                }
192            }
193            "splitlines" => {
194                let (keepends,): (Option<bool>,) = from_args(args)?;
195                return Ok(python_splitlines(s, keepends.unwrap_or(false))
196                    .into_iter()
197                    .map(|part| {
198                        if value.is_safe() {
199                            Value::from_safe_string(part.to_owned())
200                        } else {
201                            Value::from(part)
202                        }
203                    })
204                    .collect());
205            }
206            "isspace" => {
207                let () = from_args(args)?;
208                return Ok(Value::from(!s.is_empty() && s.chars().all(is_python_space)));
209            }
210            _ => {}
211        }
212    }
213    minijinja_contrib::pycompat::unknown_method_callback(state, value, method, args)
214}
215
216/// Jinja2's `trim` filter, `str.strip(chars)`, with Python's whitespace by default;
217/// see [`python_methods`].
218pub fn python_trim(value: StringInput<'_>, chars: Option<Cow<'_, str>>) -> Value {
219    match chars {
220        Some(chars) => minijinja::filters::trim(value, Some(chars)),
221        None => {
222            let trimmed = value.as_str().trim_matches(is_python_space).to_owned();
223            value.preserve_safety(trimmed)
224        }
225    }
226}
227
228/// Mirrors HF transformers' `tojson` filter, not stock Jinja2's. Transformers
229/// overrides Jinja's HTML-safe `tojson` with plain
230/// `json.dumps(x, ensure_ascii=False)` in its chat-template environment, and
231/// vLLM/SGLang render through that — so chat templates (and the models trained
232/// on their output) expect Python separators and **no** HTML escaping
233/// (`'`, `<`, `>`, `&` stay literal). serde_json leaves non-ASCII unescaped by
234/// default, matching `ensure_ascii=False`.
235pub fn tojson(value: Value, kwargs: Kwargs) -> Result<Value, Error> {
236    let value = crate::python::template_json(value).map_err(|err| {
237        Error::new(ErrorKind::BadSerialization, "cannot serialize to JSON").with_source(err)
238    })?;
239    let mut buf = Vec::new();
240    let result = if let Ok(indent) = kwargs.get("indent") {
241        // Python `json.dumps(indent=n)` separators are `(",", ": ")` with the
242        // item separator followed by newline + indent — PrettyFormatter matches.
243        let repeat = b" ".repeat(indent);
244        let formatter = PyFloats(serde_json::ser::PrettyFormatter::with_indent(&repeat));
245        let mut serializer = serde_json::Serializer::with_formatter(&mut buf, formatter);
246        value.serialize(&mut serializer)
247    } else {
248        let mut serializer = serde_json::Serializer::with_formatter(&mut buf, PyJsonFormatter);
249        value.serialize(&mut serializer)
250    };
251    result.map_err(|err| {
252        Error::new(ErrorKind::BadSerialization, "cannot serialize to JSON").with_source(err)
253    })?;
254    String::from_utf8(buf)
255        .map_err(|err| {
256            Error::new(ErrorKind::BadSerialization, "cannot serialize to JSON").with_source(err)
257        })
258        .map(Value::from_safe_string)
259}
260
261/// Parse a JSON string into a structured value.
262///
263/// HuggingFace/transformers chat-template environments expose this filter, and several
264/// published templates depend on it — e.g. Step-3.7-Flash's `tool_use` block applies it to
265/// a tool call's `arguments`, which arrive as a JSON *string*, to iterate the decoded
266/// object. Without it minijinja aborts the render with `unknown filter: fromjson`, which
267/// fails every multi-turn tool-call request.
268///
269/// Values that are not strings pass through untouched, so templates that apply the filter
270/// defensively to an already-decoded value keep rendering.
271pub fn fromjson(value: Value) -> Result<Value, Error> {
272    let Some(text) = value.as_str() else {
273        return Ok(value);
274    };
275    let parsed: serde_json::Value = serde_json::from_str(text).map_err(|err| {
276        Error::new(ErrorKind::InvalidOperation, "cannot parse JSON").with_source(err)
277    })?;
278    Ok(crate::python::template_value(&parsed))
279}
280
281pub fn strftime_now(format_str: &str) -> Result<Value, Error> {
282    let local: DateTime<Local> = Local::now();
283    Ok(Value::from_safe_string(
284        local.format(format_str).to_string(),
285    ))
286}
287
288#[cfg(test)]
289mod fromjson_tests {
290    use super::*;
291
292    #[test]
293    fn parses_json_object_string() {
294        let out = fromjson(Value::from(r#"{"location":"San Francisco","n":3}"#)).unwrap();
295        assert_eq!(
296            out.get_attr("location").unwrap().as_str(),
297            Some("San Francisco")
298        );
299        assert_eq!(out.get_attr("n").unwrap().to_string(), "3");
300    }
301
302    #[test]
303    fn parses_json_array_string() {
304        let out = fromjson(Value::from(r#"[1,2,3]"#)).unwrap();
305        assert_eq!(out.len(), Some(3));
306    }
307
308    #[test]
309    fn passes_through_non_string() {
310        // Already-decoded values must survive a defensive `| fromjson`.
311        let already = crate::python::template_value(&serde_json::json!({"a": 1}));
312        let out = fromjson(already).unwrap();
313        assert_eq!(out.get_attr("a").unwrap().to_string(), "1");
314    }
315
316    #[test]
317    fn errors_on_malformed_json() {
318        assert!(fromjson(Value::from("{not json")).is_err());
319    }
320
321    /// Regression: renders the shape of Step-3.7-Flash's `tool_use` block, where a tool
322    /// call's `arguments` arrive as a JSON string. Before the filter existed this failed
323    /// with `unknown filter: fromjson`, 500-ing every multi-turn tool-call request.
324    #[test]
325    fn renders_tool_use_block_with_json_string_arguments() {
326        let mut env = minijinja::Environment::new();
327        env.add_filter("fromjson", fromjson);
328        env.add_template(
329            "tool_use",
330            "{% for tc in tool_calls %}{% set a = tc.function.arguments | fromjson %}\
331CALL {{ tc.function.name }} loc={{ a.location }} unit={{ a.unit }}{% endfor %}",
332        )
333        .unwrap();
334        let rendered = env
335            .get_template("tool_use")
336            .unwrap()
337            .render(minijinja::context! { tool_calls => serde_json::json!([{
338                "function": {
339                    "name": "get_weather",
340                    "arguments": "{\"location\":\"San Francisco\",\"unit\":\"F\"}"
341                }
342            }])})
343            .expect("tool_use template must render once `fromjson` is registered");
344        assert_eq!(rendered, "CALL get_weather loc=San Francisco unit=F");
345    }
346}