Skip to main content

dynamo_renderer/template/
tokcfg.rs

1// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//based on: https://github.com/EricLBuehler/mistral.rs/blob/d970bb5feb863acf8e8ec90de97e18221fb959f1/mistralrs-core/src/pipeline/chat_template.rs
5
6use std::collections::HashMap;
7
8use chrono::{DateTime, Local};
9use either::Either;
10use std::borrow::Cow;
11
12use minijinja::value::{Kwargs, ObjectRepr, StringInput, ValueKind, from_args};
13use minijinja::{Error, ErrorKind, Output, State, Value, escape_formatter};
14use serde::ser::Error as _;
15use serde::{Deserialize, Serialize};
16
17use crate::python::{
18    PyJsonFormatter, is_python_space, python_float_repr, python_split_whitespace, python_splitlines,
19};
20
21#[allow(dead_code)]
22#[derive(Debug, Deserialize)]
23pub struct AddedTokensDecoder {
24    __type: Option<String>,
25    pub content: String,
26    lstrip: bool,
27    normalized: bool,
28    rstrip: bool,
29    single_word: bool,
30    special: Option<bool>,
31}
32
33pub fn raise_exception(msg: String) -> Result<String, minijinja::Error> {
34    Err(minijinja::Error::new(ErrorKind::InvalidOperation, msg))
35}
36
37#[derive(Debug, Deserialize)]
38pub struct BeginEndUnkTok(
39    #[serde(with = "either::serde_untagged")] pub Either<String, AddedTokensDecoder>,
40);
41
42/// Support older tool use patterns where the tool use template was separate from the default/chat template.
43/// Modern patterns use a single template with a `tool_use` key, e.g.
44///
45/// ```jinja
46/// {%- if tools is not none and tool_choice is not none %}
47/// ```
48#[derive(Debug, Deserialize)]
49pub struct ChatTemplateValue(
50    #[serde(with = "either::serde_untagged")] pub Either<String, Vec<HashMap<String, String>>>,
51);
52
53/// If present, pad_token is usually a single value. Deepseek R1 and it's distill's use a map.
54#[allow(dead_code)]
55#[derive(Debug, Deserialize)]
56pub struct PadTokenValue(
57    #[serde(with = "either::serde_untagged")] pub Either<String, AddedTokensDecoder>,
58);
59
60#[allow(dead_code)]
61#[derive(Debug, Deserialize, Default)]
62/// Template for chat models including bos/eos/unk as well as the chat template.
63pub struct ChatTemplate {
64    pub bos_token: Option<BeginEndUnkTok>,
65    pub eos_token: Option<BeginEndUnkTok>,
66    pub unk_token: Option<BeginEndUnkTok>,
67
68    /// Jinja format [chat templating] for chat completion.
69    ///
70    /// [chat templating]: https://huggingface.co/docs/transformers/chat_templating
71    pub chat_template: Option<ChatTemplateValue>,
72
73    // future
74    add_bos_token: Option<bool>,
75    add_eos_token: Option<bool>,
76    added_tokens_decoder: Option<HashMap<String, AddedTokensDecoder>>,
77    additional_special_tokens: Option<Vec<String>>,
78    clean_up_tokenization_spaces: Option<bool>,
79    device_map: Option<String>,
80    legacy: Option<bool>,
81    model_max_length: Option<f64>,
82    pad_token: Option<PadTokenValue>,
83    sp_model_kwargs: Option<HashMap<String, String>>,
84    spaces_between_special_tokens: Option<bool>,
85    tokenizer_class: Option<String>,
86    truncation_size: Option<String>,
87    use_default_system_prompt: Option<bool>,
88
89    /// The `added_tokens` of the model's `tokenizer.json`; see
90    /// [`ChatTemplate::with_tokenizer_json`].
91    #[serde(skip)]
92    tokenizer_added_tokens: Vec<AddedTokensDecoder>,
93
94    /// See [`ChatTemplate::with_request_control_tokens_as_text`]; `None` is the
95    /// default, on.
96    #[serde(skip)]
97    pub(crate) request_control_tokens_as_text: Option<bool>,
98}
99
100impl ChatTemplate {
101    pub fn eos_tok(&self) -> Option<String> {
102        match self.eos_token.as_ref()?.0 {
103            Either::Left(ref lit) => Some(lit.clone()),
104            Either::Right(ref added) => Some(added.content.clone()),
105        }
106    }
107
108    pub fn bos_tok(&self) -> Option<String> {
109        match self.bos_token.as_ref()?.0 {
110            Either::Left(ref lit) => Some(lit.clone()),
111            Either::Right(ref added) => Some(added.content.clone()),
112        }
113    }
114
115    pub fn unk_tok(&self) -> Option<String> {
116        match self.unk_token.as_ref()?.0 {
117            Either::Left(ref lit) => Some(lit.clone()),
118            Either::Right(ref added) => Some(added.content.clone()),
119        }
120    }
121
122    /// Add the `added_tokens` of the model's `tokenizer.json` to
123    /// [`ChatTemplate::added_token_spellings`]. Some repos declare no
124    /// `added_tokens_decoder` in `tokenizer_config.json` (DeepSeek V3/R1,
125    /// GLM-5, Gemma 4), so their chat markers are listed only there.
126    pub fn with_tokenizer_json(mut self, tokenizer_json: &str) -> anyhow::Result<Self> {
127        #[derive(Deserialize)]
128        struct TokenizerJson {
129            #[serde(default)]
130            added_tokens: Vec<AddedTokensDecoder>,
131        }
132        self.tokenizer_added_tokens =
133            serde_json::from_str::<TokenizerJson>(tokenizer_json)?.added_tokens;
134        Ok(self)
135    }
136
137    /// Whether control-token spellings in request text render as ordinary
138    /// text, so that only the template's own markers encode as control tokens.
139    /// On by default; `false` turns it off, except under a template with
140    /// Harmony framing, which always renders this way.
141    ///
142    /// The spellings come back as ordinary segments of
143    /// [`OAIPromptFormatter::render_prompt`](crate::OAIPromptFormatter::render_prompt),
144    /// to encode with [`crate::RenderedPrompt::encode_segments`]; `render`
145    /// returns the same text without them. A model that lists its markers
146    /// only in `tokenizer.json` also needs [`ChatTemplate::with_tokenizer_json`].
147    /// In assistant history, a marker the template itself reads stays a
148    /// control token.
149    pub fn with_request_control_tokens_as_text(mut self, enabled: bool) -> Self {
150        self.request_control_tokens_as_text = Some(enabled);
151        self
152    }
153
154    /// Spellings the tokenizer turns into one added token, each with whether it
155    /// is `special`: every `added_tokens_decoder` entry, the `tokenizer.json`
156    /// `added_tokens` when given, the `additional_special_tokens`, and the
157    /// bos/eos/unk literals. Whitespace-only spellings are dropped; a spelling
158    /// listed twice is special if either entry says so. Sorted for a stable
159    /// result.
160    pub(crate) fn added_token_spellings(&self) -> Vec<(String, bool)> {
161        fn insert(spellings: &mut HashMap<String, bool>, spelling: Option<String>, special: bool) {
162            if let Some(spelling) = spelling
163                && !spelling.trim().is_empty()
164            {
165                *spellings.entry(spelling).or_insert(false) |= special;
166            }
167        }
168        let mut spellings = HashMap::new();
169        for token in self
170            .added_tokens_decoder
171            .iter()
172            .flat_map(HashMap::values)
173            .chain(&self.tokenizer_added_tokens)
174        {
175            insert(
176                &mut spellings,
177                Some(token.content.clone()),
178                token.special.unwrap_or(false),
179            );
180        }
181        for token in self.additional_special_tokens.iter().flatten() {
182            insert(&mut spellings, Some(token.clone()), true);
183        }
184        insert(&mut spellings, self.bos_tok(), true);
185        insert(&mut spellings, self.eos_tok(), true);
186        insert(&mut spellings, self.unk_tok(), true);
187        let mut spellings: Vec<_> = spellings.into_iter().collect();
188        spellings.sort();
189        spellings
190    }
191}
192
193#[allow(dead_code)]
194#[derive(Debug, Deserialize)]
195pub struct GenerationConfig {
196    #[serde(with = "either::serde_untagged")]
197    bos_token_id: Either<u32, Vec<u32>>,
198    #[serde(with = "either::serde_untagged")]
199    eos_token_id: Either<u32, Vec<u32>>,
200}
201
202/// Python `str` of a non-integer float (`1e-06`, `1e+16`, `nan`); minijinja never uses
203/// an exponent (`0.000001`, `10000000000000000.0`) and prints `NaN`. `None` for every
204/// other value.
205fn python_float_str(value: &Value) -> Option<String> {
206    if !value.is_number() || value.is_integer() {
207        return None;
208    }
209    f64::try_from(value.clone()).ok().map(python_float_repr)
210}
211
212/// Output formatter for `{{ x }}`. HF renders through Python, which prints a float
213/// with `str`; see [`python_float_str`]. Other values are unchanged. `~` concatenation
214/// is a VM op that can't be overridden, and filters that stringify values themselves
215/// (`join`, `replace`, ...) use minijinja's spelling, so `'x' ~ 1e-6` and
216/// `[1e-6] | join` still render `0.000001`.
217pub fn python_formatter(out: &mut Output, state: &State, value: &Value) -> Result<(), Error> {
218    match python_float_str(value) {
219        Some(text) => out.write_str(&text).map_err(Error::from),
220        None => escape_formatter(out, state, value),
221    }
222}
223
224/// The `string` filter with Python `str` floats, matching [`python_formatter`];
225/// other values go to the builtin.
226pub fn python_string(state: &State, value: &Value) -> Result<Value, Error> {
227    match python_float_str(value) {
228        Some(text) => Ok(Value::from(text)),
229        None => minijinja::filters::string(state, value),
230    }
231}
232
233/// minijinja-contrib's pycompat string methods, with Python's whitespace where
234/// Rust's differs: `strip`, `lstrip`, `rstrip` and `split()` also treat U+001C..U+001F
235/// as whitespace, `isspace` is false for `""`, and `splitlines` ends lines at every
236/// Python line break. HF renders templates through Python, so `m.content.strip()` must
237/// strip what Python strips.
238pub fn python_methods(
239    state: &State,
240    value: &Value,
241    method: &str,
242    args: &[Value],
243) -> Result<Value, Error> {
244    if let Some(s) = value.as_str() {
245        match method {
246            "strip" | "lstrip" | "rstrip" => {
247                let (chars,): (Option<&str>,) = from_args(args)?;
248                if chars.is_none() {
249                    let trimmed = match method {
250                        "strip" => s.trim_matches(is_python_space),
251                        "lstrip" => s.trim_start_matches(is_python_space),
252                        _ => s.trim_end_matches(is_python_space),
253                    };
254                    return Ok(StringInput::new(state, value)?.preserve_safety(trimmed.to_owned()));
255                }
256            }
257            "split" => {
258                let (sep, maxsplit): (Option<&str>, Option<i64>) = from_args(args)?;
259                if sep.is_none() {
260                    let maxsplit = maxsplit.and_then(|n| usize::try_from(n).ok());
261                    return Ok(python_split_whitespace(s, maxsplit)
262                        .into_iter()
263                        .map(|part| {
264                            if value.is_safe() {
265                                Value::from_safe_string(part.to_owned())
266                            } else {
267                                Value::from(part)
268                            }
269                        })
270                        .collect());
271                }
272            }
273            "splitlines" => {
274                let (keepends,): (Option<bool>,) = from_args(args)?;
275                return Ok(python_splitlines(s, keepends.unwrap_or(false))
276                    .into_iter()
277                    .map(|part| {
278                        if value.is_safe() {
279                            Value::from_safe_string(part.to_owned())
280                        } else {
281                            Value::from(part)
282                        }
283                    })
284                    .collect());
285            }
286            "isspace" => {
287                let () = from_args(args)?;
288                return Ok(Value::from(!s.is_empty() && s.chars().all(is_python_space)));
289            }
290            _ => {}
291        }
292    }
293    minijinja_contrib::pycompat::unknown_method_callback(state, value, method, args)
294}
295
296/// Jinja2's `trim` filter, `str.strip(chars)`, with Python's whitespace by default;
297/// see [`python_methods`].
298pub fn python_trim(value: StringInput<'_>, chars: Option<Cow<'_, str>>) -> Value {
299    match chars {
300        Some(chars) => minijinja::filters::trim(value, Some(chars)),
301        None => {
302            let trimmed = value.as_str().trim_matches(is_python_space).to_owned();
303            value.preserve_safety(trimmed)
304        }
305    }
306}
307
308/// Mirrors HF transformers' `tojson` filter, not stock Jinja2's. Transformers
309/// overrides Jinja's HTML-safe `tojson` with plain
310/// `json.dumps(x, ensure_ascii=ensure_ascii, indent=indent, separators=separators,
311/// sort_keys=sort_keys)`, `ensure_ascii` defaulting to `False`, in its chat-template
312/// environment, and vLLM/SGLang render through that — so chat templates (and the
313/// models trained on their output) expect Python separators and **no** HTML
314/// escaping (`'`, `<`, `>`, `&` stay literal).
315pub fn tojson(value: Value, kwargs: Kwargs) -> Result<Value, Error> {
316    // Python takes a space count or the indent string itself.
317    let indent = kwargs
318        .get::<Option<Value>>("indent")?
319        .map(|indent| match indent.as_str() {
320            Some(text) => Ok(text.to_owned()),
321            None => i64::try_from(indent)
322                .map(|n| " ".repeat(usize::try_from(n).unwrap_or(0)))
323                .map_err(|_| {
324                    Error::new(
325                        ErrorKind::InvalidOperation,
326                        "tojson indent must be an int or a string",
327                    )
328                }),
329        })
330        .transpose()?;
331    // Python unpacks any two-item iterable of strings, so `',:'` works too.
332    let separators = kwargs
333        .get::<Option<Value>>("separators")?
334        .map(|pair| {
335            let pair: Vec<Value> = pair.try_iter().map(Iterator::collect).unwrap_or_default();
336            match pair.as_slice() {
337                [item, key] => item.as_str().zip(key.as_str()),
338                _ => None,
339            }
340            .map(|(item, key)| (item.to_owned(), key.to_owned()))
341            .ok_or_else(|| {
342                Error::new(
343                    ErrorKind::InvalidOperation,
344                    "tojson separators must be an (item, key) pair of strings",
345                )
346            })
347        })
348        .transpose()?;
349    // Python defaults to `(", ", ": ")`, or `(",", ": ")` when `indent` is set.
350    let (item_sep, key_sep) = match &separators {
351        Some((item, key)) => (item.as_str(), key.as_str()),
352        None if indent.is_some() => (",", ": "),
353        None => (", ", ": "),
354    };
355    let flag = |key| -> Result<bool, Error> {
356        Ok(kwargs
357            .get::<Option<Value>>(key)?
358            .is_some_and(|v| v.is_true()))
359    };
360    let value = if flag("sort_keys")? {
361        crate::python::template_json(PySortedKeys(value))
362    } else {
363        crate::python::template_json(value)
364    }
365    .map_err(|err| {
366        Error::new(ErrorKind::BadSerialization, "cannot serialize to JSON").with_source(err)
367    })?;
368    let formatter =
369        PyJsonFormatter::new(item_sep, key_sep, indent.as_deref(), flag("ensure_ascii")?);
370    let mut buf = Vec::new();
371    let mut serializer = serde_json::Serializer::with_formatter(&mut buf, formatter);
372    value.serialize(&mut serializer).map_err(|err| {
373        Error::new(ErrorKind::BadSerialization, "cannot serialize to JSON").with_source(err)
374    })?;
375    String::from_utf8(buf)
376        .map_err(|err| {
377            Error::new(ErrorKind::BadSerialization, "cannot serialize to JSON").with_source(err)
378        })
379        .map(Value::from_safe_string)
380}
381
382/// A template value whose maps serialize with their keys in Python's `sorted()` order.
383/// `json.dumps(sort_keys=True)` sorts the original keys and only then writes them as
384/// strings, so `{10: 'a', 2: 'b'}` puts 2 first, where sorting the strings would put
385/// `"10"` first.
386struct PySortedKeys(Value);
387
388impl Serialize for PySortedKeys {
389    fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
390        let Some(object) = self.0.as_object() else {
391            return self.0.serialize(serializer);
392        };
393        match object.repr() {
394            ObjectRepr::Map => {
395                let mut pairs: Vec<_> = object.try_iter_pairs().into_iter().flatten().collect();
396                python_sort_keys(&mut pairs).map_err(S::Error::custom)?;
397                serializer.collect_map(
398                    pairs
399                        .into_iter()
400                        .map(|(key, value)| (key, PySortedKeys(value))),
401                )
402            }
403            ObjectRepr::Seq | ObjectRepr::Iterable => {
404                serializer.collect_seq(object.try_iter().into_iter().flatten().map(PySortedKeys))
405            }
406            _ => self.0.serialize(serializer),
407        }
408    }
409}
410
411/// Python's `sorted()` over dict keys: strings order among strings, and numbers, bools
412/// included, among numbers. Python raises `TypeError` for any other mix, so it fails
413/// the render.
414fn python_sort_keys(pairs: &mut [(Value, Value)]) -> Result<(), Error> {
415    let number = |key: &Value| match key.kind() {
416        ValueKind::Bool => Some(Value::from(i64::from(key.is_true()))),
417        ValueKind::Number => Some(key.clone()),
418        _ => None,
419    };
420    if pairs.iter().all(|(key, _)| key.kind() == ValueKind::String) {
421        pairs.sort_by(|(a, _), (b, _)| a.as_str().cmp(&b.as_str()));
422    } else if pairs.iter().all(|(key, _)| number(key).is_some()) {
423        pairs.sort_by_cached_key(|(key, _)| number(key));
424    } else if pairs.len() > 1 {
425        return Err(Error::new(
426            ErrorKind::InvalidOperation,
427            "tojson sort_keys cannot order keys of different types",
428        ));
429    }
430    Ok(())
431}
432
433/// Parse a JSON string into a structured value.
434///
435/// HuggingFace/transformers chat-template environments expose this filter, and several
436/// published templates depend on it — e.g. Step-3.7-Flash's `tool_use` block applies it to
437/// a tool call's `arguments`, which arrive as a JSON *string*, to iterate the decoded
438/// object. Without it minijinja aborts the render with `unknown filter: fromjson`, which
439/// fails every multi-turn tool-call request.
440///
441/// Values that are not strings pass through untouched, so templates that apply the filter
442/// defensively to an already-decoded value keep rendering.
443pub fn fromjson(value: Value) -> Result<Value, Error> {
444    let Some(text) = value.as_str() else {
445        return Ok(value);
446    };
447    let parsed: serde_json::Value = serde_json::from_str(text).map_err(|err| {
448        Error::new(ErrorKind::InvalidOperation, "cannot parse JSON").with_source(err)
449    })?;
450    Ok(crate::python::template_value(&parsed))
451}
452
453pub fn strftime_now(format_str: &str) -> Result<Value, Error> {
454    let local: DateTime<Local> = Local::now();
455    Ok(Value::from_safe_string(
456        local.format(format_str).to_string(),
457    ))
458}
459
460#[cfg(test)]
461mod fromjson_tests {
462    use super::*;
463
464    #[test]
465    fn parses_json_object_string() {
466        let out = fromjson(Value::from(r#"{"location":"San Francisco","n":3}"#)).unwrap();
467        assert_eq!(
468            out.get_attr("location").unwrap().as_str(),
469            Some("San Francisco")
470        );
471        assert_eq!(out.get_attr("n").unwrap().to_string(), "3");
472    }
473
474    #[test]
475    fn parses_json_array_string() {
476        let out = fromjson(Value::from(r#"[1,2,3]"#)).unwrap();
477        assert_eq!(out.len(), Some(3));
478    }
479
480    #[test]
481    fn passes_through_non_string() {
482        // Already-decoded values must survive a defensive `| fromjson`.
483        let already = crate::python::template_value(&serde_json::json!({"a": 1}));
484        let out = fromjson(already).unwrap();
485        assert_eq!(out.get_attr("a").unwrap().to_string(), "1");
486    }
487
488    #[test]
489    fn errors_on_malformed_json() {
490        assert!(fromjson(Value::from("{not json")).is_err());
491    }
492
493    /// Regression: renders the shape of Step-3.7-Flash's `tool_use` block, where a tool
494    /// call's `arguments` arrive as a JSON string. Before the filter existed this failed
495    /// with `unknown filter: fromjson`, 500-ing every multi-turn tool-call request.
496    #[test]
497    fn renders_tool_use_block_with_json_string_arguments() {
498        let mut env = minijinja::Environment::new();
499        env.add_filter("fromjson", fromjson);
500        env.add_template(
501            "tool_use",
502            "{% for tc in tool_calls %}{% set a = tc.function.arguments | fromjson %}\
503CALL {{ tc.function.name }} loc={{ a.location }} unit={{ a.unit }}{% endfor %}",
504        )
505        .unwrap();
506        let rendered = env
507            .get_template("tool_use")
508            .unwrap()
509            .render(minijinja::context! { tool_calls => serde_json::json!([{
510                "function": {
511                    "name": "get_weather",
512                    "arguments": "{\"location\":\"San Francisco\",\"unit\":\"F\"}"
513                }
514            }])})
515            .expect("tool_use template must render once `fromjson` is registered");
516        assert_eq!(rendered, "CALL get_weather loc=San Francisco unit=F");
517    }
518}
519
520/// Expected strings are HF transformers' `tojson` output for the same input and
521/// kwargs, i.e. Python `json.dumps(x, ensure_ascii=False, ...)`.
522#[cfg(test)]
523mod tojson_tests {
524    use super::*;
525    use serde_json::json;
526
527    fn try_render(tmpl: &str, x: serde_json::Value) -> Result<String, Error> {
528        let mut env = minijinja::Environment::new();
529        env.add_filter("tojson", tojson);
530        env.render_str(tmpl, minijinja::context! { x })
531    }
532
533    fn render(tmpl: &str, x: serde_json::Value) -> String {
534        try_render(tmpl, x).unwrap()
535    }
536
537    fn shape() -> serde_json::Value {
538        json!({"a": [1, 2], "b": {"c": "é"}, "d": [], "e": {}})
539    }
540
541    /// Kimi K2.x `tool_declare` renders `tools | tojson(separators=(',', ':'))`.
542    #[test]
543    fn honours_separators() {
544        let tools = json!([{"type": "function", "function": {"name": "get_weather",
545            "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}}}]);
546        assert_eq!(
547            render("{{ x | tojson(separators=(',', ':')) }}", tools),
548            r#"[{"type":"function","function":{"name":"get_weather","parameters":{"type":"object","properties":{"city":{"type":"string"}}}}}]"#
549        );
550    }
551
552    #[test]
553    fn honours_separators_with_indent() {
554        assert_eq!(
555            render(
556                "{{ x | tojson(indent=2, separators=(', ', ':')) }}",
557                shape()
558            ),
559            "{\n  \"a\":[\n    1, \n    2\n  ], \n  \"b\":{\n    \"c\":\"é\"\n  }, \n  \"d\":[], \n  \"e\":{}\n}"
560        );
561    }
562
563    /// Python takes the indent string itself as well as a space count.
564    #[test]
565    fn indent_accepts_a_string() {
566        assert_eq!(
567            render("{{ x | tojson(indent='\t') }}", json!({"a": [1]})),
568            "{\n\t\"a\": [\n\t\t1\n\t]\n}"
569        );
570    }
571
572    /// Python unpacks `separators` from any two-item iterable, a string included.
573    #[test]
574    fn separators_accept_a_two_character_string() {
575        assert_eq!(
576            render("{{ x | tojson(separators=',:') }}", shape()),
577            r#"{"a":[1,2],"b":{"c":"é"},"d":[],"e":{}}"#
578        );
579    }
580
581    /// Python's `json.dumps` raises on these too.
582    #[test]
583    fn rejects_malformed_kwargs() {
584        for tmpl in [
585            "{{ x | tojson(separators=(',',)) }}",
586            "{{ x | tojson(separators=(1, 2)) }}",
587            "{{ x | tojson(indent=[2]) }}",
588        ] {
589            let err = try_render(tmpl, shape()).unwrap_err();
590            assert_eq!(err.kind(), ErrorKind::InvalidOperation, "{tmpl}");
591        }
592    }
593
594    #[test]
595    fn sort_keys_sorts_nested_maps() {
596        let x = json!({"b": 1, "a": {"d": [{"z": 1, "y": 2}], "c": 3}});
597        assert_eq!(
598            render("{{ x | tojson(sort_keys=true) }}", x),
599            r#"{"a": {"c": 3, "d": [{"y": 2, "z": 1}]}, "b": 1}"#
600        );
601    }
602
603    /// Python sorts the original keys and only then writes them as strings, so 2
604    /// precedes 10, and comparing a string key with a number key raises.
605    #[test]
606    fn sort_keys_compares_original_keys() {
607        assert_eq!(
608            render(
609                "{{ {10: 'ten', 2: 'two'} | tojson(sort_keys=true) }}",
610                json!(null)
611            ),
612            r#"{"2": "two", "10": "ten"}"#
613        );
614        assert!(
615            try_render(
616                "{{ {'2': 'str', 2: 'int'} | tojson(sort_keys=true) }}",
617                json!(null)
618            )
619            .is_err()
620        );
621    }
622
623    #[test]
624    fn ensure_ascii_escapes_like_python() {
625        let x = json!({"clé": "café 😀 \u{7f} \n"});
626        assert_eq!(
627            render("{{ x | tojson(ensure_ascii=true) }}", x),
628            r#"{"cl\u00e9": "caf\u00e9 \ud83d\ude00 \u007f \n"}"#
629        );
630    }
631
632    #[test]
633    fn defaults_unchanged() {
634        let flat = r#"{"a": [1, 2], "b": {"c": "é"}, "d": [], "e": {}}"#;
635        assert_eq!(render("{{ x | tojson }}", shape()), flat);
636        assert_eq!(
637            render(
638                "{{ x | tojson(separators=none, ensure_ascii=false) }}",
639                shape()
640            ),
641            flat
642        );
643        assert_eq!(
644            render("{{ x | tojson(indent=2) }}", shape()),
645            "{\n  \"a\": [\n    1,\n    2\n  ],\n  \"b\": {\n    \"c\": \"é\"\n  },\n  \"d\": [],\n  \"e\": {}\n}"
646        );
647    }
648
649    /// `sort_keys` re-serializes through `serde_json::Value`; floats keep Python spelling.
650    #[test]
651    fn sort_keys_keeps_python_floats() {
652        assert_eq!(
653            render(
654                "{{ x | tojson(separators=(',', ':'), sort_keys=true) }}",
655                json!({"n": 1e16, "lr": 0.000001})
656            ),
657            r#"{"lr":1e-06,"n":1e+16}"#
658        );
659    }
660}