1use std::collections::HashMap;
7
8use chrono::{DateTime, Local};
9use either::Either;
10use std::borrow::Cow;
11
12use minijinja::value::{Kwargs, ObjectRepr, StringInput, ValueKind, from_args};
13use minijinja::{Error, ErrorKind, Output, State, Value, escape_formatter};
14use serde::ser::Error as _;
15use serde::{Deserialize, Serialize};
16
17use crate::python::{
18 PyJsonFormatter, is_python_space, python_float_repr, python_split_whitespace, python_splitlines,
19};
20
21#[allow(dead_code)]
22#[derive(Debug, Deserialize)]
23pub struct AddedTokensDecoder {
24 __type: Option<String>,
25 pub content: String,
26 lstrip: bool,
27 normalized: bool,
28 rstrip: bool,
29 single_word: bool,
30 special: Option<bool>,
31}
32
33pub fn raise_exception(msg: String) -> Result<String, minijinja::Error> {
34 Err(minijinja::Error::new(ErrorKind::InvalidOperation, msg))
35}
36
37#[derive(Debug, Deserialize)]
38pub struct BeginEndUnkTok(
39 #[serde(with = "either::serde_untagged")] pub Either<String, AddedTokensDecoder>,
40);
41
42#[derive(Debug, Deserialize)]
49pub struct ChatTemplateValue(
50 #[serde(with = "either::serde_untagged")] pub Either<String, Vec<HashMap<String, String>>>,
51);
52
53#[allow(dead_code)]
55#[derive(Debug, Deserialize)]
56pub struct PadTokenValue(
57 #[serde(with = "either::serde_untagged")] pub Either<String, AddedTokensDecoder>,
58);
59
60#[allow(dead_code)]
61#[derive(Debug, Deserialize, Default)]
62pub struct ChatTemplate {
64 pub bos_token: Option<BeginEndUnkTok>,
65 pub eos_token: Option<BeginEndUnkTok>,
66 pub unk_token: Option<BeginEndUnkTok>,
67
68 pub chat_template: Option<ChatTemplateValue>,
72
73 add_bos_token: Option<bool>,
75 add_eos_token: Option<bool>,
76 added_tokens_decoder: Option<HashMap<String, AddedTokensDecoder>>,
77 additional_special_tokens: Option<Vec<String>>,
78 clean_up_tokenization_spaces: Option<bool>,
79 device_map: Option<String>,
80 legacy: Option<bool>,
81 model_max_length: Option<f64>,
82 pad_token: Option<PadTokenValue>,
83 sp_model_kwargs: Option<HashMap<String, String>>,
84 spaces_between_special_tokens: Option<bool>,
85 tokenizer_class: Option<String>,
86 truncation_size: Option<String>,
87 use_default_system_prompt: Option<bool>,
88
89 #[serde(skip)]
92 tokenizer_added_tokens: Vec<AddedTokensDecoder>,
93
94 #[serde(skip)]
97 pub(crate) request_control_tokens_as_text: Option<bool>,
98}
99
100impl ChatTemplate {
101 pub fn eos_tok(&self) -> Option<String> {
102 match self.eos_token.as_ref()?.0 {
103 Either::Left(ref lit) => Some(lit.clone()),
104 Either::Right(ref added) => Some(added.content.clone()),
105 }
106 }
107
108 pub fn bos_tok(&self) -> Option<String> {
109 match self.bos_token.as_ref()?.0 {
110 Either::Left(ref lit) => Some(lit.clone()),
111 Either::Right(ref added) => Some(added.content.clone()),
112 }
113 }
114
115 pub fn unk_tok(&self) -> Option<String> {
116 match self.unk_token.as_ref()?.0 {
117 Either::Left(ref lit) => Some(lit.clone()),
118 Either::Right(ref added) => Some(added.content.clone()),
119 }
120 }
121
122 pub fn with_tokenizer_json(mut self, tokenizer_json: &str) -> anyhow::Result<Self> {
127 #[derive(Deserialize)]
128 struct TokenizerJson {
129 #[serde(default)]
130 added_tokens: Vec<AddedTokensDecoder>,
131 }
132 self.tokenizer_added_tokens =
133 serde_json::from_str::<TokenizerJson>(tokenizer_json)?.added_tokens;
134 Ok(self)
135 }
136
137 pub fn with_request_control_tokens_as_text(mut self, enabled: bool) -> Self {
150 self.request_control_tokens_as_text = Some(enabled);
151 self
152 }
153
154 pub(crate) fn added_token_spellings(&self) -> Vec<(String, bool)> {
161 fn insert(spellings: &mut HashMap<String, bool>, spelling: Option<String>, special: bool) {
162 if let Some(spelling) = spelling
163 && !spelling.trim().is_empty()
164 {
165 *spellings.entry(spelling).or_insert(false) |= special;
166 }
167 }
168 let mut spellings = HashMap::new();
169 for token in self
170 .added_tokens_decoder
171 .iter()
172 .flat_map(HashMap::values)
173 .chain(&self.tokenizer_added_tokens)
174 {
175 insert(
176 &mut spellings,
177 Some(token.content.clone()),
178 token.special.unwrap_or(false),
179 );
180 }
181 for token in self.additional_special_tokens.iter().flatten() {
182 insert(&mut spellings, Some(token.clone()), true);
183 }
184 insert(&mut spellings, self.bos_tok(), true);
185 insert(&mut spellings, self.eos_tok(), true);
186 insert(&mut spellings, self.unk_tok(), true);
187 let mut spellings: Vec<_> = spellings.into_iter().collect();
188 spellings.sort();
189 spellings
190 }
191}
192
193#[allow(dead_code)]
194#[derive(Debug, Deserialize)]
195pub struct GenerationConfig {
196 #[serde(with = "either::serde_untagged")]
197 bos_token_id: Either<u32, Vec<u32>>,
198 #[serde(with = "either::serde_untagged")]
199 eos_token_id: Either<u32, Vec<u32>>,
200}
201
202fn python_float_str(value: &Value) -> Option<String> {
206 if !value.is_number() || value.is_integer() {
207 return None;
208 }
209 f64::try_from(value.clone()).ok().map(python_float_repr)
210}
211
212pub fn python_formatter(out: &mut Output, state: &State, value: &Value) -> Result<(), Error> {
218 match python_float_str(value) {
219 Some(text) => out.write_str(&text).map_err(Error::from),
220 None => escape_formatter(out, state, value),
221 }
222}
223
224pub fn python_string(state: &State, value: &Value) -> Result<Value, Error> {
227 match python_float_str(value) {
228 Some(text) => Ok(Value::from(text)),
229 None => minijinja::filters::string(state, value),
230 }
231}
232
233pub fn python_methods(
239 state: &State,
240 value: &Value,
241 method: &str,
242 args: &[Value],
243) -> Result<Value, Error> {
244 if let Some(s) = value.as_str() {
245 match method {
246 "strip" | "lstrip" | "rstrip" => {
247 let (chars,): (Option<&str>,) = from_args(args)?;
248 if chars.is_none() {
249 let trimmed = match method {
250 "strip" => s.trim_matches(is_python_space),
251 "lstrip" => s.trim_start_matches(is_python_space),
252 _ => s.trim_end_matches(is_python_space),
253 };
254 return Ok(StringInput::new(state, value)?.preserve_safety(trimmed.to_owned()));
255 }
256 }
257 "split" => {
258 let (sep, maxsplit): (Option<&str>, Option<i64>) = from_args(args)?;
259 if sep.is_none() {
260 let maxsplit = maxsplit.and_then(|n| usize::try_from(n).ok());
261 return Ok(python_split_whitespace(s, maxsplit)
262 .into_iter()
263 .map(|part| {
264 if value.is_safe() {
265 Value::from_safe_string(part.to_owned())
266 } else {
267 Value::from(part)
268 }
269 })
270 .collect());
271 }
272 }
273 "splitlines" => {
274 let (keepends,): (Option<bool>,) = from_args(args)?;
275 return Ok(python_splitlines(s, keepends.unwrap_or(false))
276 .into_iter()
277 .map(|part| {
278 if value.is_safe() {
279 Value::from_safe_string(part.to_owned())
280 } else {
281 Value::from(part)
282 }
283 })
284 .collect());
285 }
286 "isspace" => {
287 let () = from_args(args)?;
288 return Ok(Value::from(!s.is_empty() && s.chars().all(is_python_space)));
289 }
290 _ => {}
291 }
292 }
293 minijinja_contrib::pycompat::unknown_method_callback(state, value, method, args)
294}
295
296pub fn python_trim(value: StringInput<'_>, chars: Option<Cow<'_, str>>) -> Value {
299 match chars {
300 Some(chars) => minijinja::filters::trim(value, Some(chars)),
301 None => {
302 let trimmed = value.as_str().trim_matches(is_python_space).to_owned();
303 value.preserve_safety(trimmed)
304 }
305 }
306}
307
308pub fn tojson(value: Value, kwargs: Kwargs) -> Result<Value, Error> {
316 let indent = kwargs
318 .get::<Option<Value>>("indent")?
319 .map(|indent| match indent.as_str() {
320 Some(text) => Ok(text.to_owned()),
321 None => i64::try_from(indent)
322 .map(|n| " ".repeat(usize::try_from(n).unwrap_or(0)))
323 .map_err(|_| {
324 Error::new(
325 ErrorKind::InvalidOperation,
326 "tojson indent must be an int or a string",
327 )
328 }),
329 })
330 .transpose()?;
331 let separators = kwargs
333 .get::<Option<Value>>("separators")?
334 .map(|pair| {
335 let pair: Vec<Value> = pair.try_iter().map(Iterator::collect).unwrap_or_default();
336 match pair.as_slice() {
337 [item, key] => item.as_str().zip(key.as_str()),
338 _ => None,
339 }
340 .map(|(item, key)| (item.to_owned(), key.to_owned()))
341 .ok_or_else(|| {
342 Error::new(
343 ErrorKind::InvalidOperation,
344 "tojson separators must be an (item, key) pair of strings",
345 )
346 })
347 })
348 .transpose()?;
349 let (item_sep, key_sep) = match &separators {
351 Some((item, key)) => (item.as_str(), key.as_str()),
352 None if indent.is_some() => (",", ": "),
353 None => (", ", ": "),
354 };
355 let flag = |key| -> Result<bool, Error> {
356 Ok(kwargs
357 .get::<Option<Value>>(key)?
358 .is_some_and(|v| v.is_true()))
359 };
360 let value = if flag("sort_keys")? {
361 crate::python::template_json(PySortedKeys(value))
362 } else {
363 crate::python::template_json(value)
364 }
365 .map_err(|err| {
366 Error::new(ErrorKind::BadSerialization, "cannot serialize to JSON").with_source(err)
367 })?;
368 let formatter =
369 PyJsonFormatter::new(item_sep, key_sep, indent.as_deref(), flag("ensure_ascii")?);
370 let mut buf = Vec::new();
371 let mut serializer = serde_json::Serializer::with_formatter(&mut buf, formatter);
372 value.serialize(&mut serializer).map_err(|err| {
373 Error::new(ErrorKind::BadSerialization, "cannot serialize to JSON").with_source(err)
374 })?;
375 String::from_utf8(buf)
376 .map_err(|err| {
377 Error::new(ErrorKind::BadSerialization, "cannot serialize to JSON").with_source(err)
378 })
379 .map(Value::from_safe_string)
380}
381
382struct PySortedKeys(Value);
387
388impl Serialize for PySortedKeys {
389 fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
390 let Some(object) = self.0.as_object() else {
391 return self.0.serialize(serializer);
392 };
393 match object.repr() {
394 ObjectRepr::Map => {
395 let mut pairs: Vec<_> = object.try_iter_pairs().into_iter().flatten().collect();
396 python_sort_keys(&mut pairs).map_err(S::Error::custom)?;
397 serializer.collect_map(
398 pairs
399 .into_iter()
400 .map(|(key, value)| (key, PySortedKeys(value))),
401 )
402 }
403 ObjectRepr::Seq | ObjectRepr::Iterable => {
404 serializer.collect_seq(object.try_iter().into_iter().flatten().map(PySortedKeys))
405 }
406 _ => self.0.serialize(serializer),
407 }
408 }
409}
410
411fn python_sort_keys(pairs: &mut [(Value, Value)]) -> Result<(), Error> {
415 let number = |key: &Value| match key.kind() {
416 ValueKind::Bool => Some(Value::from(i64::from(key.is_true()))),
417 ValueKind::Number => Some(key.clone()),
418 _ => None,
419 };
420 if pairs.iter().all(|(key, _)| key.kind() == ValueKind::String) {
421 pairs.sort_by(|(a, _), (b, _)| a.as_str().cmp(&b.as_str()));
422 } else if pairs.iter().all(|(key, _)| number(key).is_some()) {
423 pairs.sort_by_cached_key(|(key, _)| number(key));
424 } else if pairs.len() > 1 {
425 return Err(Error::new(
426 ErrorKind::InvalidOperation,
427 "tojson sort_keys cannot order keys of different types",
428 ));
429 }
430 Ok(())
431}
432
433pub fn fromjson(value: Value) -> Result<Value, Error> {
444 let Some(text) = value.as_str() else {
445 return Ok(value);
446 };
447 let parsed: serde_json::Value = serde_json::from_str(text).map_err(|err| {
448 Error::new(ErrorKind::InvalidOperation, "cannot parse JSON").with_source(err)
449 })?;
450 Ok(crate::python::template_value(&parsed))
451}
452
453pub fn strftime_now(format_str: &str) -> Result<Value, Error> {
454 let local: DateTime<Local> = Local::now();
455 Ok(Value::from_safe_string(
456 local.format(format_str).to_string(),
457 ))
458}
459
460#[cfg(test)]
461mod fromjson_tests {
462 use super::*;
463
464 #[test]
465 fn parses_json_object_string() {
466 let out = fromjson(Value::from(r#"{"location":"San Francisco","n":3}"#)).unwrap();
467 assert_eq!(
468 out.get_attr("location").unwrap().as_str(),
469 Some("San Francisco")
470 );
471 assert_eq!(out.get_attr("n").unwrap().to_string(), "3");
472 }
473
474 #[test]
475 fn parses_json_array_string() {
476 let out = fromjson(Value::from(r#"[1,2,3]"#)).unwrap();
477 assert_eq!(out.len(), Some(3));
478 }
479
480 #[test]
481 fn passes_through_non_string() {
482 let already = crate::python::template_value(&serde_json::json!({"a": 1}));
484 let out = fromjson(already).unwrap();
485 assert_eq!(out.get_attr("a").unwrap().to_string(), "1");
486 }
487
488 #[test]
489 fn errors_on_malformed_json() {
490 assert!(fromjson(Value::from("{not json")).is_err());
491 }
492
493 #[test]
497 fn renders_tool_use_block_with_json_string_arguments() {
498 let mut env = minijinja::Environment::new();
499 env.add_filter("fromjson", fromjson);
500 env.add_template(
501 "tool_use",
502 "{% for tc in tool_calls %}{% set a = tc.function.arguments | fromjson %}\
503CALL {{ tc.function.name }} loc={{ a.location }} unit={{ a.unit }}{% endfor %}",
504 )
505 .unwrap();
506 let rendered = env
507 .get_template("tool_use")
508 .unwrap()
509 .render(minijinja::context! { tool_calls => serde_json::json!([{
510 "function": {
511 "name": "get_weather",
512 "arguments": "{\"location\":\"San Francisco\",\"unit\":\"F\"}"
513 }
514 }])})
515 .expect("tool_use template must render once `fromjson` is registered");
516 assert_eq!(rendered, "CALL get_weather loc=San Francisco unit=F");
517 }
518}
519
520#[cfg(test)]
523mod tojson_tests {
524 use super::*;
525 use serde_json::json;
526
527 fn try_render(tmpl: &str, x: serde_json::Value) -> Result<String, Error> {
528 let mut env = minijinja::Environment::new();
529 env.add_filter("tojson", tojson);
530 env.render_str(tmpl, minijinja::context! { x })
531 }
532
533 fn render(tmpl: &str, x: serde_json::Value) -> String {
534 try_render(tmpl, x).unwrap()
535 }
536
537 fn shape() -> serde_json::Value {
538 json!({"a": [1, 2], "b": {"c": "é"}, "d": [], "e": {}})
539 }
540
541 #[test]
543 fn honours_separators() {
544 let tools = json!([{"type": "function", "function": {"name": "get_weather",
545 "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}}}]);
546 assert_eq!(
547 render("{{ x | tojson(separators=(',', ':')) }}", tools),
548 r#"[{"type":"function","function":{"name":"get_weather","parameters":{"type":"object","properties":{"city":{"type":"string"}}}}}]"#
549 );
550 }
551
552 #[test]
553 fn honours_separators_with_indent() {
554 assert_eq!(
555 render(
556 "{{ x | tojson(indent=2, separators=(', ', ':')) }}",
557 shape()
558 ),
559 "{\n \"a\":[\n 1, \n 2\n ], \n \"b\":{\n \"c\":\"é\"\n }, \n \"d\":[], \n \"e\":{}\n}"
560 );
561 }
562
563 #[test]
565 fn indent_accepts_a_string() {
566 assert_eq!(
567 render("{{ x | tojson(indent='\t') }}", json!({"a": [1]})),
568 "{\n\t\"a\": [\n\t\t1\n\t]\n}"
569 );
570 }
571
572 #[test]
574 fn separators_accept_a_two_character_string() {
575 assert_eq!(
576 render("{{ x | tojson(separators=',:') }}", shape()),
577 r#"{"a":[1,2],"b":{"c":"é"},"d":[],"e":{}}"#
578 );
579 }
580
581 #[test]
583 fn rejects_malformed_kwargs() {
584 for tmpl in [
585 "{{ x | tojson(separators=(',',)) }}",
586 "{{ x | tojson(separators=(1, 2)) }}",
587 "{{ x | tojson(indent=[2]) }}",
588 ] {
589 let err = try_render(tmpl, shape()).unwrap_err();
590 assert_eq!(err.kind(), ErrorKind::InvalidOperation, "{tmpl}");
591 }
592 }
593
594 #[test]
595 fn sort_keys_sorts_nested_maps() {
596 let x = json!({"b": 1, "a": {"d": [{"z": 1, "y": 2}], "c": 3}});
597 assert_eq!(
598 render("{{ x | tojson(sort_keys=true) }}", x),
599 r#"{"a": {"c": 3, "d": [{"y": 2, "z": 1}]}, "b": 1}"#
600 );
601 }
602
603 #[test]
606 fn sort_keys_compares_original_keys() {
607 assert_eq!(
608 render(
609 "{{ {10: 'ten', 2: 'two'} | tojson(sort_keys=true) }}",
610 json!(null)
611 ),
612 r#"{"2": "two", "10": "ten"}"#
613 );
614 assert!(
615 try_render(
616 "{{ {'2': 'str', 2: 'int'} | tojson(sort_keys=true) }}",
617 json!(null)
618 )
619 .is_err()
620 );
621 }
622
623 #[test]
624 fn ensure_ascii_escapes_like_python() {
625 let x = json!({"clé": "café 😀 \u{7f} \n"});
626 assert_eq!(
627 render("{{ x | tojson(ensure_ascii=true) }}", x),
628 r#"{"cl\u00e9": "caf\u00e9 \ud83d\ude00 \u007f \n"}"#
629 );
630 }
631
632 #[test]
633 fn defaults_unchanged() {
634 let flat = r#"{"a": [1, 2], "b": {"c": "é"}, "d": [], "e": {}}"#;
635 assert_eq!(render("{{ x | tojson }}", shape()), flat);
636 assert_eq!(
637 render(
638 "{{ x | tojson(separators=none, ensure_ascii=false) }}",
639 shape()
640 ),
641 flat
642 );
643 assert_eq!(
644 render("{{ x | tojson(indent=2) }}", shape()),
645 "{\n \"a\": [\n 1,\n 2\n ],\n \"b\": {\n \"c\": \"é\"\n },\n \"d\": [],\n \"e\": {}\n}"
646 );
647 }
648
649 #[test]
651 fn sort_keys_keeps_python_floats() {
652 assert_eq!(
653 render(
654 "{{ x | tojson(separators=(',', ':'), sort_keys=true) }}",
655 json!({"n": 1e16, "lr": 0.000001})
656 ),
657 r#"{"lr":1e-06,"n":1e+16}"#
658 );
659 }
660}