use std::collections::HashMap;
use chrono::{DateTime, Local};
use either::Either;
use std::borrow::Cow;
use minijinja::value::{Kwargs, StringInput, from_args};
use minijinja::{Error, ErrorKind, Output, State, Value, escape_formatter};
use serde::{Deserialize, Serialize};
use crate::python::{
PyFloats, PyJsonFormatter, is_python_space, python_float_repr, python_split_whitespace,
python_splitlines,
};
#[allow(dead_code)]
#[derive(Debug, Deserialize)]
pub struct AddedTokensDecoder {
__type: Option<String>,
pub content: String,
lstrip: bool,
normalized: bool,
rstrip: bool,
single_word: bool,
special: Option<bool>,
}
pub fn raise_exception(msg: String) -> Result<String, minijinja::Error> {
Err(minijinja::Error::new(ErrorKind::InvalidOperation, msg))
}
#[derive(Debug, Deserialize)]
pub struct BeginEndUnkTok(
#[serde(with = "either::serde_untagged")] pub Either<String, AddedTokensDecoder>,
);
#[derive(Debug, Deserialize)]
pub struct ChatTemplateValue(
#[serde(with = "either::serde_untagged")] pub Either<String, Vec<HashMap<String, String>>>,
);
#[allow(dead_code)]
#[derive(Debug, Deserialize)]
pub struct PadTokenValue(
#[serde(with = "either::serde_untagged")] pub Either<String, AddedTokensDecoder>,
);
#[allow(dead_code)]
#[derive(Debug, Deserialize, Default)]
pub struct ChatTemplate {
pub bos_token: Option<BeginEndUnkTok>,
pub eos_token: Option<BeginEndUnkTok>,
pub unk_token: Option<BeginEndUnkTok>,
pub chat_template: Option<ChatTemplateValue>,
add_bos_token: Option<bool>,
add_eos_token: Option<bool>,
added_tokens_decoder: Option<HashMap<String, AddedTokensDecoder>>,
additional_special_tokens: Option<Vec<String>>,
clean_up_tokenization_spaces: Option<bool>,
device_map: Option<String>,
legacy: Option<bool>,
model_max_length: Option<f64>,
pad_token: Option<PadTokenValue>,
sp_model_kwargs: Option<HashMap<String, String>>,
spaces_between_special_tokens: Option<bool>,
tokenizer_class: Option<String>,
truncation_size: Option<String>,
use_default_system_prompt: Option<bool>,
#[serde(skip)]
tokenizer_added_tokens: Vec<AddedTokensDecoder>,
#[serde(skip)]
pub(crate) request_control_tokens_as_text: Option<bool>,
}
impl ChatTemplate {
pub fn eos_tok(&self) -> Option<String> {
match self.eos_token.as_ref()?.0 {
Either::Left(ref lit) => Some(lit.clone()),
Either::Right(ref added) => Some(added.content.clone()),
}
}
pub fn bos_tok(&self) -> Option<String> {
match self.bos_token.as_ref()?.0 {
Either::Left(ref lit) => Some(lit.clone()),
Either::Right(ref added) => Some(added.content.clone()),
}
}
pub fn unk_tok(&self) -> Option<String> {
match self.unk_token.as_ref()?.0 {
Either::Left(ref lit) => Some(lit.clone()),
Either::Right(ref added) => Some(added.content.clone()),
}
}
pub fn with_tokenizer_json(mut self, tokenizer_json: &str) -> anyhow::Result<Self> {
#[derive(Deserialize)]
struct TokenizerJson {
#[serde(default)]
added_tokens: Vec<AddedTokensDecoder>,
}
self.tokenizer_added_tokens =
serde_json::from_str::<TokenizerJson>(tokenizer_json)?.added_tokens;
Ok(self)
}
pub fn with_request_control_tokens_as_text(mut self, enabled: bool) -> Self {
self.request_control_tokens_as_text = Some(enabled);
self
}
pub(crate) fn added_token_spellings(&self) -> Vec<(String, bool)> {
fn insert(spellings: &mut HashMap<String, bool>, spelling: Option<String>, special: bool) {
if let Some(spelling) = spelling
&& !spelling.trim().is_empty()
{
*spellings.entry(spelling).or_insert(false) |= special;
}
}
let mut spellings = HashMap::new();
for token in self
.added_tokens_decoder
.iter()
.flat_map(HashMap::values)
.chain(&self.tokenizer_added_tokens)
{
insert(
&mut spellings,
Some(token.content.clone()),
token.special.unwrap_or(false),
);
}
for token in self.additional_special_tokens.iter().flatten() {
insert(&mut spellings, Some(token.clone()), true);
}
insert(&mut spellings, self.bos_tok(), true);
insert(&mut spellings, self.eos_tok(), true);
insert(&mut spellings, self.unk_tok(), true);
let mut spellings: Vec<_> = spellings.into_iter().collect();
spellings.sort();
spellings
}
}
#[allow(dead_code)]
#[derive(Debug, Deserialize)]
pub struct GenerationConfig {
#[serde(with = "either::serde_untagged")]
bos_token_id: Either<u32, Vec<u32>>,
#[serde(with = "either::serde_untagged")]
eos_token_id: Either<u32, Vec<u32>>,
}
fn python_float_str(value: &Value) -> Option<String> {
if !value.is_number() || value.is_integer() {
return None;
}
f64::try_from(value.clone()).ok().map(python_float_repr)
}
pub fn python_formatter(out: &mut Output, state: &State, value: &Value) -> Result<(), Error> {
match python_float_str(value) {
Some(text) => out.write_str(&text).map_err(Error::from),
None => escape_formatter(out, state, value),
}
}
pub fn python_string(state: &State, value: &Value) -> Result<Value, Error> {
match python_float_str(value) {
Some(text) => Ok(Value::from(text)),
None => minijinja::filters::string(state, value),
}
}
pub fn python_methods(
state: &State,
value: &Value,
method: &str,
args: &[Value],
) -> Result<Value, Error> {
if let Some(s) = value.as_str() {
match method {
"strip" | "lstrip" | "rstrip" => {
let (chars,): (Option<&str>,) = from_args(args)?;
if chars.is_none() {
let trimmed = match method {
"strip" => s.trim_matches(is_python_space),
"lstrip" => s.trim_start_matches(is_python_space),
_ => s.trim_end_matches(is_python_space),
};
return Ok(StringInput::new(state, value)?.preserve_safety(trimmed.to_owned()));
}
}
"split" => {
let (sep, maxsplit): (Option<&str>, Option<i64>) = from_args(args)?;
if sep.is_none() {
let maxsplit = maxsplit.and_then(|n| usize::try_from(n).ok());
return Ok(python_split_whitespace(s, maxsplit)
.into_iter()
.map(|part| {
if value.is_safe() {
Value::from_safe_string(part.to_owned())
} else {
Value::from(part)
}
})
.collect());
}
}
"splitlines" => {
let (keepends,): (Option<bool>,) = from_args(args)?;
return Ok(python_splitlines(s, keepends.unwrap_or(false))
.into_iter()
.map(|part| {
if value.is_safe() {
Value::from_safe_string(part.to_owned())
} else {
Value::from(part)
}
})
.collect());
}
"isspace" => {
let () = from_args(args)?;
return Ok(Value::from(!s.is_empty() && s.chars().all(is_python_space)));
}
_ => {}
}
}
minijinja_contrib::pycompat::unknown_method_callback(state, value, method, args)
}
pub fn python_trim(value: StringInput<'_>, chars: Option<Cow<'_, str>>) -> Value {
match chars {
Some(chars) => minijinja::filters::trim(value, Some(chars)),
None => {
let trimmed = value.as_str().trim_matches(is_python_space).to_owned();
value.preserve_safety(trimmed)
}
}
}
pub fn tojson(value: Value, kwargs: Kwargs) -> Result<Value, Error> {
let value = crate::python::template_json(value).map_err(|err| {
Error::new(ErrorKind::BadSerialization, "cannot serialize to JSON").with_source(err)
})?;
let mut buf = Vec::new();
let result = if let Ok(indent) = kwargs.get("indent") {
let repeat = b" ".repeat(indent);
let formatter = PyFloats(serde_json::ser::PrettyFormatter::with_indent(&repeat));
let mut serializer = serde_json::Serializer::with_formatter(&mut buf, formatter);
value.serialize(&mut serializer)
} else {
let mut serializer = serde_json::Serializer::with_formatter(&mut buf, PyJsonFormatter);
value.serialize(&mut serializer)
};
result.map_err(|err| {
Error::new(ErrorKind::BadSerialization, "cannot serialize to JSON").with_source(err)
})?;
String::from_utf8(buf)
.map_err(|err| {
Error::new(ErrorKind::BadSerialization, "cannot serialize to JSON").with_source(err)
})
.map(Value::from_safe_string)
}
pub fn fromjson(value: Value) -> Result<Value, Error> {
let Some(text) = value.as_str() else {
return Ok(value);
};
let parsed: serde_json::Value = serde_json::from_str(text).map_err(|err| {
Error::new(ErrorKind::InvalidOperation, "cannot parse JSON").with_source(err)
})?;
Ok(crate::python::template_value(&parsed))
}
pub fn strftime_now(format_str: &str) -> Result<Value, Error> {
let local: DateTime<Local> = Local::now();
Ok(Value::from_safe_string(
local.format(format_str).to_string(),
))
}
#[cfg(test)]
mod fromjson_tests {
use super::*;
#[test]
fn parses_json_object_string() {
let out = fromjson(Value::from(r#"{"location":"San Francisco","n":3}"#)).unwrap();
assert_eq!(
out.get_attr("location").unwrap().as_str(),
Some("San Francisco")
);
assert_eq!(out.get_attr("n").unwrap().to_string(), "3");
}
#[test]
fn parses_json_array_string() {
let out = fromjson(Value::from(r#"[1,2,3]"#)).unwrap();
assert_eq!(out.len(), Some(3));
}
#[test]
fn passes_through_non_string() {
let already = crate::python::template_value(&serde_json::json!({"a": 1}));
let out = fromjson(already).unwrap();
assert_eq!(out.get_attr("a").unwrap().to_string(), "1");
}
#[test]
fn errors_on_malformed_json() {
assert!(fromjson(Value::from("{not json")).is_err());
}
#[test]
fn renders_tool_use_block_with_json_string_arguments() {
let mut env = minijinja::Environment::new();
env.add_filter("fromjson", fromjson);
env.add_template(
"tool_use",
"{% for tc in tool_calls %}{% set a = tc.function.arguments | fromjson %}\
CALL {{ tc.function.name }} loc={{ a.location }} unit={{ a.unit }}{% endfor %}",
)
.unwrap();
let rendered = env
.get_template("tool_use")
.unwrap()
.render(minijinja::context! { tool_calls => serde_json::json!([{
"function": {
"name": "get_weather",
"arguments": "{\"location\":\"San Francisco\",\"unit\":\"F\"}"
}
}])})
.expect("tool_use template must render once `fromjson` is registered");
assert_eq!(rendered, "CALL get_weather loc=San Francisco unit=F");
}
}