use super::*;
use crate::{OAIChatLikeRequest, PromptRenderError, TextInput};
use minijinja::{context, value::Value};
use serde_json::json;
use std::{collections::HashMap, result::Result::Ok};
pub fn may_be_fix_tool_schema(tools: serde_json::Value) -> Option<Value> {
let mut updated_tools = Vec::new();
if let Some(arr) = tools.as_array() {
for tool in arr {
let mut tool = tool.clone();
if let Some(function) = tool.get_mut("function") {
if let Some(obj) = function.as_object_mut()
&& !matches!(obj.get("description"), Some(serde_json::Value::String(_)))
{
obj.insert(
"description".to_string(),
serde_json::Value::String(String::new()),
);
}
}
if let Some(function) = tool.get_mut("function")
&& let Some(parameters) = function.get_mut("parameters")
{
if parameters.is_object() {
let mut needs_type = false;
let mut needs_properties = false;
let is_empty = parameters
.as_object()
.map(|o| o.is_empty())
.unwrap_or(false);
if is_empty {
needs_type = true;
needs_properties = true;
} else {
if let Some(obj) = parameters.as_object() {
if !obj.contains_key("type") {
needs_type = true;
}
if !obj.contains_key("properties") {
needs_properties = true;
}
}
}
if (needs_type || needs_properties)
&& let Some(obj) = parameters.as_object_mut()
{
if needs_type {
obj.insert(
"type".to_string(),
serde_json::Value::String("object".to_string()),
);
}
if needs_properties {
obj.insert(
"properties".to_string(),
serde_json::Value::Object(Default::default()),
);
}
}
}
}
updated_tools.push(tool);
}
}
Some(crate::python::template_value(&serde_json::Value::Array(
updated_tools,
)))
}
const DEFAULT_MEDIA_TYPE_CONVERSIONS: &[(&str, &str)] = &[
("image_url", "image"),
("video_url", "video"),
("audio_url", "audio"),
];
fn custom_marker(part: &serde_json::Value) -> serde_json::Value {
json!({"type": "custom", "kind": part.get("kind")})
}
fn convert_media_url_to_placeholder(
content_array: &mut [serde_json::Value],
conversions: &[(&str, &str)],
) {
for part in content_array {
let part_type = part.get("type").and_then(|t| t.as_str()).unwrap_or("");
if part_type == "custom" {
*part = custom_marker(part);
} else if let Some((_, target_type)) = conversions.iter().find(|(src, _)| *src == part_type)
{
*part = serde_json::json!({"type": target_type});
}
}
}
fn may_be_fix_msg_content(
mut messages: serde_json::Value,
preserve_arrays: bool,
native_text_array_roles: &[&str],
merged_turns: &[usize],
image_placeholder_template: Option<&str>,
) -> serde_json::Value {
let Some(arr) = messages.as_array_mut() else {
return messages;
};
for (index, msg) in arr.iter_mut().enumerate() {
let preserve_text_array = !merged_turns.contains(&index)
&& msg
.get("role")
.and_then(serde_json::Value::as_str)
.is_some_and(|role| native_text_array_roles.contains(&role));
let Some(content) = msg.get_mut("content") else {
continue;
};
match content {
serde_json::Value::String(_) if preserve_arrays => {
let text = content.take();
*content = serde_json::Value::Array(vec![serde_json::Value::Object(
serde_json::Map::from_iter([
("type".into(), serde_json::Value::String("text".into())),
("text".into(), text),
]),
)]);
}
serde_json::Value::Array(parts) => {
convert_media_url_to_placeholder(parts, DEFAULT_MEDIA_TYPE_CONVERSIONS);
let text_only = !parts.is_empty()
&& parts
.iter()
.all(|part| part.get("type").and_then(|v| v.as_str()) == Some("text"));
if text_only && !preserve_arrays {
if !preserve_text_array {
let text = parts
.iter()
.filter_map(|part| part.get("text")?.as_str())
.collect::<Vec<_>>()
.join("\n");
*content = serde_json::Value::String(text);
}
} else if !preserve_arrays
&& !parts.iter().any(|part| part.get("type").and_then(|v| v.as_str()) == Some("custom"))
&& !parts.is_empty()
&& let Some(template) = image_placeholder_template
{
*content = serde_json::Value::String(flatten_mixed_content(parts, template));
}
}
_ => {}
}
}
messages
}
fn flatten_mixed_content(parts: &[serde_json::Value], placeholder_tpl: &str) -> String {
let mut out = String::new();
let mut img_idx: u32 = 1;
for part in parts {
let type_str = part.get("type").and_then(|t| t.as_str()).unwrap_or("");
if type_str == "text" {
if let Some(text) = part.get("text").and_then(|t| t.as_str()) {
out.push_str(text);
}
} else if !type_str.is_empty() {
let placeholder = placeholder_tpl.replace("{n}", &img_idx.to_string());
out.push_str(&placeholder);
img_idx += 1;
}
}
out
}
enum ContentPart<'a> {
Text(Option<&'a str>),
Media(&'static str),
Custom,
Other,
}
fn content_part(part: &serde_json::Value) -> ContentPart<'_> {
let field = |name: &str| part.get(name).and_then(serde_json::Value::as_str);
match part.get("type").and_then(serde_json::Value::as_str) {
Some("text") => ContentPart::Text(field("text")),
Some("refusal") => ContentPart::Text(field("refusal")),
Some("image_url") => ContentPart::Media("image"),
Some("video_url") => ContentPart::Media("video"),
Some("audio_url" | "input_audio") => ContentPart::Media("audio"),
Some("custom") => ContentPart::Custom,
_ => ContentPart::Other,
}
}
fn flatten_msg_content_to_string(
messages: &mut serde_json::Value,
media_placeholders: &MediaPlaceholders,
use_audio_in_video: bool,
) -> Result<()> {
let Some(msgs) = messages.as_array_mut() else {
return Ok(());
};
let mut media_counts = HashMap::new();
for msg in msgs {
let Some(msg) = msg.as_object_mut() else {
continue;
};
let content = match msg.get("content") {
Some(serde_json::Value::String(_)) => continue,
Some(serde_json::Value::Array(parts)) => flatten_parts_to_string(
parts,
media_placeholders,
use_audio_in_video,
&mut media_counts,
)?,
_ => String::new(),
};
msg.insert("content".to_string(), serde_json::Value::String(content));
}
Ok(())
}
fn flatten_parts_to_string(
parts: &[serde_json::Value],
media_placeholders: &MediaPlaceholders,
use_audio_in_video: bool,
media_counts: &mut HashMap<&'static str, usize>,
) -> Result<String> {
let mut texts = Vec::new();
let mut by_modality: Vec<(&str, Vec<String>)> = Vec::new();
for part in parts {
match content_part(part) {
ContentPart::Text(text) => texts.extend(text.filter(|text| !text.is_empty())),
ContentPart::Media(modality) => {
let audio_track = modality == "video"
&& use_audio_in_video
&& part["video_url"]["url"]
.as_str()
.is_some_and(|url| !url.is_empty());
for modality in std::iter::once(modality).chain(audio_track.then_some("audio")) {
let n = media_counts.entry(modality).or_default();
*n += 1;
let template = match modality {
"image" => &media_placeholders.image,
"video" => &media_placeholders.video,
_ => &media_placeholders.audio,
};
if template.is_empty() {
continue;
}
let placeholder = template.replace("{n}", &n.to_string());
match by_modality.iter_mut().find(|(m, _)| *m == modality) {
Some((_, placeholders)) => placeholders.push(placeholder),
None => by_modality.push((modality, vec![placeholder])),
}
}
}
ContentPart::Custom | ContentPart::Other => {
return Err(PromptRenderError::invalid_request(format!(
"content part type {} is not supported with the `string` chat template content format",
part["type"]
))
.into());
}
}
}
let text = texts.join("\n");
let mut counts: Vec<(String, usize)> = Vec::new();
for placeholder in by_modality.into_iter().flat_map(|(_, p)| p) {
match counts.iter_mut().find(|(p, _)| *p == placeholder) {
Some((_, count)) => *count += 1,
None => counts.push((placeholder, 1)),
}
}
let mut lines = Vec::new();
for (placeholder, count) in counts {
let present = text.matches(placeholder.as_str()).count();
if present > count {
return Err(PromptRenderError::invalid_request(format!(
"found more '{placeholder}' placeholders in the message text than media items"
))
.into());
}
lines.extend(std::iter::repeat_n(placeholder, count - present));
}
if lines.is_empty() {
return Ok(text);
}
if !text.is_empty() {
lines.push(text);
}
Ok(lines.join("\n"))
}
const KNOWN_CONTENT_PART_FIELDS: &[&str] = &[
"type",
"text",
"refusal",
"image_url",
"video_url",
"audio_url",
"input_audio",
"uuid",
"audio_embeds",
"closed",
"data",
"file",
"image_embeds",
"image_pil",
"name",
"prompt_cache_breakpoint",
"thinking",
"video_embeds",
];
fn openai_part(part: serde_json::Value) -> Option<serde_json::Value> {
let mut wrapped = match content_part(&part) {
ContentPart::Text(text) => json!({"type": "text", "text": text?}),
ContentPart::Media(modality) => json!({"type": modality}),
ContentPart::Custom => return Some(custom_marker(&part)),
ContentPart::Other => return Some(part),
};
if let serde_json::Value::Object(fields) = part {
for (key, value) in fields {
if !KNOWN_CONTENT_PART_FIELDS.contains(&key.as_str()) {
wrapped[key.as_str()] = value;
}
}
}
Some(wrapped)
}
fn wrap_msg_content_in_parts(messages: &mut serde_json::Value) {
let Some(msgs) = messages.as_array_mut() else {
return;
};
for msg in msgs {
let is_tool = msg.get("role").and_then(serde_json::Value::as_str) == Some("tool");
let parts: Vec<serde_json::Value> = match take_message_content(msg) {
serde_json::Value::String(text) => {
let mut part = json!({"type": "text"});
part["text"] = text.into();
vec![part]
}
serde_json::Value::Array(parts) => parts.into_iter().filter_map(openai_part).collect(),
_ => Vec::new(),
};
let Some(msg) = msg.as_object_mut() else {
continue;
};
let all_text = parts.iter().all(|part| part["type"] == "text");
let content = if is_tool && all_text {
let texts: Vec<&str> = parts.iter().filter_map(|p| p["text"].as_str()).collect();
serde_json::Value::String(texts.join("\n"))
} else {
serde_json::Value::Array(parts)
};
msg.insert("content".to_string(), content);
}
}
fn normalize_tool_calls_arguments_in_messages(messages: &mut serde_json::Value) {
let Some(msgs) = messages.as_array_mut() else {
return;
};
for msg in msgs.iter_mut() {
if let Some(tool_calls) = msg.get_mut("tool_calls").and_then(|v| v.as_array_mut()) {
for tc in tool_calls {
if let Some(function) = tc.get_mut("function").and_then(|v| v.as_object_mut())
&& let Some(args) = function.get_mut("arguments")
&& let Some(s) = args.as_str()
&& let Ok(parsed) = serde_json::from_str(s)
{
*args = parsed;
}
}
}
}
}
fn normalize_function_call_arguments_in_messages(messages: &mut serde_json::Value) {
let Some(msgs) = messages.as_array_mut() else {
return;
};
for msg in msgs.iter_mut() {
if let Some(function_call) = msg.get_mut("function_call").and_then(|v| v.as_object_mut())
&& let Some(args) = function_call.get_mut("arguments")
&& let Some(s) = args.as_str()
&& let Ok(parsed) = serde_json::from_str(s)
{
*args = parsed;
}
}
}
fn inject_reasoning_content_into_messages(messages: &mut serde_json::Value) {
let Some(msgs) = messages.as_array_mut() else {
return;
};
for msg in msgs.iter_mut() {
if msg.get("role").and_then(|r| r.as_str()) != Some("assistant") {
continue;
}
let reasoning = match msg.get("reasoning_content") {
Some(serde_json::Value::String(s)) if !s.is_empty() => {
format!("<think>{}</think>", s)
}
Some(serde_json::Value::Array(segments)) => {
let mut result = String::new();
for seg in segments {
if let Some(s) = seg.as_str()
&& !s.is_empty()
{
result.push_str("<think>");
result.push_str(s);
result.push_str("</think>");
}
}
if result.is_empty() {
continue;
}
result
}
_ => continue,
};
match msg.get("content") {
Some(serde_json::Value::String(s)) if !s.is_empty() => {
msg["content"] = serde_json::Value::String(format!("{}{}", reasoning, s));
}
None | Some(serde_json::Value::Null) | Some(serde_json::Value::String(_)) => {
msg["content"] = serde_json::Value::String(reasoning);
}
Some(serde_json::Value::Array(_)) => {
let think_part = serde_json::json!({
"type": "text",
"text": reasoning
});
if let Some(arr) = msg.get_mut("content").and_then(|v| v.as_array_mut()) {
arr.insert(0, think_part);
}
}
_ => continue,
}
if let Some(obj) = msg.as_object_mut() {
obj.remove("reasoning_content");
}
}
}
fn join_reasoning_content_segments_in_messages(messages: &mut serde_json::Value) {
let Some(msgs) = messages.as_array_mut() else {
return;
};
for msg in msgs.iter_mut() {
if msg.get("role").and_then(|r| r.as_str()) != Some("assistant") {
continue;
}
if let Some(reasoning) = msg.get_mut("reasoning_content")
&& let Some(segments) = reasoning.as_array()
{
let joined = segments
.iter()
.filter_map(|s| s.as_str())
.filter(|s| !s.is_empty())
.collect::<Vec<_>>()
.join("\n");
*reasoning = joined.into();
}
}
}
impl OAIChatLikeRequest for dynamo_protocols::types::CreateChatCompletionRequest {
fn model(&self) -> String {
self.model.clone()
}
fn messages(&self) -> Value {
let messages_json = self.function_template_messages();
Value::from_serialize(&messages_json)
}
fn typed_messages(&self) -> Option<&[dynamo_protocols::types::ChatCompletionRequestMessage]> {
Some(self.messages.as_slice())
}
fn tools(&self) -> Option<Value> {
let tools = self.tools.as_ref()?;
may_be_fix_tool_schema(
serde_json::to_value(
tools
.iter()
.map(|tool| tool.function_template_tool())
.collect::<Vec<_>>(),
)
.unwrap(),
)
}
fn raw_tools(&self) -> Option<Value> {
self.tools.as_ref().map(Value::from_serialize)
}
fn tool_choice(&self) -> Option<Value> {
self.tool_choice
.as_ref()
.map(|choice| Value::from_serialize(choice.function_template_choice()))
}
fn response_format(&self) -> Option<Value> {
self.response_format.as_ref().map(Value::from_serialize)
}
fn reasoning_effort(&self) -> Option<Value> {
self.reasoning_effort.as_ref().map(Value::from_serialize)
}
fn should_add_generation_prompt(&self) -> bool {
true
}
fn extract_text(&self) -> Option<TextInput> {
Some(TextInput::Single(String::new()))
}
fn mm_processor_kwargs(&self) -> Option<&serde_json::Value> {
self.mm_processor_kwargs.as_ref()
}
}
fn merge_message_content(
target: serde_json::Value,
source: serde_json::Value,
) -> serde_json::Value {
use serde_json::Value;
let text_part = |text: String| json!({"type": "text", "text": text});
match (target, source) {
(Value::String(mut target), Value::String(source)) => {
if !target.is_empty() && !source.is_empty() {
target.push_str("\n\n");
}
target.push_str(&source);
Value::String(target)
}
(Value::Array(mut target), Value::Array(source)) => {
target.extend(source);
Value::Array(target)
}
(Value::Array(mut target), Value::String(source)) => {
if !source.is_empty() {
target.push(text_part(source));
}
Value::Array(target)
}
(Value::String(target), Value::Array(source)) => {
let mut parts = Vec::with_capacity(source.len() + 1);
if !target.is_empty() {
parts.push(text_part(target));
}
parts.extend(source);
Value::Array(parts)
}
(Value::Null, source) => source,
(target, _) => target,
}
}
fn append_message_content(target: &mut serde_json::Value, source: serde_json::Value) {
let Some(target) = target.as_object_mut() else {
return;
};
let merged = merge_message_content(
target.remove("content").unwrap_or(serde_json::Value::Null),
source,
);
target.insert("content".to_string(), merged);
}
fn take_message_content(message: &mut serde_json::Value) -> serde_json::Value {
message
.get_mut("content")
.map(serde_json::Value::take)
.unwrap_or(serde_json::Value::Null)
}
fn normalize_system_messages(
messages: &mut serde_json::Value,
rules: SystemNormalization,
) -> Vec<usize> {
let mut merged_turns = Vec::new();
let serde_json::Value::Array(list) = messages else {
return merged_turns;
};
let role_is =
|m: &serde_json::Value, r: &str| m.get("role").and_then(|v| v.as_str()) == Some(r);
if rules.demote_nonleading_system {
let leading = list.iter().take_while(|m| role_is(m, "system")).count();
if leading > 1 {
for mut trailing in list.drain(1..leading).collect::<Vec<_>>() {
let content = take_message_content(&mut trailing);
append_message_content(&mut list[0], content);
}
merged_turns.push(0);
}
let leading = list.iter().take_while(|m| role_is(m, "system")).count();
for m in list.iter_mut().skip(leading) {
if role_is(m, "system")
&& let Some(m) = m.as_object_mut()
{
m.insert("role".to_string(), json!("user"));
}
}
}
if rules.coalesce_consecutive_users {
let mut coalesced: Vec<serde_json::Value> = Vec::with_capacity(list.len());
for mut m in list.drain(..) {
if role_is(&m, "user") && coalesced.last().is_some_and(|p| role_is(p, "user")) {
let content = take_message_content(&mut m);
append_message_content(coalesced.last_mut().unwrap(), content);
let index = coalesced.len() - 1;
if merged_turns.last() != Some(&index) {
merged_turns.push(index);
}
} else {
coalesced.push(m);
}
}
*list = coalesced;
}
merged_turns
}
impl HfTokenizerConfigJsonFormatter {
fn template_handles_native_tools(&self, has_tools: bool) -> bool {
if has_tools {
self.tool_use_template_handles_native_tools
} else {
self.default_template_handles_native_tools
}
}
fn render_with_token_policy(
&self,
req: &dyn OAIChatLikeRequest,
protect_literals: bool,
) -> Result<crate::RenderedPrompt> {
let mixins = Value::from_dyn_object(self.mixins.clone());
let exclude = self.exclude_tools_when_tool_choice_none
&& req.tool_choice().as_ref().and_then(Value::as_str) == Some("none");
let flat_tools = if exclude { None } else { req.tools() };
let has_tools = flat_tools
.as_ref()
.and_then(|tools| tools.len())
.is_some_and(|len| len > 0);
let tools = if !exclude && self.template_handles_native_tools(has_tools) {
req.template_tools().or(flat_tools)
} else {
flat_tools
};
let add_generation_prompt = req.should_add_generation_prompt();
tracing::trace!(
"Rendering prompt with tools: {:?}, add_generation_prompt: {}",
has_tools,
add_generation_prompt
);
let (
template_name,
template_handles_tool_calls_args_string,
template_handles_reasoning,
template_requires_reasoning_string,
system_normalization,
native_text_array_roles,
) = if has_tools {
(
"tool_use",
self.tool_use_template_handles_tool_calls_arguments_string,
self.tool_use_template_handles_reasoning,
self.tool_use_template_requires_reasoning_string,
self.tool_use_system_normalization,
&self.tool_use_native_text_array_roles,
)
} else {
(
"default",
self.default_template_handles_tool_calls_arguments_string,
self.default_template_handles_reasoning,
self.default_template_requires_reasoning_string,
self.default_system_normalization,
&self.default_native_text_array_roles,
)
};
let mut messages_for_template = crate::messages_to_json(req)?;
crate::reject_unsupported_partial_assistant(&messages_for_template)?;
crate::reject_unsupported_message_tools(&messages_for_template, &[])?;
let mut normalized_turns = if system_normalization.is_required() {
normalize_system_messages(&mut messages_for_template, system_normalization)
} else {
Vec::new()
};
match &self.content_format {
ChatTemplateContentFormat::Auto => {
if !template_handles_reasoning
&& let Some(messages) = messages_for_template.as_array()
{
normalized_turns.extend(messages.iter().enumerate().filter_map(
|(index, message)| {
if message.get("role").and_then(serde_json::Value::as_str)
!= Some("assistant")
{
return None;
}
let has_reasoning = match message.get("reasoning_content") {
Some(serde_json::Value::String(text)) => !text.is_empty(),
Some(serde_json::Value::Array(segments)) => {
segments.iter().any(|segment| {
segment.as_str().is_some_and(|text| !text.is_empty())
})
}
_ => false,
};
has_reasoning.then_some(index)
},
));
}
messages_for_template = may_be_fix_msg_content(
messages_for_template,
self.requires_content_arrays,
native_text_array_roles,
&normalized_turns,
self.image_placeholder_template,
);
}
ChatTemplateContentFormat::String(media_placeholders) => {
let use_audio_in_video = req
.mm_processor_kwargs()
.and_then(|kwargs| kwargs.get("use_audio_in_video"))
.is_some_and(|flag| Value::from_serialize(flag).is_true());
flatten_msg_content_to_string(
&mut messages_for_template,
media_placeholders,
use_audio_in_video,
)?;
}
ChatTemplateContentFormat::OpenAi => {
wrap_msg_content_in_parts(&mut messages_for_template);
}
}
if !template_handles_tool_calls_args_string {
normalize_tool_calls_arguments_in_messages(&mut messages_for_template);
}
normalize_function_call_arguments_in_messages(&mut messages_for_template);
if !template_handles_reasoning {
inject_reasoning_content_into_messages(&mut messages_for_template);
} else if template_requires_reasoning_string {
join_reasoning_content_segments_in_messages(&mut messages_for_template);
}
let tmpl = self.env.get_template(template_name)?;
let mut literals = HarmonyLiterals::default();
let mut tools = tools;
let mut response_format = req
.response_format()
.as_ref()
.map(crate::python::template_json)
.transpose()?;
let mut placeholder = "__DYNAMO_HARMONY_LITERAL_".to_string();
if protect_literals
&& tmpl.source().contains("<|start|>")
&& tmpl.source().contains("<|message|>")
{
let mut tool_json = tools
.as_ref()
.map(crate::python::template_json)
.transpose()?;
let collision_source = format!(
"{messages_for_template}{tool_json:?}{response_format:?}{:?}{}",
req.chat_template_args(),
tmpl.source()
);
while collision_source.contains(&placeholder) {
placeholder.push('_');
}
shield_harmony_literals(&mut messages_for_template, &placeholder, &mut literals);
if let Some(value) = &mut tool_json {
shield_harmony_literals(value, &placeholder, &mut literals);
tools = Some(crate::python::template_value(value));
}
if let Some(value) = &mut response_format {
shield_harmony_literals(value, &placeholder, &mut literals);
}
}
let ctx = context! {
messages => crate::python::template_value(&messages_for_template),
tools => tools.as_ref().map(crate::python::template_json).transpose()?.as_ref().map(crate::python::template_value),
response_format => response_format.as_ref()
.map(crate::python::template_value)
.unwrap_or(Value::UNDEFINED),
bos_token => self.config.bos_tok(),
eos_token => self.config.eos_tok(),
unk_token => self.config.unk_tok(),
add_generation_prompt => add_generation_prompt,
..mixins
};
let ctx = if let Some(args) = req.chat_template_args() {
let extra = crate::python::template_value(&serde_json::to_value(args)?);
context! { ..ctx, ..extra }
} else {
ctx
};
let rendered = tmpl.render(&ctx)?;
if literals.values.is_empty() {
return Ok(crate::RenderedPrompt::text(rendered));
}
let mut segments = Vec::new();
let mut remaining = rendered.as_str();
while let Some(start) = remaining.find(&placeholder) {
segments.push(crate::RenderedSegment::new(&remaining[..start], true));
let after = &remaining[start + placeholder.len()..];
let (index, suffix) = after
.split_once('_')
.ok_or_else(|| anyhow::anyhow!("damaged Harmony literal placeholder"))?;
let index: usize = index.parse()?;
segments.push(crate::RenderedSegment::new(
literals
.values
.get(index)
.ok_or_else(|| anyhow::anyhow!("unknown Harmony literal placeholder"))?,
false,
));
remaining = suffix;
}
segments.push(crate::RenderedSegment::new(remaining, true));
Ok(crate::RenderedPrompt::segmented(segments))
}
}
#[derive(Default)]
struct HarmonyLiterals {
values: Vec<String>,
indices: HashMap<String, usize>,
}
impl HarmonyLiterals {
fn index(&mut self, text: &str) -> usize {
if let Some(index) = self.indices.get(text) {
return *index;
}
let index = self.values.len();
self.values.push(text.to_owned());
self.indices.insert(text.to_owned(), index);
index
}
}
fn shield_harmony_literals(
value: &mut serde_json::Value,
prefix: &str,
literals: &mut HarmonyLiterals,
) {
match value {
serde_json::Value::String(text) => shield_harmony_text(text, prefix, literals),
serde_json::Value::Array(values) => {
for value in values {
shield_harmony_literals(value, prefix, literals);
}
}
serde_json::Value::Object(values) => {
if values.keys().any(|key| key.contains("<|")) {
for (mut key, mut value) in std::mem::take(values) {
shield_harmony_text(&mut key, prefix, literals);
shield_harmony_literals(&mut value, prefix, literals);
values.insert(key, value);
}
} else {
for value in values.values_mut() {
shield_harmony_literals(value, prefix, literals);
}
}
}
_ => {}
}
}
fn shield_harmony_text(text: &mut String, prefix: &str, literals: &mut HarmonyLiterals) {
let mut protected = String::new();
let mut remaining = text.as_str();
while let Some(start) = remaining.find("<|") {
let Some(end) = remaining[start + 2..].find("|>") else {
break;
};
let end = start + 2 + end + 2;
let marker = &remaining[start..end];
if marker
.chars()
.any(|c| c == '"' || c == '\\' || c <= '\u{1f}')
{
protected.push_str(&remaining[..start + 2]);
remaining = &remaining[start + 2..];
continue;
}
protected.push_str(&remaining[..start]);
protected.push_str(&format!("{prefix}{}_", literals.index(marker)));
remaining = &remaining[end..];
}
if remaining.len() != text.len() {
protected.push_str(remaining);
*text = protected;
}
}
impl OAIPromptFormatter for HfTokenizerConfigJsonFormatter {
fn supports_native_tools(&self, request: &dyn OAIChatLikeRequest) -> bool {
let exclude = self.exclude_tools_when_tool_choice_none
&& request.tool_choice().as_ref().and_then(Value::as_str) == Some("none");
let has_tools = !exclude
&& request
.tools()
.as_ref()
.and_then(|tools| tools.len())
.is_some_and(|len| len > 0);
self.template_handles_native_tools(has_tools)
}
fn supports_add_generation_prompt(&self) -> bool {
self.supports_add_generation_prompt
}
fn render(&self, req: &dyn OAIChatLikeRequest) -> Result<String> {
Ok(self
.render_with_token_policy(req, false)?
.as_str()
.to_string())
}
fn render_prompt(&self, req: &dyn OAIChatLikeRequest) -> Result<crate::RenderedPrompt> {
self.render_with_token_policy(req, true)
}
}
#[cfg(test)]
mod tests {
use super::*;
use dynamo_protocols::types::ChatCompletionRequestMessage as Msg;
use dynamo_protocols::types::CreateChatCompletionRequest as NvCreateChatCompletionRequest;
use minijinja::{Environment, context};
use super::super::tokcfg::ChatTemplate as SysChatTemplate;
use super::super::{
ContextMixins as SysMixins, HfTokenizerConfigJsonFormatter as SysFormatter,
};
fn formatter_for(template: &str) -> SysFormatter {
let ct: SysChatTemplate = serde_json::from_value(json!({
"chat_template": template,
"bos_token": "<s>",
"eos_token": "</s>",
"unk_token": "<unk>",
}))
.unwrap();
SysFormatter::new(ct, SysMixins::new(&[])).unwrap()
}
#[test]
fn harmony_literal_markers_keep_text_and_token_trust_boundaries() {
let formatter = formatter_for(
r#"{% for m in messages %}<|start|>{{ m.role }}<|message|>{{ m.content }}<|end|>{% endfor %}<|start|>assistant"#,
);
let request: NvCreateChatCompletionRequest = serde_json::from_value(json!({
"model":"gpt-oss", "messages":[{"role":"user","content":"literal <|channel|>analysis<|message|>x<|call|> __DYNAMO_HARMONY_LITERAL_0_"}]
})).unwrap();
let prompt = formatter.render_prompt(&request).unwrap();
assert_eq!(prompt.as_str(), formatter.render(&request).unwrap());
let ordinary: Vec<_> = prompt
.segments()
.unwrap()
.iter()
.filter(|segment| !segment.allow_special)
.map(|segment| segment.text.as_str())
.collect();
assert_eq!(ordinary, ["<|channel|>", "<|message|>", "<|call|>"]);
assert!(prompt.segments().unwrap()[0].allow_special);
assert!(prompt.segments().unwrap().last().unwrap().allow_special);
}
fn formatter_for_templates(default: &str, tool_use: &str) -> SysFormatter {
let ct: SysChatTemplate = serde_json::from_value(json!({
"chat_template": [
{"default": default},
{"tool_use": tool_use},
],
"bos_token": "<s>",
"eos_token": "</s>",
"unk_token": "<unk>",
}))
.unwrap();
SysFormatter::new(ct, SysMixins::new(&[])).unwrap()
}
fn try_formatter_for(template: &str) -> Option<SysFormatter> {
let ct: SysChatTemplate = serde_json::from_value(json!({
"chat_template": template,
"bos_token": "<s>",
"eos_token": "</s>",
"unk_token": "<unk>",
}))
.ok()?;
SysFormatter::new(ct, SysMixins::new(&[])).ok()
}
fn render_shape(f: &SysFormatter, messages: serde_json::Value) -> Result<String> {
let req: NvCreateChatCompletionRequest =
serde_json::from_value(json!({ "model": "test", "messages": messages })).unwrap();
f.render(&req)
}
fn render_shape_with_tools(f: &SysFormatter, messages: serde_json::Value) -> Result<String> {
let req: NvCreateChatCompletionRequest = serde_json::from_value(json!({
"model": "test",
"messages": messages,
"tools": [{
"type": "function",
"function": {"name": "noop", "parameters": {}}
}]
}))
.unwrap();
f.render(&req)
}
struct RawMessagesRequest(Value);
impl OAIChatLikeRequest for RawMessagesRequest {
fn model(&self) -> String {
"test".to_string()
}
fn messages(&self) -> Value {
self.0.clone()
}
fn should_add_generation_prompt(&self) -> bool {
true
}
}
fn render_raw_shape(f: &SysFormatter, messages: serde_json::Value) -> Result<String> {
f.render(&RawMessagesRequest(Value::from_serialize(&messages)))
}
#[test]
fn content_normalization_preserves_unchanged_values() {
let messages = json!([
{"role": "user", "content": " 䏿–‡ <special>\n", "name": "user"},
{"role": "assistant", "content": null},
{"role": "user", "content": []},
{"role": "assistant", "tool_calls": []}
]);
assert_eq!(
may_be_fix_msg_content(messages.clone(), false, &[], &[], Some("")),
messages
);
}
const PERMISSIVE_TMPL: &str = concat!(
"{%- for m in messages -%}",
"<|im_start|>{{ m.role }}\n{{ m.content }}<|im_end|>\n",
"{%- endfor -%}"
);
#[test]
fn jinja_templates_reject_message_level_tools() {
let f = formatter_for(PERMISSIVE_TMPL);
let error = render_shape(
&f,
json!([
{"role": "system", "tools": [{"name": "lookup", "parameters": {"type": "object"}}]},
{"role": "user", "content": "hi"}
]),
)
.unwrap_err();
assert!(matches!(
error.downcast_ref::<crate::PromptRenderError>(),
Some(crate::PromptRenderError::InvalidRequest(message))
if message.contains("message-level `tools`")
));
let error = render_raw_shape(
&f,
json!([{
"role": "user",
"content": "hi",
"tools": [{"name": "lookup", "parameters": {"type": "object"}}]
}]),
)
.unwrap_err();
assert!(matches!(
error.downcast_ref::<crate::PromptRenderError>(),
Some(crate::PromptRenderError::InvalidRequest(message))
if message.contains("message-level `tools`")
));
let rendered = render_shape(
&f,
json!([
{"role": "system", "content": "You are helpful.", "tools": []},
{"role": "user", "content": "hi"}
]),
)
.unwrap();
assert!(rendered.contains("<|im_start|>system\nYou are helpful.<|im_end|>"));
}
#[test]
fn jinja_templates_reject_unsupported_partial_assistant() {
let f = formatter_for(PERMISSIVE_TMPL);
let error = render_shape(
&f,
json!([
{"role": "user", "content": "Continue"},
{"role": "assistant", "content": "prefix", "partial": true}
]),
)
.unwrap_err();
assert!(matches!(
error.downcast_ref::<crate::PromptRenderError>(),
Some(crate::PromptRenderError::InvalidRequest(message))
if message.contains("`partial: true` is not supported")
));
let rendered = render_shape(
&f,
json!([{"role": "assistant", "content": "ordinary", "partial": false}]),
)
.unwrap();
assert!(rendered.contains("ordinary"));
}
const STRICT_LEADING_TMPL: &str = concat!(
"{%- for m in messages -%}",
"{%- if m.role == 'system' and not loop.first -%}",
"{{ raise_exception('System message must be at the beginning.') }}",
"{%- endif -%}",
"<|im_start|>{{ m.role }}\n{{ m.content }}<|im_end|>\n",
"{%- endfor -%}"
);
const ALTERNATION_TMPL: &str = concat!(
"{%- set ns = namespace(prev='') -%}",
"{%- for m in messages -%}",
"{%- if m.role == 'user' and ns.prev == 'user' -%}",
"{{ raise_exception('Conversation roles must alternate.') }}",
"{%- endif -%}",
"<|im_start|>{{ m.role }}\n{{ m.content }}<|im_end|>\n",
"{%- set ns.prev = m.role -%}",
"{%- endfor -%}"
);
const STRICT_BOTH_TMPL: &str = concat!(
"{%- set ns = namespace(prev='') -%}",
"{%- for m in messages -%}",
"{%- if m.role == 'system' and not loop.first -%}",
"{{ raise_exception('System message must be at the beginning.') }}",
"{%- endif -%}",
"{%- if m.role == 'user' and ns.prev == 'user' -%}",
"{{ raise_exception('Conversation roles must alternate.') }}",
"{%- endif -%}",
"<|im_start|>{{ m.role }}\n{{ m.content }}<|im_end|>\n",
"{%- set ns.prev = m.role -%}",
"{%- endfor -%}"
);
const DEFAULT_NONE_GATED_TMPL: &str = concat!(
"{%- set strict = tools is not none -%}",
"{%- for m in messages -%}",
"{%- if strict and m.role == 'system' and not loop.first -%}",
"{{ raise_exception('System message must be at the beginning.') }}",
"{%- endif -%}",
"<|im_start|>{{ m.role }}\n{{ m.content }}<|im_end|>\n",
"{%- endfor -%}"
);
const TOOL_NONEMPTY_GATED_TMPL: &str = concat!(
"{%- set strict = tools|length > 0 -%}",
"{%- for m in messages -%}",
"{%- if strict and m.role == 'system' and not loop.first -%}",
"{{ raise_exception('System message must be at the beginning.') }}",
"{%- endif -%}",
"<|im_start|>{{ m.role }}\n{{ m.content }}<|im_end|>\n",
"{%- endfor -%}"
);
const STRICT_ARRAY_TMPL: &str = concat!(
"{%- for m in messages -%}",
"{%- if m.role == 'system' and not loop.first -%}",
"{{ raise_exception('System message must be at the beginning.') }}",
"{%- endif -%}",
"<|im_start|>{{ m.role }}\n",
"{%- if m.content is not string -%}",
"{%- for part in m.content -%}{{ part.text }}{%- endfor -%}",
"{%- endif -%}",
"<|im_end|>\n",
"{%- endfor -%}"
);
fn claude_shape() -> serde_json::Value {
json!([
{"role": "system", "content": "You are Claude Code."},
{"role": "user", "content": "hello"},
{"role": "system", "content": "mid-conversation reminder"},
])
}
fn all_restrictions() -> SystemNormalization {
SystemNormalization {
demote_nonleading_system: true,
coalesce_consecutive_users: true,
}
}
#[test]
fn permissive_template_is_not_flagged_and_renders_untouched() {
let f = formatter_for(PERMISSIVE_TMPL);
assert!(!f.default_system_normalization.is_required());
assert!(!f.tool_use_system_normalization.is_required());
let out = render_shape(&f, claude_shape()).unwrap();
assert!(out.contains("<|im_start|>system\nmid-conversation reminder<|im_end|>"));
}
#[test]
fn strict_leading_template_demotes_mid_system_but_keeps_user_turns_apart() {
let f = formatter_for(STRICT_LEADING_TMPL);
assert!(f.default_system_normalization.demote_nonleading_system);
assert!(!f.default_system_normalization.coalesce_consecutive_users);
let out = render_shape(&f, claude_shape()).unwrap();
assert_eq!(out.matches("<|im_start|>system").count(), 1);
assert!(out.contains("<|im_start|>user\nhello<|im_end|>"));
assert!(out.contains("<|im_start|>user\nmid-conversation reminder<|im_end|>"));
}
#[test]
fn alternation_template_coalesces_users_but_keeps_mid_system() {
let f = formatter_for(ALTERNATION_TMPL);
assert!(f.default_system_normalization.coalesce_consecutive_users);
assert!(!f.default_system_normalization.demote_nonleading_system);
let out = render_shape(&f, claude_shape()).unwrap();
assert!(out.contains("<|im_start|>system\nmid-conversation reminder<|im_end|>"));
let out = render_shape(
&f,
json!([
{"role": "system", "content": "s"},
{"role": "user", "content": "hello"},
{"role": "user", "content": "again"},
]),
)
.unwrap();
assert_eq!(out.matches("<|im_start|>user").count(), 1);
assert!(out.contains("<|im_start|>user\nhello\n\nagain<|im_end|>"));
}
#[test]
fn strict_both_template_demotes_then_coalesces() {
let f = formatter_for(STRICT_BOTH_TMPL);
assert!(f.default_system_normalization.demote_nonleading_system);
assert!(f.default_system_normalization.coalesce_consecutive_users);
let out = render_shape(&f, claude_shape()).unwrap();
assert_eq!(out.matches("<|im_start|>system").count(), 1);
assert!(out.contains("<|im_start|>user\nhello\n\nmid-conversation reminder<|im_end|>"));
}
#[test]
fn system_normalization_flag_is_selected_per_template() {
let f = formatter_for_templates(PERMISSIVE_TMPL, STRICT_LEADING_TMPL);
assert!(!f.default_system_normalization.is_required());
assert!(f.tool_use_system_normalization.is_required());
let no_tools = render_shape(&f, claude_shape()).unwrap();
assert!(no_tools.contains("<|im_start|>system\nmid-conversation reminder<|im_end|>"));
let with_tools = render_shape_with_tools(&f, claude_shape()).unwrap();
assert_eq!(with_tools.matches("<|im_start|>system").count(), 1);
assert!(with_tools.contains("<|im_start|>user\nmid-conversation reminder<|im_end|>"));
let f = formatter_for_templates(STRICT_LEADING_TMPL, PERMISSIVE_TMPL);
assert!(f.default_system_normalization.is_required());
assert!(!f.tool_use_system_normalization.is_required());
let with_tools = render_shape_with_tools(&f, claude_shape()).unwrap();
assert!(with_tools.contains("<|im_start|>system\nmid-conversation reminder<|im_end|>"));
}
#[test]
fn system_normalization_probe_uses_runtime_tools_shape() {
let f = formatter_for_templates(DEFAULT_NONE_GATED_TMPL, TOOL_NONEMPTY_GATED_TMPL);
assert!(!f.default_system_normalization.is_required());
assert!(f.tool_use_system_normalization.is_required());
let no_tools = render_shape(&f, claude_shape()).unwrap();
assert!(no_tools.contains("<|im_start|>system\nmid-conversation reminder<|im_end|>"));
let with_tools = render_shape_with_tools(&f, claude_shape()).unwrap();
assert_eq!(with_tools.matches("<|im_start|>system").count(), 1);
assert!(with_tools.contains("<|im_start|>user\nmid-conversation reminder<|im_end|>"));
}
#[test]
fn system_normalization_precedes_required_content_array_conversion() {
let f = formatter_for(STRICT_ARRAY_TMPL);
assert!(f.requires_content_arrays);
assert!(f.default_system_normalization.demote_nonleading_system);
let out = render_shape(
&f,
json!([
{"role": "system", "content": "A"},
{"role": "system", "content": "B"},
{"role": "user", "content": "hello"},
]),
)
.unwrap();
assert!(out.contains("A\n\nB"));
}
#[test]
fn normalize_preserves_multimodal_user_content_and_fields() {
let mut m = json!([
{
"role": "user",
"name": "kept",
"content": [
{"type": "text", "text": "look"},
{"type": "image"},
],
},
{"role": "system", "content": "remember"},
]);
normalize_system_messages(&mut m, all_restrictions());
assert_eq!(
m,
json!([{
"role": "user",
"name": "kept",
"content": [
{"type": "text", "text": "look"},
{"type": "image"},
{"type": "text", "text": "remember"},
],
}])
);
}
#[test]
fn coalesce_preserves_multimodal_content_of_the_merged_turn() {
let mut m = json!([
{"role": "user", "content": "look"},
{"role": "user", "content": [
{"type": "text", "text": "at this"},
{"type": "image_url", "image_url": {"url": "http://img"}},
]},
]);
normalize_system_messages(&mut m, all_restrictions());
assert_eq!(
m,
json!([{
"role": "user",
"content": [
{"type": "text", "text": "look"},
{"type": "text", "text": "at this"},
{"type": "image_url", "image_url": {"url": "http://img"}},
],
}])
);
}
#[test]
fn normalize_merges_leading_run_and_coalesces() {
let mut m = json!([
{"role": "system", "content": "A"},
{"role": "system", "content": "B"},
{"role": "user", "content": "hi"},
{"role": "system", "content": "reminder"},
]);
normalize_system_messages(&mut m, all_restrictions());
assert_eq!(
m,
json!([
{"role": "system", "content": "A\n\nB"},
{"role": "user", "content": "hi\n\nreminder"},
])
);
}
#[test]
fn each_restriction_applies_only_its_own_rewrite() {
let shape = json!([
{"role": "system", "content": "A"},
{"role": "system", "content": "B"},
{"role": "user", "content": "hi"},
{"role": "system", "content": "reminder"},
]);
let mut demote_only = shape.clone();
normalize_system_messages(
&mut demote_only,
SystemNormalization {
demote_nonleading_system: true,
coalesce_consecutive_users: false,
},
);
assert_eq!(
demote_only,
json!([
{"role": "system", "content": "A\n\nB"},
{"role": "user", "content": "hi"},
{"role": "user", "content": "reminder"},
])
);
let mut coalesce_only = shape.clone();
normalize_system_messages(
&mut coalesce_only,
SystemNormalization {
demote_nonleading_system: false,
coalesce_consecutive_users: true,
},
);
assert_eq!(coalesce_only, shape);
}
#[test]
fn normalize_preserves_array_system_content() {
let mut m = json!([
{"role": "user", "content": "hi"},
{"role": "system", "content": [{"type": "text", "text": "one"},
{"type": "text", "text": "two"}]},
]);
normalize_system_messages(&mut m, all_restrictions());
assert_eq!(
m,
json!([{"role": "user", "content": [
{"type": "text", "text": "hi"},
{"type": "text", "text": "one"},
{"type": "text", "text": "two"},
]}])
);
}
#[test]
#[ignore]
fn adaptive_system_corpus_audit() {
let dir =
std::env::var("TEMPLATE_CORPUS").expect("set TEMPLATE_CORPUS to the templates dir");
let manifest: serde_json::Value =
serde_json::from_str(&std::fs::read_to_string(format!("{dir}/manifest.json")).unwrap())
.unwrap();
let sys = |c: &str| json!({"role": "system", "content": c});
let usr = |c: &str| json!({"role": "user", "content": c});
let asst = |c: &str| json!({"role": "assistant", "content": c});
let shapes: Vec<(&str, serde_json::Value)> = vec![
("turn1", json!([sys("s"), usr("u"), sys("mid")])),
(
"multiturn",
json!([sys("s"), usr("u"), sys("mid"), asst("a"), usr("u2")]),
),
(
"mid_after_asst",
json!([sys("s"), usr("u"), asst("a"), sys("mid"), usr("u2")]),
),
("double_leading", json!([sys("s0"), sys("s1"), usr("u")])),
("consec_user", json!([sys("s"), usr("u0"), usr("u1")])),
(
"tail_reminder",
json!([
sys("s"),
usr("u"),
asst("a"),
usr("u2"),
sys("mid"),
usr("u3")
]),
),
("leading_only_baseline", json!([sys("s"), usr("u")])),
];
let mut total = 0usize;
let mut flagged = 0usize;
let mut demote_only = 0usize;
let mut coalesce = 0usize;
let mut failures: Vec<String> = Vec::new();
for (file, meta) in manifest.as_object().unwrap() {
let tmpl = std::fs::read_to_string(format!("{dir}/{file}.jinja")).unwrap();
let model = meta["model"].as_str().unwrap_or(file);
let f = match try_formatter_for(&tmpl) {
Some(f) => f,
None => {
eprintln!("[skip-compile] {model}");
continue;
}
};
if render_shape(&f, json!([sys("s"), usr("u")])).is_err() {
eprintln!("[skip-baseline] {model}");
continue;
}
total += 1;
let rules = f.default_system_normalization;
let flag = rules.is_required();
if flag {
flagged += 1;
}
if rules.demote_nonleading_system {
demote_only += usize::from(!rules.coalesce_consecutive_users);
}
if rules.coalesce_consecutive_users {
coalesce += 1;
}
for (name, shape) in &shapes {
if render_shape(&f, shape.clone()).is_err() {
failures.push(format!("{model} | shape={name} | flag={flag}"));
}
}
if flag {
eprintln!(
"[ok] demote={} coalesce={} {model}",
rules.demote_nonleading_system, rules.coalesce_consecutive_users
);
}
}
eprintln!(
"\naudited {total} templates ({flagged} flagged: {demote_only} demote-only, \
{coalesce} coalescing); {} shape failures",
failures.len()
);
for f in &failures {
eprintln!(" FAIL {f}");
}
assert!(
failures.is_empty(),
"{} template/shape combinations did not render (probe insufficient or normalization insufficient)",
failures.len()
);
}
#[test]
fn test_render_long_conversation_does_not_overflow_stack() {
let handle = std::thread::Builder::new()
.stack_size(2 * 1024 * 1024)
.spawn(|| {
let template_string = concat!(
"{%- set ns = namespace(items=[]) -%}",
"{%- for m in messages -%}",
"{%- set ns.items = ns.items + [m] -%}",
"{%- endfor -%}",
"COUNT={{ ns.items | length }}"
);
let chat_template: ChatTemplate =
serde_json::from_value(serde_json::json!({ "chat_template": template_string }))
.unwrap();
let formatter =
HfTokenizerConfigJsonFormatter::new(chat_template, ContextMixins::new(&[]))
.unwrap();
let n = 3000;
let messages: Vec<serde_json::Value> = (0..n)
.map(|i| serde_json::json!({"role": "user", "content": format!("turn {i}")}))
.collect();
let request: NvCreateChatCompletionRequest =
serde_json::from_value(serde_json::json!({
"model": "test",
"messages": messages,
}))
.unwrap();
let rendered = formatter.render(&request).unwrap();
assert_eq!(rendered.trim(), format!("COUNT={n}"));
})
.unwrap();
handle.join().unwrap();
}
#[test]
#[ignore]
fn dump_gptoss_tool_prompt() {
use super::tokcfg::ChatTemplate;
use super::{ContextMixins, HfTokenizerConfigJsonFormatter};
let path = std::env::var("GPTOSS_CHAT_TEMPLATE").expect(
"set GPTOSS_CHAT_TEMPLATE to the tokenizer_config.json, chat_template.jinja, or model dir path",
);
let input_path = std::path::Path::new(&path);
let file_path = if input_path.is_dir() {
input_path.join("tokenizer_config.json")
} else {
input_path.to_path_buf()
};
let raw = std::fs::read_to_string(&file_path).expect("read chat template file");
let template_string: String = match serde_json::from_str::<serde_json::Value>(&raw) {
Ok(v) if v.get("chat_template").is_some() => v["chat_template"]
.as_str()
.expect("chat_template field must be a string")
.to_string(),
_ => {
let sibling = std::path::Path::new(&path)
.parent()
.map(|d| d.join("chat_template.jinja"));
match sibling {
Some(p) if p.exists() => {
eprintln!(
"[info] {path} had no chat_template field; using {}",
p.display()
);
std::fs::read_to_string(&p).expect("read sibling chat_template.jinja")
}
_ => raw,
}
}
};
assert!(
template_string.contains("{%") || template_string.contains("{{"),
"resolved template has no Jinja tags — GPTOSS_CHAT_TEMPLATE ({path}) is probably \
tokenizer_config.json with no chat_template field and no sibling chat_template.jinja. \
Point it at the chat_template.jinja file."
);
let chat_template: ChatTemplate =
serde_json::from_value(serde_json::json!({ "chat_template": template_string }))
.unwrap();
let formatter =
HfTokenizerConfigJsonFormatter::new(chat_template, ContextMixins::new(&[])).unwrap();
let request: NvCreateChatCompletionRequest = serde_json::from_str(
r#"{
"model": "openai/gpt-oss-120b",
"messages": [{"role":"user","content":"Search the repo for the string \"countHook\"."}],
"tools": [
{"type":"function","function":{"name":"grep","description":"search files","parameters":{"type":"object","properties":{"pattern":{"type":"string"},"path":{"type":"string"}},"required":["pattern"]}}},
{"type":"function","function":{"name":"read","description":"read a file","parameters":{"type":"object","properties":{"filePath":{"type":"string"}},"required":["filePath"]}}}
]
}"#,
)
.unwrap();
let rendered = formatter.render(&request).unwrap();
eprintln!("================ RENDERED gpt-oss PROMPT (tools declared) ================");
eprintln!("{rendered}");
eprintln!("================ END RENDERED PROMPT ================");
eprintln!("[diagnostics] does the rendered prompt contain…");
for needle in [
"commentary",
"Calls to these tools",
"functions",
"# Tools",
"<|channel|>",
"constrain",
"analysis",
] {
eprintln!(
" {:>22}: {}",
format!("{needle:?}"),
rendered.contains(needle)
);
}
}
#[test]
fn test_convert_media_url_to_placeholder_single_type() {
let mut content_array = vec![
serde_json::json!({"type": "text", "text": "Check this image:"}),
serde_json::json!({"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}),
serde_json::json!({"type": "text", "text": "What do you see?"}),
];
let conversions = &[("image_url", "image")];
convert_media_url_to_placeholder(&mut content_array, conversions);
assert_eq!(content_array.len(), 3);
assert_eq!(content_array[0]["type"], "text");
assert_eq!(content_array[0]["text"], "Check this image:");
assert_eq!(content_array[1]["type"], "image");
assert!(content_array[1].get("image_url").is_none());
assert_eq!(content_array[2]["type"], "text");
assert_eq!(content_array[2]["text"], "What do you see?");
}
#[test]
fn test_convert_media_url_to_placeholder_multiple_same_type() {
let mut content_array = vec![
serde_json::json!({"type": "image_url", "image_url": {"url": "https://example.com/image1.jpg"}}),
serde_json::json!({"type": "text", "text": "vs"}),
serde_json::json!({"type": "image_url", "image_url": {"url": "https://example.com/image2.jpg"}}),
];
let conversions = &[("image_url", "image")];
convert_media_url_to_placeholder(&mut content_array, conversions);
assert_eq!(content_array.len(), 3);
assert_eq!(content_array[0]["type"], "image");
assert_eq!(content_array[1]["type"], "text");
assert_eq!(content_array[2]["type"], "image");
}
#[test]
fn test_convert_media_url_to_placeholder_selective_conversion() {
let mut content_array = vec![
serde_json::json!({"type": "audio_url", "audio_url": {"url": "https://example.com/audio.mp3"}}),
serde_json::json!({"type": "video_url", "video_url": {"url": "https://example.com/video.mp4"}}),
serde_json::json!({"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}),
];
let conversions = &[("image_url", "image")];
convert_media_url_to_placeholder(&mut content_array, conversions);
assert_eq!(content_array.len(), 3);
assert_eq!(content_array[0]["type"], "audio_url");
assert!(content_array[0].get("audio_url").is_some());
assert_eq!(content_array[1]["type"], "video_url");
assert!(content_array[1].get("video_url").is_some());
assert_eq!(content_array[2]["type"], "image");
assert!(content_array[2].get("image_url").is_none());
}
#[test]
fn test_convert_media_url_to_placeholder_multiple_types() {
let mut content_array = vec![
serde_json::json!({"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}),
serde_json::json!({"type": "text", "text": "and listen to"}),
serde_json::json!({"type": "audio_url", "audio_url": {"url": "https://example.com/audio.mp3"}}),
serde_json::json!({"type": "text", "text": "and watch"}),
serde_json::json!({"type": "video_url", "video_url": {"url": "https://example.com/video.mp4"}}),
];
let conversions = &[
("image_url", "image"),
("audio_url", "audio"),
("video_url", "video"),
];
convert_media_url_to_placeholder(&mut content_array, conversions);
assert_eq!(content_array.len(), 5);
assert_eq!(content_array[0]["type"], "image");
assert!(content_array[0].get("image_url").is_none());
assert_eq!(content_array[1]["type"], "text");
assert_eq!(content_array[2]["type"], "audio");
assert!(content_array[2].get("audio_url").is_none());
assert_eq!(content_array[3]["type"], "text");
assert_eq!(content_array[4]["type"], "video");
assert!(content_array[4].get("video_url").is_none());
}
#[test]
fn test_convert_media_url_to_placeholder_no_conversions() {
let mut content_array = vec![
serde_json::json!({"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}),
serde_json::json!({"type": "text", "text": "hello"}),
];
let conversions: &[(&str, &str)] = &[];
convert_media_url_to_placeholder(&mut content_array, conversions);
assert_eq!(content_array.len(), 2);
assert_eq!(content_array[0]["type"], "image_url");
assert!(content_array[0].get("image_url").is_some());
assert_eq!(content_array[1]["type"], "text");
}
#[test]
fn test_default_media_type_conversions_only_converts_image_url() {
let mut content_array = vec![
serde_json::json!({"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}),
serde_json::json!({"type": "video_url", "video_url": {"url": "https://example.com/video.mp4"}}),
serde_json::json!({"type": "audio_url", "audio_url": {"url": "https://example.com/audio.mp3"}}),
serde_json::json!({"type": "text", "text": "hello"}),
];
convert_media_url_to_placeholder(&mut content_array, DEFAULT_MEDIA_TYPE_CONVERSIONS);
assert_eq!(content_array.len(), 4);
assert_eq!(content_array[0]["type"], "image");
assert!(content_array[0].get("image_url").is_none());
assert_eq!(content_array[1]["type"], "video");
assert!(content_array[1].get("video_url").is_none());
assert_eq!(content_array[2]["type"], "audio");
assert!(content_array[2].get("audio_url").is_none());
assert_eq!(content_array[3]["type"], "text");
assert_eq!(content_array[3]["text"], "hello");
}
#[test]
fn test_may_be_fix_tool_schema_missing_type_and_properties() {
let json_str = r#"{
"model": "gpt-4o",
"messages": [],
"tools": [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather in a given location",
"parameters": {},
"strict": null
}
}
]
}"#;
let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
let tools = serde_json::to_value(request.tools()).unwrap();
assert!(tools[0]["function"]["parameters"]["type"] == "object");
assert!(
tools[0]["function"]["parameters"]["properties"]
== serde_json::Value::Object(Default::default())
);
}
#[test]
fn test_may_be_fix_tool_schema_missing_type() {
let json_str = r#"{
"model": "gpt-4o",
"messages": [],
"tools": [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather in a given location",
"parameters": {
"properties": {
"location": {
"type": "string",
"description": "City and state, e.g., 'San Francisco, CA'"
}
}
},
"strict": null
}
}
]
}"#;
let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
let tools = serde_json::to_value(request.tools()).unwrap();
assert_eq!(tools[0]["function"]["parameters"]["type"], "object");
let mut expected_properties = serde_json::Map::new();
let mut location = serde_json::Map::new();
location.insert(
"type".to_string(),
serde_json::Value::String("string".to_string()),
);
location.insert(
"description".to_string(),
serde_json::Value::String("City and state, e.g., 'San Francisco, CA'".to_string()),
);
expected_properties.insert("location".to_string(), serde_json::Value::Object(location));
assert_eq!(
tools[0]["function"]["parameters"]["properties"],
serde_json::Value::Object(expected_properties)
);
}
#[test]
fn test_may_be_fix_tool_schema_missing_properties() {
let json_str = r#"{
"model": "gpt-4o",
"messages": [],
"tools": [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather in a given location",
"parameters": {"type": "object"},
"strict": null
}
}
]
}"#;
let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
let tools = serde_json::to_value(request.tools()).unwrap();
assert_eq!(
tools[0]["function"]["parameters"]["properties"],
serde_json::Value::Object(Default::default())
);
assert_eq!(tools[0]["function"]["parameters"]["type"], "object");
}
#[test]
fn test_may_be_fix_tool_schema_missing_description() {
let json_str = r#"{
"model": "gpt-4o",
"messages": [],
"tools": [
{
"type": "function",
"function": {
"name": "noop",
"parameters": {
"type": "object",
"properties": { "x": { "type": "string" } },
"required": ["x"],
"additionalProperties": false
},
"strict": null
}
}
]
}"#;
let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
let tools = serde_json::to_value(request.tools()).unwrap();
assert_eq!(
tools[0]["function"]["description"],
serde_json::Value::String(String::new())
);
}
#[test]
fn test_may_be_fix_tool_schema_null_description() {
let json_str = r#"{
"model": "gpt-4o",
"messages": [],
"tools": [
{
"type": "function",
"function": {
"name": "noop",
"description": null,
"parameters": {"type": "object", "properties": {}},
"strict": null
}
}
]
}"#;
let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
let tools = serde_json::to_value(request.tools()).unwrap();
assert_eq!(
tools[0]["function"]["description"],
serde_json::Value::String(String::new())
);
}
#[test]
fn test_may_be_fix_tool_schema_preserves_description() {
let json_str = r#"{
"model": "gpt-4o",
"messages": [],
"tools": [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather in a given location",
"parameters": {"type": "object", "properties": {}},
"strict": null
}
}
]
}"#;
let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
let tools = serde_json::to_value(request.tools()).unwrap();
assert_eq!(
tools[0]["function"]["description"],
"Get the current weather in a given location"
);
}
#[test]
fn test_may_be_fix_msg_content_user_multipart() {
let json_str = r#"{
"model": "gpt-4o",
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "part 1"},
{"type": "text", "text": "part 2"}
]
}
]
}"#;
let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
let messages_raw = serde_json::to_value(request.messages()).unwrap();
let messages =
serde_json::to_value(may_be_fix_msg_content(messages_raw, false, &[], &[], None))
.unwrap();
assert_eq!(
messages[0]["content"],
serde_json::Value::String("part 1\npart 2".to_string())
);
}
#[test]
fn test_may_be_fix_msg_content_mixed_messages() {
let json_str = r#"{
"model": "gpt-4o",
"messages": [
{
"role": "system",
"content": "You are a helpful assistant"
},
{
"role": "user",
"content": [
{"type": "text", "text": "Hello"},
{"type": "text", "text": "World"}
]
},
{
"role": "assistant",
"content": "Hi there!"
},
{
"role": "user",
"content": [
{"type": "text", "text": "Another"},
{"type": "text", "text": "multi-part"},
{"type": "text", "text": "message"}
]
}
]
}"#;
let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
let messages_raw = serde_json::to_value(request.messages()).unwrap();
let messages =
serde_json::to_value(may_be_fix_msg_content(messages_raw, false, &[], &[], None))
.unwrap();
assert_eq!(
messages[0]["content"],
serde_json::Value::String("You are a helpful assistant".to_string())
);
assert_eq!(
messages[1]["content"],
serde_json::Value::String("Hello\nWorld".to_string())
);
assert_eq!(
messages[2]["content"],
serde_json::Value::String("Hi there!".to_string())
);
assert_eq!(
messages[3]["content"],
serde_json::Value::String("Another\nmulti-part\nmessage".to_string())
);
}
#[test]
fn test_may_be_fix_msg_content_empty_array() {
let json_str = r#"{
"model": "gpt-4o",
"messages": [
{
"role": "user",
"content": []
}
]
}"#;
let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
let messages_raw = serde_json::to_value(request.messages()).unwrap();
let messages =
serde_json::to_value(may_be_fix_msg_content(messages_raw, false, &[], &[], None))
.unwrap();
assert!(messages[0]["content"].is_array());
assert_eq!(messages[0]["content"].as_array().unwrap().len(), 0);
}
#[test]
fn test_may_be_fix_msg_content_empty_array_with_placeholder_template() {
let json_str = r#"{
"model": "phi-3-vision",
"messages": [
{
"role": "user",
"content": []
}
]
}"#;
let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
let messages_raw = serde_json::to_value(request.messages()).unwrap();
let messages = serde_json::to_value(may_be_fix_msg_content(
messages_raw,
false,
&[],
&[],
Some("<|image_{n}|>"),
))
.unwrap();
assert!(
messages[0]["content"].is_array(),
"empty array should be preserved as `[]`, not flattened to `\"\"`"
);
assert_eq!(messages[0]["content"].as_array().unwrap().len(), 0);
}
#[test]
fn test_may_be_fix_msg_content_single_text() {
let json_str = r#"{
"model": "gpt-4o",
"messages": [
{
"role": "user",
"content": "Simple text message"
}
]
}"#;
let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
let messages_raw = serde_json::to_value(request.messages()).unwrap();
let messages =
serde_json::to_value(may_be_fix_msg_content(messages_raw, false, &[], &[], None))
.unwrap();
assert_eq!(
messages[0]["content"],
serde_json::Value::String("Simple text message".to_string())
);
}
#[test]
fn test_may_be_fix_msg_content_mixed_types() {
let json_str = r#"{
"model": "gpt-4o",
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "Check this image:"},
{"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}},
{"type": "text", "text": "What do you see?"}
]
}
]
}"#;
let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
let messages_raw = serde_json::to_value(request.messages()).unwrap();
let messages =
serde_json::to_value(may_be_fix_msg_content(messages_raw, false, &[], &[], None))
.unwrap();
assert!(messages[0]["content"].is_array());
let content_array = messages[0]["content"].as_array().unwrap();
assert_eq!(content_array.len(), 3);
assert_eq!(content_array[0]["type"], "text");
assert_eq!(content_array[1]["type"], "image");
assert!(content_array[1].get("image_url").is_none());
assert_eq!(content_array[2]["type"], "text");
}
#[test]
fn test_may_be_fix_msg_content_flattens_phi3_style() {
let json_str = r#"{
"model": "phi-3-vision",
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "First "},
{"type": "image_url", "image_url": {"url": "https://example.com/a.jpg"}},
{"type": "text", "text": " then "},
{"type": "image_url", "image_url": {"url": "https://example.com/b.jpg"}},
{"type": "text", "text": "?"}
]
}
]
}"#;
let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
let messages_raw = serde_json::to_value(request.messages()).unwrap();
let messages = serde_json::to_value(may_be_fix_msg_content(
messages_raw,
false,
&[],
&[],
Some("<|image_{n}|>"),
))
.unwrap();
let content = messages[0]["content"].as_str().expect("content flattened");
assert_eq!(content, "First <|image_1|> then <|image_2|>?");
}
#[test]
fn test_may_be_fix_msg_content_flattens_llava_style() {
let json_str = r#"{
"model": "llava-1.5-7b-hf",
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "Describe: "},
{"type": "image_url", "image_url": {"url": "https://example.com/x.jpg"}}
]
}
]
}"#;
let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
let messages_raw = serde_json::to_value(request.messages()).unwrap();
let messages = serde_json::to_value(may_be_fix_msg_content(
messages_raw,
false,
&[],
&[],
Some("<image>"),
))
.unwrap();
let content = messages[0]["content"].as_str().expect("content flattened");
assert_eq!(content, "Describe: <image>");
}
#[test]
fn test_may_be_fix_msg_content_flattens_empty_placeholder() {
let json_str = r#"{
"model": "nvidia/NVIDIA-Nemotron-Parse-v1.2",
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "</s><s><predict_bbox><predict_classes><output_markdown><predict_no_text_in_pic>"},
{"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}
]
}
]
}"#;
let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
let messages_raw = serde_json::to_value(request.messages()).unwrap();
let messages = serde_json::to_value(may_be_fix_msg_content(
messages_raw,
false,
&[],
&[],
Some(""),
))
.unwrap();
let content = messages[0]["content"].as_str().expect("content flattened");
assert_eq!(
content,
"</s><s><predict_bbox><predict_classes><output_markdown><predict_no_text_in_pic>"
);
}
#[test]
fn test_render_nemotron_parse_passthrough() {
use super::super::tokcfg::ChatTemplate;
use super::{ContextMixins, HfTokenizerConfigJsonFormatter};
let chat_template: ChatTemplate = serde_json::from_value(serde_json::json!({
"chat_template": "{% for message in messages %}{{ message['content'] }}{% endfor %}"
}))
.unwrap();
let formatter =
HfTokenizerConfigJsonFormatter::new(chat_template, ContextMixins::new(&[])).unwrap();
for prompt in [
"</s><s><predict_bbox><predict_classes><output_markdown><predict_no_text_in_pic>",
"</s><s><predict_bbox><predict_classes><output_markdown><predict_text_in_pic>",
] {
let request: NvCreateChatCompletionRequest =
serde_json::from_value(serde_json::json!({
"model": "nvidia/NVIDIA-Nemotron-Parse-v1.2",
"messages": [{
"role": "user",
"content": [
{"type": "text", "text": prompt},
{"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}
]
}]
}))
.unwrap();
let rendered = formatter.render(&request).unwrap();
assert_eq!(
rendered, prompt,
"rendered prompt must be the control tokens only, with no JSON-serialized image array"
);
}
}
#[test]
fn test_may_be_fix_msg_content_non_text_only() {
let json_str = r#"{
"model": "gpt-4o",
"messages": [
{
"role": "user",
"content": [
{"type": "image_url", "image_url": {"url": "https://example.com/image1.jpg"}},
{"type": "image_url", "image_url": {"url": "https://example.com/image2.jpg"}}
]
}
]
}"#;
let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
let messages_raw = serde_json::to_value(request.messages()).unwrap();
let messages =
serde_json::to_value(may_be_fix_msg_content(messages_raw, false, &[], &[], None))
.unwrap();
assert!(messages[0]["content"].is_array());
let content_array = messages[0]["content"].as_array().unwrap();
assert_eq!(content_array.len(), 2);
assert_eq!(content_array[0]["type"], "image");
assert_eq!(content_array[1]["type"], "image");
}
#[test]
fn test_none_tools_safe_for_all_templates() {
use super::tokcfg::ChatTemplate;
use super::{ContextMixins, HfTokenizerConfigJsonFormatter};
let length_template = r#"
{%- if tools is iterable and tools | length > 0 %}
Tools available: {{ tools | length }}
{%- else %}
No tools
{%- endif %}
"#;
let no_tool_template = r#"
{%- if tools is not none %}
TOOL MODE
{%- else %}
NORMAL MODE
{%- endif %}
"#;
let chat_template: ChatTemplate = serde_json::from_value(serde_json::json!({
"chat_template": [
{"safe_length": length_template},
{"no_tool": no_tool_template}
]
}))
.unwrap();
let formatter =
HfTokenizerConfigJsonFormatter::new(chat_template, ContextMixins::new(&[])).unwrap();
let ctx = context! { tools => Option::<Value>::None };
let result1 = formatter
.env
.get_template("safe_length")
.unwrap()
.render(&ctx);
println!("Safe length template with no tools => None: {:?}", result1);
assert!(
result1.is_ok(),
"Jinja template with and conditional and length filter should handle None: {:?}",
result1
);
assert!(
result1.unwrap().contains("No tools"),
"Should show 'No tools'"
);
let result2 = formatter.env.get_template("no_tool").unwrap().render(&ctx);
println!("Default template with no tools => None: {:?}", result2);
assert!(
result2.is_ok(),
"Jinja template with if tools is not none conditional should handle None: {:?}",
result2
);
assert!(result2.unwrap().contains("NORMAL MODE"));
}
#[test]
fn test_may_be_fix_msg_content_multiple_content_types() {
let json_str = r#"{
"model": "gpt-4o",
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "Listen to this:"},
{"type": "audio_url", "audio_url": {"url": "https://example.com/audio.mp3"}},
{"type": "text", "text": "And look at:"},
{"type": "image_url", "image_url": {"url": "https://example.com/img.jpg"}},
{"type": "text", "text": "What do you think?"}
]
}
]
}"#;
let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
let messages_raw = serde_json::to_value(request.messages()).unwrap();
let messages =
serde_json::to_value(may_be_fix_msg_content(messages_raw, false, &[], &[], None))
.unwrap();
assert!(messages[0]["content"].is_array());
let content_array = messages[0]["content"].as_array().unwrap();
assert_eq!(content_array.len(), 5);
assert_eq!(content_array[0]["type"], "text");
assert_eq!(content_array[1]["type"], "audio");
assert_eq!(content_array[2]["type"], "text");
assert_eq!(content_array[3]["type"], "image");
assert_eq!(content_array[4]["type"], "text");
let json_str = r#"{
"model": "gpt-4o",
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "Check this:"},
{"type": "video_url", "video_url": {"url": "https://example.com/vid.mp4"}},
{"type": "text", "text": "Interesting?"}
]
}
]
}"#;
let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
let messages_raw = serde_json::to_value(request.messages()).unwrap();
let messages =
serde_json::to_value(may_be_fix_msg_content(messages_raw, false, &[], &[], None))
.unwrap();
assert!(messages[0]["content"].is_array());
assert_eq!(messages[0]["content"].as_array().unwrap().len(), 3);
}
#[test]
fn test_normalize_tool_arguments_tojson() {
let tmpl = r#"{{ messages[0].tool_calls[0].function.arguments | tojson }}"#;
let mut messages = serde_json::Value::Array(vec![serde_json::json!({
"role": "assistant",
"tool_calls": [{
"type": "function",
"function": {
"name": "get_current_weather",
"arguments": "{\"format\":\"celsius\",\"location\":\"San Francisco, CA\"}"
}
}]
})]);
normalize_tool_calls_arguments_in_messages(&mut messages);
let mut env = Environment::new();
env.add_filter("tojson", super::super::tokcfg::tojson);
env.add_template("t", tmpl).unwrap();
let out = env
.get_template("t")
.unwrap()
.render(context! { messages => crate::python::template_value(&messages) })
.unwrap();
assert_eq!(
out,
r#"{"format": "celsius", "location": "San Francisco, CA"}"#
);
}
#[test]
fn test_normalize_tool_arguments_items_loop() {
let tmpl = r#"{% for k, v in messages[0].tool_calls[0].function.arguments|items %}{{k}}={{v}};{% endfor %}"#;
let mut messages = serde_json::Value::Array(vec![serde_json::json!({
"role": "assistant",
"tool_calls": [{
"type": "function",
"function": {
"name": "f",
"arguments": "{\"a\":1,\"b\":\"x\"}"
}
}]
})]);
normalize_tool_calls_arguments_in_messages(&mut messages);
let mut env = Environment::new();
env.add_template("t", tmpl).unwrap();
let out = env
.get_template("t")
.unwrap()
.render(context! { messages => crate::python::template_value(&messages) })
.unwrap();
assert!(out == "a=1;b=x;" || out == "b=x;a=1;");
}
#[test]
fn test_normalize_tool_arguments_legacy_function_call() {
let mut messages = serde_json::Value::Array(vec![serde_json::json!({
"role": "assistant",
"function_call": {
"name": "get_weather",
"arguments": "{\"location\":\"NYC\"}"
}
})]);
normalize_function_call_arguments_in_messages(&mut messages);
assert_eq!(
messages[0]["function_call"]["arguments"],
serde_json::json!({"location": "NYC"})
);
}
#[test]
fn test_normalize_tool_arguments_malformed_json_passthrough() {
let mut messages = serde_json::Value::Array(vec![serde_json::json!({
"role": "assistant",
"tool_calls": [{
"type": "function",
"function": {
"name": "f",
"arguments": "not valid json at all"
}
}]
})]);
normalize_tool_calls_arguments_in_messages(&mut messages);
assert_eq!(
messages[0]["tool_calls"][0]["function"]["arguments"],
serde_json::Value::String("not valid json at all".to_string())
);
}
#[test]
fn test_normalize_tool_arguments_with_multimodal_content() {
let json_str = r#"{
"model": "gpt-4o",
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "Check this:"},
{"type": "video_url", "video_url": {"url": "https://example.com/vid.mp4"}},
{"type": "text", "text": "Interesting?"}
]
},
{
"role": "assistant",
"tool_calls": [{
"id": "call_123",
"type": "function",
"function": {
"name": "analyze_video",
"arguments": "{\"url\":\"https://example.com/vid.mp4\",\"format\":\"mp4\"}"
}
}]
}
]
}"#;
let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
let messages_raw = serde_json::to_value(request.messages()).unwrap();
let mut messages =
serde_json::to_value(may_be_fix_msg_content(messages_raw, false, &[], &[], None))
.unwrap();
normalize_tool_calls_arguments_in_messages(&mut messages);
assert!(messages[0]["content"].is_array());
assert_eq!(messages[0]["content"].as_array().unwrap().len(), 3);
assert!(messages[1]["tool_calls"][0]["function"]["arguments"].is_object());
assert_eq!(
messages[1]["tool_calls"][0]["function"]["arguments"]["url"],
"https://example.com/vid.mp4"
);
}
#[test]
fn test_minimax_m3_history_tool_call_float_arguments_match_hf() {
let template = r#"{%- set ns_token = ']<]minimax[>[' -%}
{%- macro to_xml(val, ns) -%}
{%- if val is mapping -%}
{%- for k, v in val.items() if v is not none -%}
{{ ns }}<{{ k }}>{{ to_xml(v, ns) }}{{ ns }}</{{ k }}>
{%- endfor -%}
{%- elif val is iterable and val is not string -%}
{%- for item in val -%}
{{ ns }}<item>{{ to_xml(item, ns) }}{{ ns }}</item>
{%- endfor -%}
{%- elif val is none -%}
{%- elif val is boolean -%}
{{ val | tojson }}
{%- else -%}
{{ val }}
{%- endif -%}
{%- endmacro -%}
{%- for message in messages if message.tool_calls -%}
{%- for tool_call in message.tool_calls -%}
{%- if tool_call.function -%}
{%- set tool_call = tool_call.function -%}
{%- endif -%}
{{- ns_token + '<invoke name="' + tool_call.name + '">' }}
{%- set _args = tool_call.arguments -%}
{%- for k, v in _args.items() if v is not none %}
{{- ns_token + '<' + k + '>' -}}
{{- to_xml(v, ns_token) -}}
{{- ns_token + '</' + k + '>' }}
{%- endfor -%}
{{- ns_token + '</invoke>' ~ '\n' }}
{%- endfor -%}
{%- endfor -%}"#;
let rendered = render_shape(
&formatter_for(template),
json!([
{"role": "user", "content": "u"},
{"role": "assistant", "content": "", "tool_calls": [{
"id": "c1",
"type": "function",
"function": {
"name": "fit",
"arguments": r#"{"tolerance": 1e-07, "bounds": [0.00001, 1e16]}"#
}
}]}
]),
)
.unwrap();
assert_eq!(
rendered,
"]<]minimax[>[<invoke name=\"fit\">]<]minimax[>[<tolerance>1e-07]<]minimax[>[</tolerance>]<]minimax[>[<bounds>]<]minimax[>[<item>1e-05]<]minimax[>[</item>]<]minimax[>[<item>1e+16]<]minimax[>[</item>]<]minimax[>[</bounds>]<]minimax[>[</invoke>\n"
);
}
#[test]
fn test_qwen3_coder_history_tool_call_float_arguments_match_hf() {
let template = r#"{%- for message in messages if message.tool_calls -%}
{%- for tool_call in message.tool_calls %}
{%- if tool_call.function is defined %}
{%- set tool_call = tool_call.function %}
{%- endif %}
{%- for args_name, args_value in tool_call.arguments|items %}
{{- '<parameter=' + args_name + '>\n' }}
{%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
{{- args_value }}
{{- '\n</parameter>\n' }}
{%- endfor %}
{%- endfor %}
{%- endfor %}"#;
let rendered = render_shape(
&formatter_for(template),
json!([
{"role": "user", "content": "u"},
{"role": "assistant", "content": "", "tool_calls": [{
"id": "c1",
"type": "function",
"function": {
"name": "fit",
"arguments": r#"{"tolerance": 1e-07, "bounds": [0.00001, 1e16]}"#
}
}]}
]),
)
.unwrap();
assert_eq!(
rendered,
"<parameter=tolerance>\n1e-07\n</parameter>\n<parameter=bounds>\n[1e-05, 1e+16]\n</parameter>\n"
);
}
#[test]
fn test_may_be_fix_msg_content_string_to_array() {
let json_str = r#"{
"model": "gpt-4o",
"messages": [
{
"role": "user",
"content": "Hello, how are you?"
}
]
}"#;
let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
let messages_raw = serde_json::to_value(request.messages()).unwrap();
let messages =
serde_json::to_value(may_be_fix_msg_content(messages_raw, true, &[], &[], None))
.unwrap();
assert!(messages[0]["content"].is_array());
let content_array = messages[0]["content"].as_array().unwrap();
assert_eq!(content_array.len(), 1);
assert_eq!(content_array[0]["type"], "text");
assert_eq!(content_array[0]["text"], "Hello, how are you?");
}
#[test]
fn test_may_be_fix_msg_content_array_preserved_with_multimodal() {
let json_str = r#"{
"model": "gpt-4o",
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "part 1"},
{"type": "text", "text": "part 2"}
]
}
]
}"#;
let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
let messages_raw = serde_json::to_value(request.messages()).unwrap();
let messages =
serde_json::to_value(may_be_fix_msg_content(messages_raw, true, &[], &[], None))
.unwrap();
assert!(messages[0]["content"].is_array());
let content_array = messages[0]["content"].as_array().unwrap();
assert_eq!(content_array.len(), 2);
assert_eq!(content_array[0]["text"], "part 1");
assert_eq!(content_array[1]["text"], "part 2");
}
fn text_parts(first: &str, second: &str) -> serde_json::Value {
json!([{"type": "text", "text": first}, {"type": "text", "text": second}])
}
const TEXT_PARTS_TMPL: &str = concat!(
"{%- for m in messages -%}<{{ m.role }}>",
"{%- if m.content is string -%}{{ m.content }}{%- else -%}",
"{%- for p in m.content -%}{{ p.text }}",
"{%- if not loop.last -%}{{ ARRAY_SEPARATOR }}{%- endif -%}",
"{%- endfor -%}{%- endif -%}</{{ m.role }}>{%- endfor -%}"
);
#[test]
fn text_part_arrays_keep_template_separators_and_framing() {
let shape = json!([{"role": "user", "content": text_parts("part one", "part two")}]);
for separator in [" ", "", "\n", " :: "] {
let template = TEXT_PARTS_TMPL.replace(
"ARRAY_SEPARATOR",
&serde_json::to_string(separator).unwrap(),
);
let f = formatter_for(&template);
assert!(!f.requires_content_arrays);
assert_eq!(
render_shape(&f, shape.clone()).unwrap(),
format!("<user>part one{separator}part two</user>"),
);
}
let template = TEXT_PARTS_TMPL
.replace("ARRAY_SEPARATOR", "''")
.replace("{{ p.text }}", "<text>{{ p.text }}</text>");
assert_eq!(
render_shape(&formatter_for(&template), shape).unwrap(),
"<user><text>part one</text><text>part two</text></user>",
);
}
#[test]
fn preserved_text_arrays_do_not_fall_through_to_media_flattening() {
let shape = json!([{"role": "user", "content": text_parts("part one", "part two")}]);
assert_eq!(
may_be_fix_msg_content(shape.clone(), false, &["user"], &[], Some("<image>")),
shape,
);
}
#[test]
fn text_array_support_is_selected_per_role_including_array_only_roles() {
let template = concat!(
"{%- for m in messages -%}<{{ m.role }}>",
"{%- if m.role == 'system' -%}{{ m.content }}",
"{%- elif m.role == 'assistant' and m.content is string and m.content|length > 0 -%}",
"{{ raise_exception('Assistant content must be an array.') }}",
"{%- elif m.content is string -%}{{ m.content }}{%- else -%}",
"{%- for p in m.content -%}{{ p.text }}",
"{%- if not loop.last -%}|{%- endif -%}{%- endfor -%}",
"{%- endif -%}</{{ m.role }}>{%- endfor -%}"
);
let f = formatter_for(template);
assert!(!f.requires_content_arrays);
assert!(f.default_native_text_array_roles.contains(&"assistant"));
assert!(!f.default_native_text_array_roles.contains(&"system"));
let parts = text_parts("part one", "part two");
let shape = json!([
{"role": "system", "content": parts},
{"role": "developer", "content": parts},
{"role": "user", "content": parts},
{"role": "assistant", "content": parts, "tool_calls": [{
"id": "c1", "type": "function", "function": {"name": "noop", "arguments": "{}"}
}]},
{"role": "tool", "tool_call_id": "c1", "content": parts},
]);
let expected = concat!(
"<system>part one\npart two</system>",
"<developer>part one|part two</developer>",
"<user>part one|part two</user>",
"<assistant>part one|part two</assistant>",
"<tool>part one|part two</tool>"
);
assert_eq!(render_shape(&f, shape.clone()).unwrap(), expected);
assert_eq!(render_shape_with_tools(&f, shape).unwrap(), expected);
}
#[test]
fn text_array_support_is_selected_per_template_variant() {
let native = TEXT_PARTS_TMPL.replace("ARRAY_SEPARATOR", "' '");
let string =
"{%- for m in messages -%}<{{ m.role }}>{{ m.content }}</{{ m.role }}>{%- endfor -%}";
let shape = json!([{"role": "user", "content": text_parts("part one", "part two")}]);
for (default, tool_use, default_text, tool_text) in [
(
native.as_str(),
string,
"part one part two",
"part one\npart two",
),
(
string,
native.as_str(),
"part one\npart two",
"part one part two",
),
] {
let f = formatter_for_templates(default, tool_use);
assert_eq!(
render_shape(&f, shape.clone()).unwrap(),
format!("<user>{default_text}</user>"),
);
assert_eq!(
render_shape_with_tools(&f, shape.clone()).unwrap(),
format!("<user>{tool_text}</user>"),
);
}
}
#[test]
fn merged_turns_keep_their_existing_join_with_native_text_arrays() {
let strict = concat!(
"{%- set ns = namespace(prev='') -%}{%- for m in messages -%}",
"{%- if m.role == 'system' and not loop.first -%}",
"{{ raise_exception('System message must be first.') }}",
"{%- elif m.role == 'user' and ns.prev == 'user' -%}",
"{{ raise_exception('Conversation roles must alternate.') }}{%- endif -%}",
"<{{ m.role }}>{%- if m.content is string -%}{{ m.content }}{%- else -%}",
"{%- for p in m.content -%}{{ p.text }}",
"{%- if not loop.last -%}|{%- endif -%}{%- endfor -%}",
"{%- endif -%}</{{ m.role }}>{%- set ns.prev = m.role -%}{%- endfor -%}"
);
let f = formatter_for(strict);
assert!(f.default_system_normalization.demote_nonleading_system);
assert!(f.default_system_normalization.coalesce_consecutive_users);
let shape = json!([
{"role": "system", "content": text_parts("s1", "s2")},
{"role": "system", "content": "rules"},
{"role": "user", "content": text_parts("u1", "u2")},
{"role": "assistant", "content": "a"},
{"role": "user", "content": "first"},
{"role": "user", "content": text_parts("u3", "u4")},
{"role": "user", "content": text_parts("u5", "u6")},
{"role": "assistant", "content": "a2"},
{"role": "user", "content": text_parts("u7", "u8")},
{"role": "system", "content": text_parts("s3", "s4")},
{"role": "assistant", "content": "a3"},
{"role": "system", "content": text_parts("s5", "s6")},
]);
let mut normalized = shape.clone();
assert_eq!(
normalize_system_messages(&mut normalized, f.default_system_normalization),
[0, 3, 5],
);
assert_eq!(
render_shape(&f, shape).unwrap(),
concat!(
"<system>s1\ns2\nrules</system>",
"<user>u1|u2</user><assistant>a</assistant>",
"<user>first\nu3\nu4\nu5\nu6</user><assistant>a2</assistant>",
"<user>u7\nu8\ns3\ns4</user><assistant>a3</assistant>",
"<user>s5|s6</user>"
),
);
}
#[test]
fn demoted_unmerged_text_arrays_use_their_new_role() {
let template = concat!(
"{%- for m in messages -%}",
"{%- if m.role == 'system' and not loop.first -%}",
"{{ raise_exception('System message must be first.') }}{%- endif -%}",
"<{{ m.role }}>{%- if m.role == 'system' or m.content is string -%}",
"{{ m.content }}{%- else -%}{%- for p in m.content -%}{{ p.text }}",
"{%- if not loop.last -%}{{ ' ' }}{%- endif -%}{%- endfor -%}",
"{%- endif -%}</{{ m.role }}>{%- endfor -%}"
);
let f = formatter_for(template);
assert!(!f.default_native_text_array_roles.contains(&"system"));
assert!(f.default_native_text_array_roles.contains(&"user"));
assert_eq!(
render_shape(
&f,
json!([
{"role": "user", "content": "first"},
{"role": "assistant", "content": "done"},
{"role": "system", "content": text_parts("part one", "part two")},
])
)
.unwrap(),
"<user>first</user><assistant>done</assistant><user>part one part two</user>",
);
}
#[test]
fn explicit_content_formats_override_native_text_array_detection() {
let template = TEXT_PARTS_TMPL.replace("ARRAY_SEPARATOR", "' '");
let mut f = formatter_for(&template);
let shape = json!([{"role": "user", "content": text_parts("part one", "part two")}]);
f.content_format = ChatTemplateContentFormat::String(MediaPlaceholders::default());
assert_eq!(
render_shape(&f, shape.clone()).unwrap(),
"<user>part one\npart two</user>",
);
f.content_format = ChatTemplateContentFormat::OpenAi;
assert_eq!(
render_shape(&f, shape).unwrap(),
"<user>part one part two</user>",
);
}
#[test]
fn injected_reasoning_keeps_existing_text_array_boundaries() {
let template = TEXT_PARTS_TMPL.replace("ARRAY_SEPARATOR", "' | '");
for template in [
template.clone(),
template.replace("{{ p.text }}", "<text>{{ p.text }}</text>"),
] {
let f = formatter_for(&template);
assert!(!f.default_template_handles_reasoning);
assert!(f.default_native_text_array_roles.contains(&"assistant"));
for (reasoning, content, expected) in [
(
json!("r"),
json!([{"type": "text", "text": "part one"}]),
"<assistant><think>r</think>part one</assistant>",
),
(
json!(["", "r", ""]),
text_parts("part one", "part two"),
"<assistant><think>r</think>part one\npart two</assistant>",
),
(
json!(["r", "s"]),
text_parts("part one", "part two"),
"<assistant><think>r</think><think>s</think>part one\npart two</assistant>",
),
(
json!(" "),
text_parts("part one", "part two"),
"<assistant><think> </think>part one\npart two</assistant>",
),
] {
let messages = json!([{
"role": "assistant", "content": content, "reasoning_content": reasoning
}]);
assert_eq!(render_shape(&f, messages.clone()).unwrap(), expected);
assert_eq!(render_shape_with_tools(&f, messages).unwrap(), expected);
}
}
let f = formatter_for(&template);
for reasoning in [json!(""), json!([]), json!(["", ""])] {
let messages = json!([{
"role": "assistant", "content": text_parts("part one", "part two"),
"reasoning_content": reasoning
}]);
assert_eq!(
render_shape(&f, messages).unwrap(),
"<assistant>part one | part two</assistant>",
);
}
let native_reasoning = template.replace(
"<{{ m.role }}>",
"<{{ m.role }}>{% if m.reasoning_content is string %}<native>{{ m.reasoning_content }}</native>{% endif %}",
);
let f = formatter_for(&native_reasoning);
assert!(f.default_template_handles_reasoning);
assert_eq!(
render_shape(
&f,
json!([{
"role": "assistant", "content": text_parts("part one", "part two"),
"reasoning_content": "r"
}])
)
.unwrap(),
"<assistant><native>r</native>part one | part two</assistant>",
);
}
fn user() -> Msg {
Msg::User(Default::default())
}
fn tool() -> Msg {
Msg::Tool(Default::default())
}
fn dummy_state(messages: Vec<Msg>) -> NvCreateChatCompletionRequest {
let json = serde_json::json!({
"model": "test-model",
"messages": messages
});
serde_json::from_value(json).unwrap()
}
#[test]
fn add_after_user() {
let s = dummy_state(vec![user()]);
assert!(s.should_add_generation_prompt());
}
#[test]
fn add_after_tool() {
let s = dummy_state(vec![tool()]);
assert!(s.should_add_generation_prompt());
}
#[test]
fn add_when_empty() {
let s = dummy_state(vec![]);
assert!(s.should_add_generation_prompt());
}
fn tool_aware_formatter(
exclude_tools_when_tool_choice_none: bool,
) -> HfTokenizerConfigJsonFormatter {
let template = r#"
{%- if tools is iterable and tools | length > 0 %}
TOOL_MODE tools={{ tools | length }}
{%- else %}
NORMAL_MODE
{%- endif %}
{{ messages[0].content }}"#;
let chat_template: super::tokcfg::ChatTemplate =
serde_json::from_value(serde_json::json!({ "chat_template": template })).unwrap();
HfTokenizerConfigJsonFormatter::with_options(
chat_template,
ContextMixins::new(&[]),
exclude_tools_when_tool_choice_none,
)
.unwrap()
}
fn gemma4_tool_template_for_tests() -> &'static str {
r#"
{{ bos_token }}
{%- set loop_messages = messages -%}
{%- set ns_turn = namespace(last_user_idx=-1) -%}
{%- for i in range(loop_messages | length) -%}
{%- if loop_messages[i]['role'] == 'user' -%}
{%- set ns_turn.last_user_idx = i -%}
{%- endif -%}
{%- endfor -%}
{%- for message in loop_messages -%}
{%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%}
{{- '<|turn>' + role + '\n' }}
{%- if message.get('reasoning') and loop.index0 > ns_turn.last_user_idx and message.get('tool_calls') -%}
{{- '<|channel>thought\n' + message['reasoning'] + '\n<channel|>'}}
{%- endif -%}
{%- if message['tool_calls'] -%}
{%- for tool_call in message['tool_calls'] -%}
{%- set function = tool_call['function'] -%}
{{- '<|tool_call>call:' + function['name'] + '{' -}}
{%- if function['arguments'] is mapping -%}
{%- set ns_args = namespace(found_first=false) -%}
{%- for key, value in function['arguments'] | dictsort -%}
{%- if ns_args.found_first %},{% endif -%}
{%- set ns_args.found_first = true -%}
{{- key -}}:{{- value -}}
{%- endfor -%}
{%- elif function['arguments'] is string -%}
{{- function['arguments'] -}}
{%- endif -%}
{{- '}<tool_call|>' -}}
{%- endfor -%}
{%- endif -%}
{%- if message['content'] is string -%}
{{- message['content'] -}}
{%- endif -%}
{{- '<turn|>\n' -}}
{%- endfor -%}
"#
}
fn make_gemma4_tool_formatter_for_tests() -> HfTokenizerConfigJsonFormatter {
let chat_template: ChatTemplate = serde_json::from_value(serde_json::json!({
"chat_template": gemma4_tool_template_for_tests()
}))
.unwrap();
HfTokenizerConfigJsonFormatter::new(chat_template, ContextMixins::new(&[])).unwrap()
}
fn request_with_tool_choice(tool_choice: &str) -> NvCreateChatCompletionRequest {
serde_json::from_value(serde_json::json!({
"model": "test",
"messages": [{"role": "user", "content": "hello"}],
"tools": [{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get weather",
"parameters": {"type": "object", "properties": {"location": {"type": "string"}}}
}
}],
"tool_choice": tool_choice
}))
.unwrap()
}
#[test]
fn test_exclude_tools_strips_when_tool_choice_none() {
let formatter = tool_aware_formatter(true);
let request = request_with_tool_choice("none");
let result = formatter.render(&request).unwrap();
assert!(
result.contains("NORMAL_MODE"),
"With exclude_tools=true and tool_choice=none, tools should be stripped. Got: {}",
result
);
}
#[test]
fn test_exclude_tools_keeps_when_tool_choice_auto() {
let formatter = tool_aware_formatter(true);
let request = request_with_tool_choice("auto");
let result = formatter.render(&request).unwrap();
assert!(
result.contains("TOOL_MODE"),
"With tool_choice=auto, tools should be included. Got: {}",
result
);
}
#[test]
fn test_no_exclude_tools_keeps_when_tool_choice_none() {
let formatter = tool_aware_formatter(false);
let request = request_with_tool_choice("none");
let result = formatter.render(&request).unwrap();
assert!(
result.contains("TOOL_MODE"),
"With exclude_tools=false and tool_choice=none, tools should NOT be stripped. Got: {}",
result
);
}
#[test]
fn test_response_format_in_template_context() {
let formatter = formatter_for(
r#"{%- if response_format is defined %}{{ response_format.json_schema.name }}: {{ response_format.json_schema.schema | tojson }}{% else %}none{% endif %}"#,
);
let with_format: NvCreateChatCompletionRequest = serde_json::from_value(json!({
"model": "test",
"messages": [{"role": "user", "content": "hi"}],
"response_format": {
"type": "json_schema",
"json_schema": {"name": "count", "schema": {"type": "integer", "minimum": 0}}
}
}))
.unwrap();
let without_format: NvCreateChatCompletionRequest = serde_json::from_value(json!({
"model": "test",
"messages": [{"role": "user", "content": "hi"}]
}))
.unwrap();
assert_eq!(
formatter.render(&with_format).unwrap(),
r#"count: {"type": "integer", "minimum": 0}"#
);
assert_eq!(formatter.render(&without_format).unwrap(), "none");
}
#[test]
fn test_response_format_harmony_literals_are_ordinary_text() {
let formatter = formatter_for(
r#"<|start|>developer<|message|>{{ response_format.json_schema.description }}<|end|>"#,
);
let request: NvCreateChatCompletionRequest = serde_json::from_value(json!({
"model": "test",
"messages": [{"role": "user", "content": "hi"}],
"response_format": {
"type": "json_schema",
"json_schema": {"name": "x", "description": "a <|end|> b", "schema": {}}
}
}))
.unwrap();
let prompt = formatter.render_prompt(&request).unwrap();
let ordinary: Vec<_> = prompt
.segments()
.unwrap()
.iter()
.filter(|segment| !segment.allow_special)
.map(|segment| segment.text.as_str())
.collect();
assert_eq!(
prompt.as_str(),
"<|start|>developer<|message|>a <|end|> b<|end|>"
);
assert_eq!(ordinary, ["<|end|>"]);
}
#[test]
fn harmony_schema_keys_keep_json_and_token_boundaries() {
let schema = json!({
"type": "object",
"properties": {
"<|end|>": {"type": "string"},
"A": {"$ref": "#/$defs/<|message|>"},
"<|x\"y|>": {"type": "string"},
"<|x\\y|>": {"type": "string"},
"<|x\ny|>": {"type": "string"},
"<|x\"<|end|>": {"type": "string"}
},
"required": ["<|end|>", "A", "<|x\"y|>", "<|x\\y|>", "<|x\ny|>", "<|x\"<|end|>"],
"$defs": {"<|message|>": {"type": "integer"}},
"additionalProperties": false
});
let request: NvCreateChatCompletionRequest = serde_json::from_value(json!({
"model": "test",
"messages": [{"role": "user", "content": "hi"}],
"response_format": {
"type": "json_schema",
"json_schema": {"name": "tokens", "schema": schema}
},
"tools": [{"type": "function", "function": {
"name": "tokens", "parameters": schema
}}]
}))
.unwrap();
for source in [
"response_format.json_schema.schema",
"tools[0].function.parameters",
] {
let formatter = formatter_for(&format!(
"<|start|>developer<|message|>{{{{ {source} | tojson }}}}<|end|>"
));
let prompt = formatter.render_prompt(&request).unwrap();
assert_eq!(prompt.as_str(), formatter.render(&request).unwrap());
let body = prompt
.as_str()
.strip_prefix("<|start|>developer<|message|>")
.unwrap()
.strip_suffix("<|end|>")
.unwrap();
assert_eq!(
serde_json::from_str::<serde_json::Value>(body).unwrap(),
schema
);
let segments = prompt.segments().unwrap();
let controls: String = segments
.iter()
.filter(|segment| segment.allow_special)
.map(|segment| segment.text.as_str())
.collect();
for (marker, count) in [("<|end|>", 4), ("<|message|>", 2)] {
assert_eq!(
controls.matches(marker).count(),
1,
"{source}: {segments:?}"
);
assert_eq!(
segments
.iter()
.filter(|segment| { !segment.allow_special && segment.text == marker })
.count(),
count,
"{source}: {segments:?}"
);
}
}
}
#[test]
fn harmony_schema_key_lookups_match_required_names() {
let formatter = formatter_for(
"{% set schema = response_format.json_schema.schema %}<|start|>developer<|message|>{{ schema.properties[schema.required[0]].type }}:{% for name, property in schema.properties.items() %}{{ name }}={{ name in schema.required }};{% endfor %}<|end|>",
);
let request: NvCreateChatCompletionRequest = serde_json::from_value(json!({
"model": "test",
"messages": [{"role": "user", "content": "hi"}],
"response_format": {"type": "json_schema", "json_schema": {
"name": "tokens", "schema": {
"type": "object",
"properties": {"<|end|>": {"type": "integer"}, "A": {"type": "string"}},
"required": ["<|end|>"]
}
}}
}))
.unwrap();
let prompt = formatter.render_prompt(&request).unwrap();
assert_eq!(prompt.as_str(), formatter.render(&request).unwrap());
assert!(
prompt.as_str().contains("integer:<|end|>=True;A=False;"),
"{prompt:?}"
);
assert!(
prompt
.segments()
.unwrap()
.iter()
.any(|segment| { !segment.allow_special && segment.text == "<|end|>" })
);
}
#[test]
fn test_inject_reasoning_content_segments_with_tool_calls() {
let mut messages = serde_json::json!([
{
"role": "user",
"content": "What is sqrt(144) and sqrt(256)?"
},
{
"role": "assistant",
"content": "Let me calculate those.",
"reasoning_content": ["I need to compute sqrt(144)", "Now sqrt(256)", ""],
"tool_calls": [
{
"id": "call_0",
"type": "function",
"function": {
"name": "calculator",
"arguments": "{\"expr\": \"sqrt(144)\"}"
}
},
{
"id": "call_1",
"type": "function",
"function": {
"name": "calculator",
"arguments": "{\"expr\": \"sqrt(256)\"}"
}
}
]
}
]);
inject_reasoning_content_into_messages(&mut messages);
let assistant = &messages[1];
assert!(
assistant.get("reasoning_content").is_none(),
"reasoning_content should be removed after injection"
);
let content = assistant["content"].as_str().unwrap();
assert!(
content.starts_with("<think>I need to compute sqrt(144)</think>"),
"content should start with first reasoning segment, got: {}",
content
);
assert!(
content.contains("<think>Now sqrt(256)</think>"),
"content should contain second reasoning segment"
);
assert!(
!content.contains("<think></think>"),
"empty segments should be skipped"
);
assert!(
content.ends_with("Let me calculate those."),
"original content should be at the end, got: {}",
content
);
assert!(assistant.get("tool_calls").is_some());
assert_eq!(assistant["tool_calls"].as_array().unwrap().len(), 2);
}
#[test]
fn test_gemma4_template_renders_reasoning_content_segments_around_tool_calls() {
let formatter = make_gemma4_tool_formatter_for_tests();
assert!(
formatter.tool_use_template_handles_reasoning,
"Gemma4 template adaptation should make reasoning_content native"
);
let request: NvCreateChatCompletionRequest = serde_json::from_value(serde_json::json!({
"model": "gemma4-test",
"messages": [
{"role": "user", "content": "inspect two things"},
{
"role": "assistant",
"content": null,
"reasoning_content": [
"Think before the first call.",
"Think before the second call.",
"Think after both calls."
],
"tool_calls": [
{
"id": "call_0",
"type": "function",
"function": {
"name": "first_tool",
"arguments": "{\"path\":\".\"}"
}
},
{
"id": "call_1",
"type": "function",
"function": {
"name": "second_tool",
"arguments": "{\"path\":\"/tmp\"}"
}
}
]
}
]
}))
.unwrap();
let rendered = formatter.render(&request).unwrap();
let expected = concat!(
"<|channel>thought\nThink before the first call.\n<channel|>",
"<|tool_call>call:first_tool{path:.}<tool_call|>",
"<|channel>thought\nThink before the second call.\n<channel|>",
"<|tool_call>call:second_tool{path:/tmp}<tool_call|>",
"<|channel>thought\nThink after both calls.\n<channel|>"
);
assert!(
rendered.contains(expected),
"Gemma4 reasoning segments should stay adjacent to their tool calls, got: {rendered}"
);
assert!(!rendered.contains("<think>"));
assert!(!rendered.contains("reasoning_content"));
}
#[test]
fn test_gemma4_template_renders_reasoning_content_without_tool_calls() {
let formatter = make_gemma4_tool_formatter_for_tests();
let request: NvCreateChatCompletionRequest = serde_json::from_value(serde_json::json!({
"model": "gemma4-test",
"messages": [
{"role": "user", "content": "answer directly"},
{
"role": "assistant",
"content": "Direct answer.",
"reasoning_content": "Private thought."
}
]
}))
.unwrap();
let rendered = formatter.render(&request).unwrap();
assert!(
rendered.contains("<|channel>thought\nPrivate thought.\n<channel|>Direct answer."),
"Gemma4 reasoning_content should render in the thought channel, got: {rendered}"
);
assert!(!rendered.contains("<think>"));
assert!(!rendered.contains("reasoning_content"));
}
#[test]
fn test_reasoning_flag_is_per_template_not_global() {
const PLAIN_DEFAULT: &str = "{{ bos_token }}{%- for message in messages -%}\
{{ message['role'] }}: {{ message['content'] }}\n{%- endfor -%}";
let chat_template: ChatTemplate = serde_json::from_value(serde_json::json!({
"chat_template": [
{"default": PLAIN_DEFAULT},
{"tool_use": gemma4_tool_template_for_tests()},
]
}))
.unwrap();
let formatter =
HfTokenizerConfigJsonFormatter::new(chat_template, ContextMixins::new(&[])).unwrap();
assert!(
formatter.tool_use_template_handles_reasoning,
"adapted gemma4 tool_use template should handle reasoning natively"
);
assert!(
!formatter.default_template_handles_reasoning,
"plain default template does not reference reasoning_content"
);
let request: NvCreateChatCompletionRequest = serde_json::from_value(serde_json::json!({
"model": "gemma4-test",
"messages": [
{"role": "user", "content": "answer directly"},
{
"role": "assistant",
"content": "Direct answer.",
"reasoning_content": "Private thought."
}
]
}))
.unwrap();
let rendered = formatter.render(&request).unwrap();
assert!(
rendered.contains("<think>Private thought.</think>Direct answer."),
"reasoning must be injected on the no-tool default path, got: {rendered}"
);
}
fn reasoning_tool_call_turn(reasoning: serde_json::Value) -> serde_json::Value {
let call = |id: &str, expr: &str| {
json!({"id": id, "type": "function",
"function": {"name": "calc", "arguments": json!({"expr": expr}).to_string()}})
};
json!([
{"role": "user", "content": "sqrt(144) + sqrt(256)?"},
{"role": "assistant", "content": null, "reasoning_content": reasoning,
"tool_calls": [call("call_0", "sqrt(144)"), call("call_1", "sqrt(256)")]},
{"role": "tool", "tool_call_id": "call_0", "content": "12"},
{"role": "tool", "tool_call_id": "call_1", "content": "16"}
])
}
#[test]
fn test_string_reasoning_template_joins_reasoning_content_segments() {
const MINIMAX_REASONING_TMPL: &str = r#"{%- for message in messages -%}
{%- if message.role == 'assistant' -%}
{{- ']~b]ai' ~ '\n' -}}
{%- set reasoning_content = '' -%}
{%- if message.reasoning_content is string -%}
{%- set reasoning_content = message.reasoning_content -%}
{%- endif -%}
{%- if reasoning_content -%}
{{- '<think>' ~ '\n' ~ reasoning_content ~ '\n' ~ '</think>' ~ '\n\n' -}}
{%- endif -%}
{%- for tool_call in message.tool_calls -%}
{{- '<invoke name="' ~ tool_call.function.name ~ '">' -}}
{%- endfor -%}
{{- '[e~[\n' -}}
{%- else -%}
{{- ']~b]' ~ message.role ~ '\n' ~ message.content ~ '[e~[\n' -}}
{%- endif -%}
{%- endfor -%}"#;
let f = formatter_for(MINIMAX_REASONING_TMPL);
let expected = concat!(
"]~b]user\nsqrt(144) + sqrt(256)?[e~[\n",
"]~b]ai\n<think>\nCheck both.\nThen add.\n</think>\n\n",
"<invoke name=\"calc\"><invoke name=\"calc\">[e~[\n",
"]~b]tool\n12[e~[\n]~b]tool\n16[e~[\n",
);
for render in [render_shape, render_shape_with_tools] {
for reasoning in [
json!(["Check both.", "Then add.", ""]),
json!("Check both.\nThen add."),
] {
assert_eq!(
render(&f, reasoning_tool_call_turn(reasoning)).unwrap(),
expected
);
}
}
}
#[test]
fn test_inject_reasoning_content_text_variant() {
let mut messages = serde_json::json!([
{
"role": "assistant",
"content": "The answer is 42.",
"reasoning_content": "Let me think about this carefully."
}
]);
inject_reasoning_content_into_messages(&mut messages);
let assistant = &messages[0];
assert!(assistant.get("reasoning_content").is_none());
let content = assistant["content"].as_str().unwrap();
assert_eq!(
content,
"<think>Let me think about this carefully.</think>The answer is 42."
);
}
#[test]
fn test_inject_reasoning_content_null_content() {
let mut messages = serde_json::json!([
{
"role": "assistant",
"content": null,
"reasoning_content": "Thinking...",
"tool_calls": [{"id": "call_0", "type": "function", "function": {"name": "f", "arguments": "{}"}}]
}
]);
inject_reasoning_content_into_messages(&mut messages);
let content = messages[0]["content"].as_str().unwrap();
assert_eq!(content, "<think>Thinking...</think>");
assert!(messages[0].get("reasoning_content").is_none());
}
#[test]
fn test_inject_reasoning_content_skips_non_assistant() {
let mut messages = serde_json::json!([
{
"role": "user",
"content": "hello",
"reasoning_content": "should not be touched"
}
]);
inject_reasoning_content_into_messages(&mut messages);
assert!(messages[0].get("reasoning_content").is_some());
}
fn make_test_formatter() -> HfTokenizerConfigJsonFormatter {
use super::tokcfg::ChatTemplate;
use super::{ContextMixins, HfTokenizerConfigJsonFormatter};
let template = r#"{%- for message in messages %}{{ message.role }}: {{ message.content }}
{%- endfor %}
{%- if add_generation_prompt %}assistant:{%- endif %}"#;
let chat_template: ChatTemplate = serde_json::from_value(serde_json::json!({
"chat_template": template
}))
.unwrap();
HfTokenizerConfigJsonFormatter::new(chat_template, ContextMixins::new(&[])).unwrap()
}
#[test]
fn test_reasoning_content_text_roundtrip_render() {
use super::OAIPromptFormatter;
let formatter = make_test_formatter();
let request: NvCreateChatCompletionRequest = serde_json::from_value(serde_json::json!({
"model": "test-model",
"messages": [
{"role": "user", "content": "What is sqrt(144)?"},
{
"role": "assistant",
"content": "The answer is 12.",
"reasoning_content": "I need to compute the square root of 144."
},
{"role": "user", "content": "Are you sure?"}
]
}))
.unwrap();
let rendered = formatter.render(&request).unwrap();
assert!(
rendered.contains("<think>I need to compute the square root of 144.</think>"),
"reasoning_content must appear as <think> block, got: {}",
rendered
);
assert!(
rendered.contains("The answer is 12."),
"original content must be preserved"
);
assert!(
!rendered.contains("reasoning_content"),
"raw reasoning_content field should not leak into prompt"
);
}
#[test]
fn test_reasoning_content_agentic_tool_call_roundtrip_render() {
use super::OAIPromptFormatter;
let formatter = make_test_formatter();
let request: NvCreateChatCompletionRequest = serde_json::from_value(serde_json::json!({
"model": "test-model",
"messages": [
{"role": "user", "content": "What is sqrt(144) + sqrt(256)?"},
{
"role": "assistant",
"content": null,
"reasoning_content": "I need to compute both square roots. Let me start with sqrt(144).",
"tool_calls": [{
"id": "call_0",
"type": "function",
"function": {
"name": "calculator",
"arguments": "{\"expr\": \"sqrt(144)\"}"
}
}]
},
{
"role": "tool",
"tool_call_id": "call_0",
"content": "12"
},
{
"role": "assistant",
"content": "sqrt(144) = 12 and sqrt(256) = 16, so the answer is 28.",
"reasoning_content": "Got 12 for sqrt(144). Now sqrt(256) = 16. Sum is 28."
},
{"role": "user", "content": "Thanks!"}
]
}))
.unwrap();
let rendered = formatter.render(&request).unwrap();
assert!(
rendered.contains("<think>I need to compute both square roots"),
"first turn reasoning must be in prompt, got: {}",
rendered
);
assert!(
rendered.contains("<think>Got 12 for sqrt(144)"),
"second turn reasoning must be in prompt"
);
assert!(
rendered.contains("the answer is 28"),
"final answer content must be preserved"
);
assert!(
!rendered.contains("reasoning_content"),
"raw reasoning_content field should not leak into prompt"
);
}
#[test]
fn test_reasoning_injected_when_template_ignores_it() {
use super::OAIPromptFormatter;
let formatter = make_test_formatter();
assert!(!formatter.default_template_handles_reasoning);
assert!(!formatter.tool_use_template_handles_reasoning);
let request: NvCreateChatCompletionRequest = serde_json::from_value(serde_json::json!({
"model": "test-model",
"messages": [
{"role": "user", "content": "Hello"},
{
"role": "assistant",
"content": "Hi.",
"reasoning_content": "The user said hello."
},
{"role": "user", "content": "Bye"}
]
}))
.unwrap();
let rendered = formatter.render(&request).unwrap();
assert!(
rendered.contains("<think>The user said hello.</think>"),
"injection must happen when template ignores reasoning_content, got: {}",
rendered
);
}
#[test]
fn test_reasoning_not_injected_when_template_handles_it() {
use super::tokcfg::ChatTemplate;
use super::{ContextMixins, HfTokenizerConfigJsonFormatter, OAIPromptFormatter};
let template = r#"{%- for message in messages %}{%- if message.role == "assistant" and message.reasoning_content is defined and message.reasoning_content %}<think>{{ message.reasoning_content }}</think>
{%- endif %}{{ message.role }}: {{ message.content }}
{%- endfor %}
{%- if add_generation_prompt %}assistant:{%- endif %}"#;
let chat_template: ChatTemplate = serde_json::from_value(serde_json::json!({
"chat_template": template
}))
.unwrap();
let formatter =
HfTokenizerConfigJsonFormatter::new(chat_template, ContextMixins::new(&[])).unwrap();
assert!(formatter.default_template_handles_reasoning);
assert!(formatter.tool_use_template_handles_reasoning);
let request: NvCreateChatCompletionRequest = serde_json::from_value(serde_json::json!({
"model": "test-model",
"messages": [
{"role": "user", "content": "Hello"},
{
"role": "assistant",
"content": "Hi.",
"reasoning_content": "The user said hello."
},
{"role": "user", "content": "Bye"}
]
}))
.unwrap();
let rendered = formatter.render(&request).unwrap();
assert!(
rendered.contains("<think>The user said hello.</think>"),
"template must render reasoning_content natively, got: {}",
rendered
);
let think_count = rendered.matches("<think>").count();
assert_eq!(
think_count, 1,
"must have exactly one <think> block (from template), got {} in: {}",
think_count, rendered
);
}
const QWEN3_THINKING_TEMPLATE: &str = r##"{%- if tools %}
{{- '<|im_start|>system\n' }}
{%- if messages[0].role == 'system' %}
{{- messages[0].content + '\n\n' }}
{%- endif %}
{{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
{%- for tool in tools %}
{{- "\n" }}
{{- tool | tojson }}
{%- endfor %}
{{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
{%- else %}
{%- if messages[0].role == 'system' %}
{{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
{%- endif %}
{%- endif %}
{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
{%- for message in messages[::-1] %}
{%- set index = (messages|length - 1) - loop.index0 %}
{%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
{%- set ns.multi_step_tool = false %}
{%- set ns.last_query_index = index %}
{%- endif %}
{%- endfor %}
{%- for message in messages %}
{%- if message.content is string %}
{%- set content = message.content %}
{%- else %}
{%- set content = '' %}
{%- endif %}
{%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
{{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
{%- elif message.role == "assistant" %}
{%- set reasoning_content = '' %}
{%- if message.reasoning_content is string %}
{%- set reasoning_content = message.reasoning_content %}
{%- else %}
{%- if '</think>' in content %}
{%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
{%- set content = content.split('</think>')[-1].lstrip('\n') %}
{%- endif %}
{%- endif %}
{%- if loop.index0 > ns.last_query_index %}
{%- if loop.last or (not loop.last and reasoning_content) %}
{{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
{%- else %}
{{- '<|im_start|>' + message.role + '\n' + content }}
{%- endif %}
{%- else %}
{{- '<|im_start|>' + message.role + '\n' + content }}
{%- endif %}
{%- if message.tool_calls %}
{%- for tool_call in message.tool_calls %}
{%- if (loop.first and content) or (not loop.first) %}
{{- '\n' }}
{%- endif %}
{%- if tool_call.function %}
{%- set tool_call = tool_call.function %}
{%- endif %}
{{- '<tool_call>\n{"name": "' }}
{{- tool_call.name }}
{{- '", "arguments": ' }}
{%- if tool_call.arguments is string %}
{{- tool_call.arguments }}
{%- else %}
{{- tool_call.arguments | tojson }}
{%- endif %}
{{- '}\n</tool_call>' }}
{%- endfor %}
{%- endif %}
{{- '<|im_end|>\n' }}
{%- elif message.role == "tool" %}
{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
{{- '<|im_start|>user' }}
{%- endif %}
{{- '\n<tool_response>\n' }}
{{- content }}
{{- '\n</tool_response>' }}
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
{{- '<|im_end|>\n' }}
{%- endif %}
{%- endif %}
{%- endfor %}
{%- if add_generation_prompt %}
{{- '<|im_start|>assistant\n<think>\n' }}
{%- endif %}"##;
fn qwen3_thinking_formatter() -> HfTokenizerConfigJsonFormatter {
let chat_template: ChatTemplate = serde_json::from_value(serde_json::json!({
"chat_template": QWEN3_THINKING_TEMPLATE,
}))
.unwrap();
HfTokenizerConfigJsonFormatter::new(chat_template, ContextMixins::new(&[])).unwrap()
}
#[test]
fn test_qwen3_thinking_template_flags_detected() {
let formatter = qwen3_thinking_formatter();
assert!(
formatter.tool_use_template_handles_reasoning,
"template references reasoning_content directly"
);
assert!(
formatter.default_template_handles_tool_calls_arguments_string,
"default template branches on `arguments is string`"
);
assert!(
formatter.tool_use_template_handles_tool_calls_arguments_string,
"tool_use template branches on `arguments is string`"
);
}
const QWEN38_REJECTS_STRING_ARGS_TEMPLATE: &str = r##"{%- for message in messages %}
{%- if message.role == "assistant" and message.tool_calls %}
{%- for tool_call in message.tool_calls %}
{%- if tool_call.function %}
{%- set tool_call = tool_call.function %}
{%- endif %}
{{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
{%- if tool_call.arguments is mapping %}
{%- for args_name, args_value in tool_call.arguments|items %}
{{- '<parameter=' + args_name + '>\n' + args_value + '\n</parameter>\n' }}
{%- endfor %}
{%- elif tool_call.arguments is string %}
{%- if tool_call.arguments|trim %}
{{- raise_exception('Tool call arguments were passed as a JSON string.') }}
{%- endif %}
{%- endif %}
{{- '</function>\n</tool_call>' }}
{%- endfor %}
{%- else %}
{{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>\n' }}
{%- endif %}
{%- endfor %}"##;
#[test]
fn test_template_rejecting_string_arguments_gets_objects() {
let chat_template: ChatTemplate = serde_json::from_value(serde_json::json!({
"chat_template": QWEN38_REJECTS_STRING_ARGS_TEMPLATE,
}))
.unwrap();
let formatter =
HfTokenizerConfigJsonFormatter::new(chat_template, ContextMixins::new(&[])).unwrap();
assert!(!formatter.default_template_handles_tool_calls_arguments_string);
assert!(!formatter.tool_use_template_handles_tool_calls_arguments_string);
let request: NvCreateChatCompletionRequest = serde_json::from_value(serde_json::json!({
"model": "qwen3.8",
"messages": [
{"role": "user", "content": "What's the weather in San Francisco?"},
{"role": "assistant", "content": "", "tool_calls": [{
"id": "call_sf",
"type": "function",
"function": {"name": "get_weather", "arguments": "{\"location\": \"San Francisco\"}"}
}]},
{"role": "tool", "tool_call_id": "call_sf", "content": "Foggy"}
],
}))
.unwrap();
let rendered = formatter.render(&request).unwrap();
assert!(
rendered.contains("<parameter=location>\nSan Francisco\n</parameter>"),
"{rendered}"
);
}
#[test]
fn test_qwen3_thinking_append_only_across_tool_use_turn() {
let formatter = qwen3_thinking_formatter();
let tools = serde_json::json!([{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather for a location",
"parameters": {
"type": "object",
"properties": {
"location": {"type": "string"},
"unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}
},
"required": ["location"]
}
}
}]);
let turn1_request: NvCreateChatCompletionRequest =
serde_json::from_value(serde_json::json!({
"model": "qwen3-thinking",
"messages": [
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "What's the weather in San Francisco?"},
],
"tools": tools,
}))
.unwrap();
let p1 = formatter.render(&turn1_request).unwrap();
let model_emitted = "I'll call get_weather for SF.\n\
</think>\n\n\
<tool_call>\n\
{\"name\": \"get_weather\", \"arguments\": {\"location\": \"San Francisco\", \"unit\": \"celsius\"}}\n\
</tool_call><|im_end|>\n";
let wire_after_t1 = format!("{p1}{model_emitted}");
let turn2_request: NvCreateChatCompletionRequest =
serde_json::from_value(serde_json::json!({
"model": "qwen3-thinking",
"messages": [
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "What's the weather in San Francisco?"},
{
"role": "assistant",
"content": "",
"reasoning_content": "I'll call get_weather for SF.",
"tool_calls": [{
"id": "call_sf",
"type": "function",
"function": {
"name": "get_weather",
"arguments": "{\"location\": \"San Francisco\", \"unit\": \"celsius\"}"
}
}]
},
{
"role": "tool",
"tool_call_id": "call_sf",
"content": "{\"temp\": 18, \"conditions\": \"Foggy\"}"
}
],
"tools": tools,
}))
.unwrap();
let p2 = formatter.render(&turn2_request).unwrap();
if !p2.starts_with(&wire_after_t1) {
let div = wire_after_t1
.as_bytes()
.iter()
.zip(p2.as_bytes())
.position(|(a, b)| a != b)
.unwrap_or_else(|| wire_after_t1.len().min(p2.len()));
let lo = div.saturating_sub(40);
panic!(
"turn-2 prompt is NOT a prefix-extension of [turn-1 + model bytes]\n \
diverges at byte {div}\n \
wire ends: ...{}|{}\n \
t2 has: ...{}|{}",
String::from_utf8_lossy(&wire_after_t1.as_bytes()[lo..div]),
String::from_utf8_lossy(
&wire_after_t1.as_bytes()[div..(div + 60).min(wire_after_t1.len())]
),
String::from_utf8_lossy(&p2.as_bytes()[lo..div]),
String::from_utf8_lossy(&p2.as_bytes()[div..(div + 60).min(p2.len())]),
);
}
let suffix = &p2[wire_after_t1.len()..];
assert!(
suffix.contains("<tool_response>"),
"appended bytes must include the tool response, got: {suffix}"
);
assert!(
suffix.ends_with("<|im_start|>assistant\n<think>\n"),
"appended bytes must end with the next generation prompt, got: {suffix}"
);
}
#[test]
fn test_qwen3_thinking_renders_reasoning_content_segments_as_string() {
let formatter = qwen3_thinking_formatter();
assert!(formatter.default_template_requires_reasoning_string);
assert!(formatter.tool_use_template_requires_reasoning_string);
for render in [render_shape, render_shape_with_tools] {
let segments = json!(["Check both.", "Then add.", ""]);
let rendered = render(&formatter, reasoning_tool_call_turn(segments)).unwrap();
assert!(
rendered.contains(
"<|im_start|>assistant\n<think>\nCheck both.\nThen add.\n</think>\n\n<tool_call>"
),
"{rendered}"
);
let string = json!("Check both.\nThen add.");
assert_eq!(
rendered,
render(&formatter, reasoning_tool_call_turn(string)).unwrap()
);
}
}
fn contents(messages: &serde_json::Value) -> serde_json::Value {
messages
.as_array()
.unwrap()
.iter()
.map(|m| m["content"].clone())
.collect()
}
#[test]
fn content_format_string_flattens_like_vllm() {
let placeholders = MediaPlaceholders {
image: "<|image_{n}|>".into(),
video: "<video>".into(),
audio: "<|audio_{n}|>".into(),
};
let image = json!({"type": "image_url", "image_url": {"url": "x"}});
let flatten = |mut messages: serde_json::Value, use_audio_in_video| {
flatten_msg_content_to_string(&mut messages, &placeholders, use_audio_in_video)
.map(|()| contents(&messages))
};
let messages = json!([
{"role": "user", "content": [
{"type": "text", "text": "a"}, &image,
{"type": "video_url", "video_url": {"url": "y"}},
{"type": "text", "text": "b"}, &image
]},
{"role": "user", "content": [&image]}
]);
assert_eq!(
flatten(messages, false).unwrap(),
json!(["<|image_1|>\n<|image_2|>\n<video>\na\nb", "<|image_3|>"])
);
let messages = json!([
{"role": "system", "content": "plain"},
{"role": "assistant"},
{"role": "assistant", "content": [
{"type": "text", "text": "a"},
{"type": "text", "text": ""},
{"type": "refusal", "refusal": ""},
{"type": "refusal", "refusal": "b"}
]}
]);
assert_eq!(
flatten(messages, false).unwrap(),
json!(["plain", "", "a\nb"])
);
let messages = json!([{"role": "user", "content": [
{"type": "text", "text": "see <|image_1|> here"}, &image, &image
]}]);
assert_eq!(
flatten(messages, false).unwrap(),
json!(["<|image_2|>\nsee <|image_1|> here"])
);
let messages = json!([{"role": "user", "content": [
{"type": "video_url", "video_url": {"url": "y"}},
{"type": "video_url", "video_url": null, "uuid": "cached"},
{"type": "video_url", "video_url": {"url": ""}, "uuid": "cached"},
{"type": "input_audio", "input_audio": {"data": "", "format": "wav"}},
{"type": "text", "text": "x"}
]}]);
assert_eq!(
flatten(messages, true).unwrap(),
json!(["<video>\n<video>\n<video>\n<|audio_1|>\n<|audio_2|>\nx"])
);
for content in [
json!([{"type": "text", "text": "<|image_1|><|image_1|>"}, &image]),
json!([{"type": "custom", "kind": "chemistry", "fields": {}}]),
] {
let error = flatten(json!([{"role": "user", "content": content}]), false).unwrap_err();
assert!(
matches!(
error.downcast_ref::<PromptRenderError>(),
Some(PromptRenderError::InvalidRequest(_))
),
"{error}"
);
}
}
#[test]
fn content_format_openai_wraps_content_in_parts_like_vllm() {
let mut messages = json!([
{"role": "system", "content": "sys"},
{"role": "user", "content": [
{"type": "text", "text": "look", "source_lang_code": "en"},
{"type": "image_url", "image_url": {"url": "x"}, "uuid": "i", "target_lang_code": "de"},
{"type": "input_audio", "input_audio": {"data": "", "format": "wav"}},
{"type": "custom", "kind": "chemistry", "fields": {"data_base64": "x"}}
]},
{"role": "assistant", "content": [{"type": "refusal", "refusal": "no"}]},
{"role": "assistant"},
{"role": "tool", "tool_call_id": "1", "content": [
{"type": "text", "text": "a"},
{"type": "text", "text": "b"}
]},
{"role": "tool", "tool_call_id": "2", "content": "done"}
]);
wrap_msg_content_in_parts(&mut messages);
assert_eq!(
contents(&messages),
json!([
[{"type": "text", "text": "sys"}],
[
{"type": "text", "text": "look", "source_lang_code": "en"},
{"type": "image", "target_lang_code": "de"},
{"type": "audio"},
{"type": "custom", "kind": "chemistry"}
],
[{"type": "text", "text": "no"}],
[],
"a\nb",
"done",
])
);
}
#[test]
fn content_format_reaches_render_through_from_parts() {
let template = "{% for m in messages %}{% if m.content is string %}{{ m.content }}\
{% else %}{% for p in m.content %}[{{ p.type }}{% if p.text %} {{ p.text }}{% endif %}]\
{% endfor %}{% endif %}|{% endfor %}";
let request: NvCreateChatCompletionRequest = serde_json::from_value(json!({
"model": "test",
"messages": [
{"role": "system", "content": "sys"},
{"role": "user", "content": [
{"type": "text", "text": "hi"},
{"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}},
{"type": "video_url", "video_url": {"url": "https://example.com/v.mp4"}}
]}
],
"mm_processor_kwargs": {"use_audio_in_video": 1}
}))
.unwrap();
let render = |content_format| {
let config: SysChatTemplate =
serde_json::from_value(json!({ "chat_template": template })).unwrap();
let crate::PromptFormatter::OAI(formatter) =
crate::PromptFormatter::from_parts_with_content_format(
config,
SysMixins::new(&[]),
true,
content_format,
)
.unwrap();
formatter.render(&request).unwrap()
};
assert_eq!(
render(ChatTemplateContentFormat::Auto),
"sys|[text hi][image][video]|"
);
assert_eq!(
render(ChatTemplateContentFormat::OpenAi),
"[text sys]|[text hi][image][video]|"
);
let placeholders = MediaPlaceholders {
image: "<image>".into(),
video: "<video>".into(),
audio: "<audio>".into(),
};
assert_eq!(
render(ChatTemplateContentFormat::String(placeholders)),
"sys|<image>\n<video>\n<audio>\nhi|"
);
}
}