1use super::*;
5
6use crate::{OAIChatLikeRequest, PromptRenderError, TextInput};
7use minijinja::{context, value::Value};
8use serde_json::json;
9use std::{collections::HashMap, result::Result::Ok};
10
11pub fn may_be_fix_tool_schema(tools: serde_json::Value) -> Option<Value> {
15 let mut updated_tools = Vec::new();
19 if let Some(arr) = tools.as_array() {
20 for tool in arr {
21 let mut tool = tool.clone();
22 if let Some(function) = tool.get_mut("function") {
23 if let Some(obj) = function.as_object_mut()
28 && !matches!(obj.get("description"), Some(serde_json::Value::String(_)))
29 {
30 obj.insert(
31 "description".to_string(),
32 serde_json::Value::String(String::new()),
33 );
34 }
35 }
36 if let Some(function) = tool.get_mut("function")
37 && let Some(parameters) = function.get_mut("parameters")
38 {
39 if parameters.is_object() {
41 let mut needs_type = false;
42 let mut needs_properties = false;
43 let is_empty = parameters
44 .as_object()
45 .map(|o| o.is_empty())
46 .unwrap_or(false);
47
48 if is_empty {
50 needs_type = true;
51 needs_properties = true;
52 } else {
53 if let Some(obj) = parameters.as_object() {
55 if !obj.contains_key("type") {
56 needs_type = true;
57 }
58 if !obj.contains_key("properties") {
59 needs_properties = true;
60 }
61 }
62 }
63
64 if (needs_type || needs_properties)
65 && let Some(obj) = parameters.as_object_mut()
66 {
67 if needs_type {
68 obj.insert(
69 "type".to_string(),
70 serde_json::Value::String("object".to_string()),
71 );
72 }
73 if needs_properties {
74 obj.insert(
75 "properties".to_string(),
76 serde_json::Value::Object(Default::default()),
77 );
78 }
79 }
80 }
81 }
82 updated_tools.push(tool);
83 }
84 }
85 Some(crate::python::template_value(&serde_json::Value::Array(
86 updated_tools,
87 )))
88}
89
90const DEFAULT_MEDIA_TYPE_CONVERSIONS: &[(&str, &str)] = &[
93 ("image_url", "image"),
94 ("video_url", "video"),
95 ("audio_url", "audio"),
96];
97
98fn custom_marker(part: &serde_json::Value) -> serde_json::Value {
100 json!({"type": "custom", "kind": part.get("kind")})
101}
102
103fn convert_media_url_to_placeholder(
105 content_array: &mut [serde_json::Value],
106 conversions: &[(&str, &str)],
107) {
108 for part in content_array {
109 let part_type = part.get("type").and_then(|t| t.as_str()).unwrap_or("");
110 if part_type == "custom" {
111 *part = custom_marker(part);
112 } else if let Some((_, target_type)) = conversions.iter().find(|(src, _)| *src == part_type)
113 {
114 *part = serde_json::json!({"type": target_type});
115 }
116 }
117}
118
119fn may_be_fix_msg_content(
120 mut messages: serde_json::Value,
121 preserve_arrays: bool,
122 native_text_array_roles: &[&str],
123 merged_turns: &[usize],
124 image_placeholder_template: Option<&str>,
125) -> serde_json::Value {
126 let Some(arr) = messages.as_array_mut() else {
127 return messages;
128 };
129 for (index, msg) in arr.iter_mut().enumerate() {
130 let preserve_text_array = !merged_turns.contains(&index)
133 && msg
134 .get("role")
135 .and_then(serde_json::Value::as_str)
136 .is_some_and(|role| native_text_array_roles.contains(&role));
137 let Some(content) = msg.get_mut("content") else {
138 continue;
139 };
140 match content {
141 serde_json::Value::String(_) if preserve_arrays => {
142 let text = content.take();
143 *content = serde_json::Value::Array(vec![serde_json::Value::Object(
144 serde_json::Map::from_iter([
145 ("type".into(), serde_json::Value::String("text".into())),
146 ("text".into(), text),
147 ]),
148 )]);
149 }
150 serde_json::Value::Array(parts) => {
151 convert_media_url_to_placeholder(parts, DEFAULT_MEDIA_TYPE_CONVERSIONS);
152 let text_only = !parts.is_empty()
155 && parts
156 .iter()
157 .all(|part| part.get("type").and_then(|v| v.as_str()) == Some("text"));
158 if text_only && !preserve_arrays {
159 if !preserve_text_array {
160 let text = parts
161 .iter()
162 .filter_map(|part| part.get("text")?.as_str())
163 .collect::<Vec<_>>()
164 .join("\n");
165 *content = serde_json::Value::String(text);
166 }
167 } else if !preserve_arrays
168 && !parts.iter().any(|part| part.get("type").and_then(|v| v.as_str()) == Some("custom"))
171 && !parts.is_empty()
172 && let Some(template) = image_placeholder_template
173 {
174 *content = serde_json::Value::String(flatten_mixed_content(parts, template));
175 }
176 }
177 _ => {}
178 }
179 }
180 messages
181}
182
183fn flatten_mixed_content(parts: &[serde_json::Value], placeholder_tpl: &str) -> String {
206 let mut out = String::new();
207 let mut img_idx: u32 = 1;
208 for part in parts {
209 let type_str = part.get("type").and_then(|t| t.as_str()).unwrap_or("");
210 if type_str == "text" {
211 if let Some(text) = part.get("text").and_then(|t| t.as_str()) {
212 out.push_str(text);
213 }
214 } else if !type_str.is_empty() {
215 let placeholder = placeholder_tpl.replace("{n}", &img_idx.to_string());
216 out.push_str(&placeholder);
217 img_idx += 1;
218 }
219 }
220 out
221}
222
223enum ContentPart<'a> {
225 Text(Option<&'a str>),
227 Media(&'static str),
229 Custom,
231 Other,
232}
233
234fn content_part(part: &serde_json::Value) -> ContentPart<'_> {
236 let field = |name: &str| part.get(name).and_then(serde_json::Value::as_str);
237 match part.get("type").and_then(serde_json::Value::as_str) {
238 Some("text") => ContentPart::Text(field("text")),
239 Some("refusal") => ContentPart::Text(field("refusal")),
240 Some("image_url") => ContentPart::Media("image"),
241 Some("video_url") => ContentPart::Media("video"),
242 Some("audio_url" | "input_audio") => ContentPart::Media("audio"),
243 Some("custom") => ContentPart::Custom,
244 _ => ContentPart::Other,
245 }
246}
247
248fn flatten_msg_content_to_string(
252 messages: &mut serde_json::Value,
253 media_placeholders: &MediaPlaceholders,
254 use_audio_in_video: bool,
255) -> Result<()> {
256 let Some(msgs) = messages.as_array_mut() else {
257 return Ok(());
258 };
259 let mut media_counts = HashMap::new();
260 for msg in msgs {
261 let Some(msg) = msg.as_object_mut() else {
262 continue;
263 };
264 let content = match msg.get("content") {
265 Some(serde_json::Value::String(_)) => continue,
266 Some(serde_json::Value::Array(parts)) => flatten_parts_to_string(
267 parts,
268 media_placeholders,
269 use_audio_in_video,
270 &mut media_counts,
271 )?,
272 _ => String::new(),
274 };
275 msg.insert("content".to_string(), serde_json::Value::String(content));
276 }
277 Ok(())
278}
279
280fn flatten_parts_to_string(
285 parts: &[serde_json::Value],
286 media_placeholders: &MediaPlaceholders,
287 use_audio_in_video: bool,
288 media_counts: &mut HashMap<&'static str, usize>,
289) -> Result<String> {
290 let mut texts = Vec::new();
291 let mut by_modality: Vec<(&str, Vec<String>)> = Vec::new();
292 for part in parts {
293 match content_part(part) {
294 ContentPart::Text(text) => texts.extend(text.filter(|text| !text.is_empty())),
295 ContentPart::Media(modality) => {
296 let audio_track = modality == "video"
298 && use_audio_in_video
299 && part["video_url"]["url"]
300 .as_str()
301 .is_some_and(|url| !url.is_empty());
302 for modality in std::iter::once(modality).chain(audio_track.then_some("audio")) {
303 let n = media_counts.entry(modality).or_default();
304 *n += 1;
305 let template = match modality {
306 "image" => &media_placeholders.image,
307 "video" => &media_placeholders.video,
308 _ => &media_placeholders.audio,
309 };
310 if template.is_empty() {
311 continue;
312 }
313 let placeholder = template.replace("{n}", &n.to_string());
314 match by_modality.iter_mut().find(|(m, _)| *m == modality) {
315 Some((_, placeholders)) => placeholders.push(placeholder),
316 None => by_modality.push((modality, vec![placeholder])),
317 }
318 }
319 }
320 ContentPart::Custom | ContentPart::Other => {
321 return Err(PromptRenderError::invalid_request(format!(
322 "content part type {} is not supported with the `string` chat template content format",
323 part["type"]
324 ))
325 .into());
326 }
327 }
328 }
329 let text = texts.join("\n");
330
331 let mut counts: Vec<(String, usize)> = Vec::new();
333 for placeholder in by_modality.into_iter().flat_map(|(_, p)| p) {
334 match counts.iter_mut().find(|(p, _)| *p == placeholder) {
335 Some((_, count)) => *count += 1,
336 None => counts.push((placeholder, 1)),
337 }
338 }
339 let mut lines = Vec::new();
340 for (placeholder, count) in counts {
341 let present = text.matches(placeholder.as_str()).count();
342 if present > count {
343 return Err(PromptRenderError::invalid_request(format!(
344 "found more '{placeholder}' placeholders in the message text than media items"
345 ))
346 .into());
347 }
348 lines.extend(std::iter::repeat_n(placeholder, count - present));
349 }
350 if lines.is_empty() {
351 return Ok(text);
352 }
353 if !text.is_empty() {
354 lines.push(text);
355 }
356 Ok(lines.join("\n"))
357}
358
359const KNOWN_CONTENT_PART_FIELDS: &[&str] = &[
363 "type",
364 "text",
365 "refusal",
366 "image_url",
367 "video_url",
368 "audio_url",
369 "input_audio",
370 "uuid",
371 "audio_embeds",
372 "closed",
373 "data",
374 "file",
375 "image_embeds",
376 "image_pil",
377 "name",
378 "prompt_cache_breakpoint",
379 "thinking",
380 "video_embeds",
381];
382
383fn openai_part(part: serde_json::Value) -> Option<serde_json::Value> {
386 let mut wrapped = match content_part(&part) {
387 ContentPart::Text(text) => json!({"type": "text", "text": text?}),
388 ContentPart::Media(modality) => json!({"type": modality}),
389 ContentPart::Custom => return Some(custom_marker(&part)),
390 ContentPart::Other => return Some(part),
392 };
393 if let serde_json::Value::Object(fields) = part {
394 for (key, value) in fields {
395 if !KNOWN_CONTENT_PART_FIELDS.contains(&key.as_str()) {
396 wrapped[key.as_str()] = value;
397 }
398 }
399 }
400 Some(wrapped)
401}
402
403fn wrap_msg_content_in_parts(messages: &mut serde_json::Value) {
407 let Some(msgs) = messages.as_array_mut() else {
408 return;
409 };
410 for msg in msgs {
411 let is_tool = msg.get("role").and_then(serde_json::Value::as_str) == Some("tool");
412 let parts: Vec<serde_json::Value> = match take_message_content(msg) {
413 serde_json::Value::String(text) => {
414 let mut part = json!({"type": "text"});
415 part["text"] = text.into();
416 vec![part]
417 }
418 serde_json::Value::Array(parts) => parts.into_iter().filter_map(openai_part).collect(),
419 _ => Vec::new(),
420 };
421 let Some(msg) = msg.as_object_mut() else {
422 continue;
423 };
424 let all_text = parts.iter().all(|part| part["type"] == "text");
425 let content = if is_tool && all_text {
426 let texts: Vec<&str> = parts.iter().filter_map(|p| p["text"].as_str()).collect();
427 serde_json::Value::String(texts.join("\n"))
428 } else {
429 serde_json::Value::Array(parts)
430 };
431 msg.insert("content".to_string(), content);
432 }
433}
434
435fn normalize_tool_calls_arguments_in_messages(messages: &mut serde_json::Value) {
436 let Some(msgs) = messages.as_array_mut() else {
441 return;
442 };
443
444 for msg in msgs.iter_mut() {
445 if let Some(tool_calls) = msg.get_mut("tool_calls").and_then(|v| v.as_array_mut()) {
446 for tc in tool_calls {
447 if let Some(function) = tc.get_mut("function").and_then(|v| v.as_object_mut())
448 && let Some(args) = function.get_mut("arguments")
449 && let Some(s) = args.as_str()
450 && let Ok(parsed) = serde_json::from_str(s)
451 {
452 *args = parsed;
453 }
454 }
455 }
456 }
457}
458
459fn normalize_function_call_arguments_in_messages(messages: &mut serde_json::Value) {
460 let Some(msgs) = messages.as_array_mut() else {
465 return;
466 };
467
468 for msg in msgs.iter_mut() {
469 if let Some(function_call) = msg.get_mut("function_call").and_then(|v| v.as_object_mut())
470 && let Some(args) = function_call.get_mut("arguments")
471 && let Some(s) = args.as_str()
472 && let Ok(parsed) = serde_json::from_str(s)
473 {
474 *args = parsed;
475 }
476 }
477}
478
479fn inject_reasoning_content_into_messages(messages: &mut serde_json::Value) {
493 let Some(msgs) = messages.as_array_mut() else {
494 return;
495 };
496
497 for msg in msgs.iter_mut() {
498 if msg.get("role").and_then(|r| r.as_str()) != Some("assistant") {
499 continue;
500 }
501
502 let reasoning = match msg.get("reasoning_content") {
503 Some(serde_json::Value::String(s)) if !s.is_empty() => {
504 format!("<think>{}</think>", s)
505 }
506 Some(serde_json::Value::Array(segments)) => {
507 let mut result = String::new();
508 for seg in segments {
509 if let Some(s) = seg.as_str()
510 && !s.is_empty()
511 {
512 result.push_str("<think>");
513 result.push_str(s);
514 result.push_str("</think>");
515 }
516 }
517 if result.is_empty() {
518 continue;
519 }
520 result
521 }
522 _ => continue,
523 };
524
525 match msg.get("content") {
526 Some(serde_json::Value::String(s)) if !s.is_empty() => {
528 msg["content"] = serde_json::Value::String(format!("{}{}", reasoning, s));
529 }
530 None | Some(serde_json::Value::Null) | Some(serde_json::Value::String(_)) => {
531 msg["content"] = serde_json::Value::String(reasoning);
532 }
533 Some(serde_json::Value::Array(_)) => {
535 let think_part = serde_json::json!({
536 "type": "text",
537 "text": reasoning
538 });
539 if let Some(arr) = msg.get_mut("content").and_then(|v| v.as_array_mut()) {
540 arr.insert(0, think_part);
541 }
542 }
543 _ => continue,
545 }
546
547 if let Some(obj) = msg.as_object_mut() {
550 obj.remove("reasoning_content");
551 }
552 }
553}
554
555fn join_reasoning_content_segments_in_messages(messages: &mut serde_json::Value) {
561 let Some(msgs) = messages.as_array_mut() else {
562 return;
563 };
564
565 for msg in msgs.iter_mut() {
566 if msg.get("role").and_then(|r| r.as_str()) != Some("assistant") {
567 continue;
568 }
569 if let Some(reasoning) = msg.get_mut("reasoning_content")
570 && let Some(segments) = reasoning.as_array()
571 {
572 let joined = segments
573 .iter()
574 .filter_map(|s| s.as_str())
575 .filter(|s| !s.is_empty())
576 .collect::<Vec<_>>()
577 .join("\n");
578 *reasoning = joined.into();
579 }
580 }
581}
582
583impl OAIChatLikeRequest for dynamo_protocols::types::CreateChatCompletionRequest {
589 fn model(&self) -> String {
590 self.model.clone()
591 }
592
593 fn messages(&self) -> Value {
594 let messages_json = self.function_template_messages();
595 Value::from_serialize(&messages_json)
596 }
597
598 fn typed_messages(&self) -> Option<&[dynamo_protocols::types::ChatCompletionRequestMessage]> {
599 Some(self.messages.as_slice())
600 }
601
602 fn tools(&self) -> Option<Value> {
603 let tools = self.tools.as_ref()?;
604 may_be_fix_tool_schema(
605 serde_json::to_value(
606 tools
607 .iter()
608 .map(|tool| tool.function_template_tool())
609 .collect::<Vec<_>>(),
610 )
611 .unwrap(),
612 )
613 }
614
615 fn raw_tools(&self) -> Option<Value> {
616 self.tools.as_ref().map(Value::from_serialize)
617 }
618
619 fn tool_choice(&self) -> Option<Value> {
620 self.tool_choice
621 .as_ref()
622 .map(|choice| Value::from_serialize(choice.function_template_choice()))
623 }
624
625 fn response_format(&self) -> Option<Value> {
626 self.response_format.as_ref().map(Value::from_serialize)
627 }
628
629 fn reasoning_effort(&self) -> Option<Value> {
630 self.reasoning_effort.as_ref().map(Value::from_serialize)
631 }
632
633 fn should_add_generation_prompt(&self) -> bool {
634 true
636 }
637
638 fn extract_text(&self) -> Option<TextInput> {
639 Some(TextInput::Single(String::new()))
640 }
641
642 fn mm_processor_kwargs(&self) -> Option<&serde_json::Value> {
643 self.mm_processor_kwargs.as_ref()
644 }
645}
646
647fn merge_message_content(
651 target: serde_json::Value,
652 source: serde_json::Value,
653) -> serde_json::Value {
654 use serde_json::Value;
655 let text_part = |text: String| json!({"type": "text", "text": text});
656 match (target, source) {
657 (Value::String(mut target), Value::String(source)) => {
658 if !target.is_empty() && !source.is_empty() {
659 target.push_str("\n\n");
660 }
661 target.push_str(&source);
662 Value::String(target)
663 }
664 (Value::Array(mut target), Value::Array(source)) => {
665 target.extend(source);
666 Value::Array(target)
667 }
668 (Value::Array(mut target), Value::String(source)) => {
669 if !source.is_empty() {
670 target.push(text_part(source));
671 }
672 Value::Array(target)
673 }
674 (Value::String(target), Value::Array(source)) => {
675 let mut parts = Vec::with_capacity(source.len() + 1);
676 if !target.is_empty() {
677 parts.push(text_part(target));
678 }
679 parts.extend(source);
680 Value::Array(parts)
681 }
682 (Value::Null, source) => source,
683 (target, _) => target,
685 }
686}
687
688fn append_message_content(target: &mut serde_json::Value, source: serde_json::Value) {
690 let Some(target) = target.as_object_mut() else {
691 return;
692 };
693 let merged = merge_message_content(
694 target.remove("content").unwrap_or(serde_json::Value::Null),
695 source,
696 );
697 target.insert("content".to_string(), merged);
698}
699
700fn take_message_content(message: &mut serde_json::Value) -> serde_json::Value {
701 message
702 .get_mut("content")
703 .map(serde_json::Value::take)
704 .unwrap_or(serde_json::Value::Null)
705}
706
707fn normalize_system_messages(
712 messages: &mut serde_json::Value,
713 rules: SystemNormalization,
714) -> Vec<usize> {
715 let mut merged_turns = Vec::new();
716 let serde_json::Value::Array(list) = messages else {
717 return merged_turns;
718 };
719 let role_is =
720 |m: &serde_json::Value, r: &str| m.get("role").and_then(|v| v.as_str()) == Some(r);
721
722 if rules.demote_nonleading_system {
723 let leading = list.iter().take_while(|m| role_is(m, "system")).count();
726 if leading > 1 {
727 for mut trailing in list.drain(1..leading).collect::<Vec<_>>() {
728 let content = take_message_content(&mut trailing);
729 append_message_content(&mut list[0], content);
730 }
731 merged_turns.push(0);
732 }
733
734 let leading = list.iter().take_while(|m| role_is(m, "system")).count();
737 for m in list.iter_mut().skip(leading) {
738 if role_is(m, "system")
739 && let Some(m) = m.as_object_mut()
740 {
741 m.insert("role".to_string(), json!("user"));
742 }
743 }
744 }
745
746 if rules.coalesce_consecutive_users {
747 let mut coalesced: Vec<serde_json::Value> = Vec::with_capacity(list.len());
748 for mut m in list.drain(..) {
749 if role_is(&m, "user") && coalesced.last().is_some_and(|p| role_is(p, "user")) {
750 let content = take_message_content(&mut m);
751 append_message_content(coalesced.last_mut().unwrap(), content);
752 let index = coalesced.len() - 1;
753 if merged_turns.last() != Some(&index) {
754 merged_turns.push(index);
755 }
756 } else {
757 coalesced.push(m);
758 }
759 }
760 *list = coalesced;
761 }
762 merged_turns
763}
764
765impl HfTokenizerConfigJsonFormatter {
766 fn template_handles_native_tools(&self, has_tools: bool) -> bool {
767 if has_tools {
768 self.tool_use_template_handles_native_tools
769 } else {
770 self.default_template_handles_native_tools
771 }
772 }
773
774 fn render_with_token_policy(
775 &self,
776 req: &dyn OAIChatLikeRequest,
777 protect_literals: bool,
778 ) -> Result<crate::RenderedPrompt> {
779 let mixins = Value::from_dyn_object(self.mixins.clone());
780
781 let exclude = self.exclude_tools_when_tool_choice_none
782 && req.tool_choice().as_ref().and_then(Value::as_str) == Some("none");
783 let flat_tools = if exclude { None } else { req.tools() };
784 let has_tools = flat_tools
786 .as_ref()
787 .and_then(|tools| tools.len())
788 .is_some_and(|len| len > 0);
789 let tools = if !exclude && self.template_handles_native_tools(has_tools) {
793 req.template_tools().or(flat_tools)
794 } else {
795 flat_tools
796 };
797 let add_generation_prompt = req.should_add_generation_prompt();
798
799 tracing::trace!(
800 "Rendering prompt with tools: {:?}, add_generation_prompt: {}",
801 has_tools,
802 add_generation_prompt
803 );
804
805 let (
808 template_name,
809 template_handles_tool_calls_args_string,
810 template_handles_reasoning,
811 template_requires_reasoning_string,
812 system_normalization,
813 native_text_array_roles,
814 ) = if has_tools {
815 (
816 "tool_use",
817 self.tool_use_template_handles_tool_calls_arguments_string,
818 self.tool_use_template_handles_reasoning,
819 self.tool_use_template_requires_reasoning_string,
820 self.tool_use_system_normalization,
821 &self.tool_use_native_text_array_roles,
822 )
823 } else {
824 (
825 "default",
826 self.default_template_handles_tool_calls_arguments_string,
827 self.default_template_handles_reasoning,
828 self.default_template_requires_reasoning_string,
829 self.default_system_normalization,
830 &self.default_native_text_array_roles,
831 )
832 };
833
834 let mut messages_for_template = crate::messages_to_json(req)?;
835
836 crate::reject_unsupported_partial_assistant(&messages_for_template)?;
837 crate::reject_unsupported_message_tools(&messages_for_template, &[])?;
838
839 let mut normalized_turns = if system_normalization.is_required() {
840 normalize_system_messages(&mut messages_for_template, system_normalization)
841 } else {
842 Vec::new()
843 };
844
845 match &self.content_format {
846 ChatTemplateContentFormat::Auto => {
847 if !template_handles_reasoning
848 && let Some(messages) = messages_for_template.as_array()
849 {
850 normalized_turns.extend(messages.iter().enumerate().filter_map(
851 |(index, message)| {
852 if message.get("role").and_then(serde_json::Value::as_str)
853 != Some("assistant")
854 {
855 return None;
856 }
857 let has_reasoning = match message.get("reasoning_content") {
858 Some(serde_json::Value::String(text)) => !text.is_empty(),
859 Some(serde_json::Value::Array(segments)) => {
860 segments.iter().any(|segment| {
861 segment.as_str().is_some_and(|text| !text.is_empty())
862 })
863 }
864 _ => false,
865 };
866 has_reasoning.then_some(index)
867 },
868 ));
869 }
870 messages_for_template = may_be_fix_msg_content(
871 messages_for_template,
872 self.requires_content_arrays,
873 native_text_array_roles,
874 &normalized_turns,
875 self.image_placeholder_template,
876 );
877 }
878 ChatTemplateContentFormat::String(media_placeholders) => {
879 let use_audio_in_video = req
880 .mm_processor_kwargs()
881 .and_then(|kwargs| kwargs.get("use_audio_in_video"))
882 .is_some_and(|flag| Value::from_serialize(flag).is_true());
884 flatten_msg_content_to_string(
885 &mut messages_for_template,
886 media_placeholders,
887 use_audio_in_video,
888 )?;
889 }
890 ChatTemplateContentFormat::OpenAi => {
891 wrap_msg_content_in_parts(&mut messages_for_template);
892 }
893 }
894
895 if !template_handles_tool_calls_args_string {
902 normalize_tool_calls_arguments_in_messages(&mut messages_for_template);
903 }
904 normalize_function_call_arguments_in_messages(&mut messages_for_template);
908
909 if !template_handles_reasoning {
914 inject_reasoning_content_into_messages(&mut messages_for_template);
915 } else if template_requires_reasoning_string {
916 join_reasoning_content_segments_in_messages(&mut messages_for_template);
917 }
918
919 let tmpl = self.env.get_template(template_name)?;
920 let mut literals = HarmonyLiterals::default();
921 let mut tools = tools;
922 let mut response_format = req
923 .response_format()
924 .as_ref()
925 .map(crate::python::template_json)
926 .transpose()?;
927 let mut placeholder = "__DYNAMO_HARMONY_LITERAL_".to_string();
928 if protect_literals
929 && tmpl.source().contains("<|start|>")
930 && tmpl.source().contains("<|message|>")
931 {
932 let mut tool_json = tools
935 .as_ref()
936 .map(crate::python::template_json)
937 .transpose()?;
938 let collision_source = format!(
939 "{messages_for_template}{tool_json:?}{response_format:?}{:?}{}",
940 req.chat_template_args(),
941 tmpl.source()
942 );
943 while collision_source.contains(&placeholder) {
944 placeholder.push('_');
945 }
946 shield_harmony_literals(&mut messages_for_template, &placeholder, &mut literals);
947 if let Some(value) = &mut tool_json {
948 shield_harmony_literals(value, &placeholder, &mut literals);
949 tools = Some(crate::python::template_value(value));
950 }
951 if let Some(value) = &mut response_format {
952 shield_harmony_literals(value, &placeholder, &mut literals);
953 }
954 }
955
956 let ctx = context! {
957 messages => crate::python::template_value(&messages_for_template),
958 tools => tools.as_ref().map(crate::python::template_json).transpose()?.as_ref().map(crate::python::template_value),
959 response_format => response_format.as_ref()
960 .map(crate::python::template_value)
961 .unwrap_or(Value::UNDEFINED),
962 bos_token => self.config.bos_tok(),
963 eos_token => self.config.eos_tok(),
964 unk_token => self.config.unk_tok(),
965 add_generation_prompt => add_generation_prompt,
966 ..mixins
967 };
968
969 let ctx = if let Some(args) = req.chat_template_args() {
971 let extra = crate::python::template_value(&serde_json::to_value(args)?);
972 context! { ..ctx, ..extra }
973 } else {
974 ctx
975 };
976
977 let rendered = tmpl.render(&ctx)?;
978 if literals.values.is_empty() {
979 return Ok(crate::RenderedPrompt::text(rendered));
980 }
981 let mut segments = Vec::new();
982 let mut remaining = rendered.as_str();
983 while let Some(start) = remaining.find(&placeholder) {
984 segments.push(crate::RenderedSegment::new(&remaining[..start], true));
985 let after = &remaining[start + placeholder.len()..];
986 let (index, suffix) = after
987 .split_once('_')
988 .ok_or_else(|| anyhow::anyhow!("damaged Harmony literal placeholder"))?;
989 let index: usize = index.parse()?;
990 segments.push(crate::RenderedSegment::new(
991 literals
992 .values
993 .get(index)
994 .ok_or_else(|| anyhow::anyhow!("unknown Harmony literal placeholder"))?,
995 false,
996 ));
997 remaining = suffix;
998 }
999 segments.push(crate::RenderedSegment::new(remaining, true));
1000 Ok(crate::RenderedPrompt::segmented(segments))
1001 }
1002}
1003
1004#[derive(Default)]
1007struct HarmonyLiterals {
1008 values: Vec<String>,
1009 indices: HashMap<String, usize>,
1010}
1011
1012impl HarmonyLiterals {
1013 fn index(&mut self, text: &str) -> usize {
1014 if let Some(index) = self.indices.get(text) {
1015 return *index;
1016 }
1017 let index = self.values.len();
1018 self.values.push(text.to_owned());
1019 self.indices.insert(text.to_owned(), index);
1020 index
1021 }
1022}
1023
1024fn shield_harmony_literals(
1025 value: &mut serde_json::Value,
1026 prefix: &str,
1027 literals: &mut HarmonyLiterals,
1028) {
1029 match value {
1030 serde_json::Value::String(text) => shield_harmony_text(text, prefix, literals),
1031 serde_json::Value::Array(values) => {
1032 for value in values {
1033 shield_harmony_literals(value, prefix, literals);
1034 }
1035 }
1036 serde_json::Value::Object(values) => {
1037 if values.keys().any(|key| key.contains("<|")) {
1038 for (mut key, mut value) in std::mem::take(values) {
1039 shield_harmony_text(&mut key, prefix, literals);
1040 shield_harmony_literals(&mut value, prefix, literals);
1041 values.insert(key, value);
1042 }
1043 } else {
1044 for value in values.values_mut() {
1045 shield_harmony_literals(value, prefix, literals);
1046 }
1047 }
1048 }
1049 _ => {}
1050 }
1051}
1052
1053fn shield_harmony_text(text: &mut String, prefix: &str, literals: &mut HarmonyLiterals) {
1054 let mut protected = String::new();
1055 let mut remaining = text.as_str();
1056 while let Some(start) = remaining.find("<|") {
1057 let Some(end) = remaining[start + 2..].find("|>") else {
1058 break;
1059 };
1060 let end = start + 2 + end + 2;
1061 let marker = &remaining[start..end];
1062 if marker
1064 .chars()
1065 .any(|c| c == '"' || c == '\\' || c <= '\u{1f}')
1066 {
1067 protected.push_str(&remaining[..start + 2]);
1068 remaining = &remaining[start + 2..];
1069 continue;
1070 }
1071 protected.push_str(&remaining[..start]);
1072 protected.push_str(&format!("{prefix}{}_", literals.index(marker)));
1073 remaining = &remaining[end..];
1074 }
1075 if remaining.len() != text.len() {
1076 protected.push_str(remaining);
1077 *text = protected;
1078 }
1079}
1080
1081impl OAIPromptFormatter for HfTokenizerConfigJsonFormatter {
1082 fn supports_native_tools(&self, request: &dyn OAIChatLikeRequest) -> bool {
1083 let exclude = self.exclude_tools_when_tool_choice_none
1085 && request.tool_choice().as_ref().and_then(Value::as_str) == Some("none");
1086 let has_tools = !exclude
1087 && request
1088 .tools()
1089 .as_ref()
1090 .and_then(|tools| tools.len())
1091 .is_some_and(|len| len > 0);
1092 self.template_handles_native_tools(has_tools)
1093 }
1094
1095 fn supports_add_generation_prompt(&self) -> bool {
1096 self.supports_add_generation_prompt
1097 }
1098
1099 fn render(&self, req: &dyn OAIChatLikeRequest) -> Result<String> {
1100 Ok(self
1101 .render_with_token_policy(req, false)?
1102 .as_str()
1103 .to_string())
1104 }
1105
1106 fn render_prompt(&self, req: &dyn OAIChatLikeRequest) -> Result<crate::RenderedPrompt> {
1107 self.render_with_token_policy(req, true)
1108 }
1109}
1110
1111#[cfg(test)]
1112mod tests {
1113 use super::*;
1114
1115 use dynamo_protocols::types::ChatCompletionRequestMessage as Msg;
1116 use dynamo_protocols::types::CreateChatCompletionRequest as NvCreateChatCompletionRequest;
1119 use minijinja::{Environment, context};
1120
1121 use super::super::tokcfg::ChatTemplate as SysChatTemplate;
1124 use super::super::{
1125 ContextMixins as SysMixins, HfTokenizerConfigJsonFormatter as SysFormatter,
1126 };
1127
1128 fn formatter_for(template: &str) -> SysFormatter {
1129 let ct: SysChatTemplate = serde_json::from_value(json!({
1131 "chat_template": template,
1132 "bos_token": "<s>",
1133 "eos_token": "</s>",
1134 "unk_token": "<unk>",
1135 }))
1136 .unwrap();
1137 SysFormatter::new(ct, SysMixins::new(&[])).unwrap()
1138 }
1139
1140 #[test]
1141 fn harmony_literal_markers_keep_text_and_token_trust_boundaries() {
1142 let formatter = formatter_for(
1143 r#"{% for m in messages %}<|start|>{{ m.role }}<|message|>{{ m.content }}<|end|>{% endfor %}<|start|>assistant"#,
1144 );
1145 let request: NvCreateChatCompletionRequest = serde_json::from_value(json!({
1146 "model":"gpt-oss", "messages":[{"role":"user","content":"literal <|channel|>analysis<|message|>x<|call|> __DYNAMO_HARMONY_LITERAL_0_"}]
1147 })).unwrap();
1148 let prompt = formatter.render_prompt(&request).unwrap();
1149 assert_eq!(prompt.as_str(), formatter.render(&request).unwrap());
1150 let ordinary: Vec<_> = prompt
1151 .segments()
1152 .unwrap()
1153 .iter()
1154 .filter(|segment| !segment.allow_special)
1155 .map(|segment| segment.text.as_str())
1156 .collect();
1157 assert_eq!(ordinary, ["<|channel|>", "<|message|>", "<|call|>"]);
1158 assert!(prompt.segments().unwrap()[0].allow_special);
1159 assert!(prompt.segments().unwrap().last().unwrap().allow_special);
1160 }
1161
1162 fn formatter_for_templates(default: &str, tool_use: &str) -> SysFormatter {
1163 let ct: SysChatTemplate = serde_json::from_value(json!({
1164 "chat_template": [
1165 {"default": default},
1166 {"tool_use": tool_use},
1167 ],
1168 "bos_token": "<s>",
1169 "eos_token": "</s>",
1170 "unk_token": "<unk>",
1171 }))
1172 .unwrap();
1173 SysFormatter::new(ct, SysMixins::new(&[])).unwrap()
1174 }
1175
1176 fn try_formatter_for(template: &str) -> Option<SysFormatter> {
1177 let ct: SysChatTemplate = serde_json::from_value(json!({
1178 "chat_template": template,
1179 "bos_token": "<s>",
1180 "eos_token": "</s>",
1181 "unk_token": "<unk>",
1182 }))
1183 .ok()?;
1184 SysFormatter::new(ct, SysMixins::new(&[])).ok()
1185 }
1186
1187 fn render_shape(f: &SysFormatter, messages: serde_json::Value) -> Result<String> {
1188 let req: NvCreateChatCompletionRequest =
1189 serde_json::from_value(json!({ "model": "test", "messages": messages })).unwrap();
1190 f.render(&req)
1191 }
1192
1193 fn render_shape_with_tools(f: &SysFormatter, messages: serde_json::Value) -> Result<String> {
1194 let req: NvCreateChatCompletionRequest = serde_json::from_value(json!({
1195 "model": "test",
1196 "messages": messages,
1197 "tools": [{
1198 "type": "function",
1199 "function": {"name": "noop", "parameters": {}}
1200 }]
1201 }))
1202 .unwrap();
1203 f.render(&req)
1204 }
1205
1206 struct RawMessagesRequest(Value);
1207
1208 impl OAIChatLikeRequest for RawMessagesRequest {
1209 fn model(&self) -> String {
1210 "test".to_string()
1211 }
1212
1213 fn messages(&self) -> Value {
1214 self.0.clone()
1215 }
1216
1217 fn should_add_generation_prompt(&self) -> bool {
1218 true
1219 }
1220 }
1221
1222 fn render_raw_shape(f: &SysFormatter, messages: serde_json::Value) -> Result<String> {
1223 f.render(&RawMessagesRequest(Value::from_serialize(&messages)))
1224 }
1225
1226 #[test]
1227 fn content_normalization_preserves_unchanged_values() {
1228 let messages = json!([
1229 {"role": "user", "content": " 中文 <special>\n", "name": "user"},
1230 {"role": "assistant", "content": null},
1231 {"role": "user", "content": []},
1232 {"role": "assistant", "tool_calls": []}
1233 ]);
1234 assert_eq!(
1235 may_be_fix_msg_content(messages.clone(), false, &[], &[], Some("")),
1236 messages
1237 );
1238 }
1239
1240 const PERMISSIVE_TMPL: &str = concat!(
1241 "{%- for m in messages -%}",
1242 "<|im_start|>{{ m.role }}\n{{ m.content }}<|im_end|>\n",
1243 "{%- endfor -%}"
1244 );
1245
1246 #[test]
1247 fn jinja_templates_reject_message_level_tools() {
1248 let f = formatter_for(PERMISSIVE_TMPL);
1249 let error = render_shape(
1250 &f,
1251 json!([
1252 {"role": "system", "tools": [{"name": "lookup", "parameters": {"type": "object"}}]},
1253 {"role": "user", "content": "hi"}
1254 ]),
1255 )
1256 .unwrap_err();
1257 assert!(matches!(
1258 error.downcast_ref::<crate::PromptRenderError>(),
1259 Some(crate::PromptRenderError::InvalidRequest(message))
1260 if message.contains("message-level `tools`")
1261 ));
1262
1263 let error = render_raw_shape(
1264 &f,
1265 json!([{
1266 "role": "user",
1267 "content": "hi",
1268 "tools": [{"name": "lookup", "parameters": {"type": "object"}}]
1269 }]),
1270 )
1271 .unwrap_err();
1272 assert!(matches!(
1273 error.downcast_ref::<crate::PromptRenderError>(),
1274 Some(crate::PromptRenderError::InvalidRequest(message))
1275 if message.contains("message-level `tools`")
1276 ));
1277
1278 let rendered = render_shape(
1279 &f,
1280 json!([
1281 {"role": "system", "content": "You are helpful.", "tools": []},
1282 {"role": "user", "content": "hi"}
1283 ]),
1284 )
1285 .unwrap();
1286 assert!(rendered.contains("<|im_start|>system\nYou are helpful.<|im_end|>"));
1287 }
1288
1289 #[test]
1290 fn jinja_templates_reject_unsupported_partial_assistant() {
1291 let f = formatter_for(PERMISSIVE_TMPL);
1292 let error = render_shape(
1293 &f,
1294 json!([
1295 {"role": "user", "content": "Continue"},
1296 {"role": "assistant", "content": "prefix", "partial": true}
1297 ]),
1298 )
1299 .unwrap_err();
1300 assert!(matches!(
1301 error.downcast_ref::<crate::PromptRenderError>(),
1302 Some(crate::PromptRenderError::InvalidRequest(message))
1303 if message.contains("`partial: true` is not supported")
1304 ));
1305
1306 let rendered = render_shape(
1307 &f,
1308 json!([{"role": "assistant", "content": "ordinary", "partial": false}]),
1309 )
1310 .unwrap();
1311 assert!(rendered.contains("ordinary"));
1312 }
1313 const STRICT_LEADING_TMPL: &str = concat!(
1315 "{%- for m in messages -%}",
1316 "{%- if m.role == 'system' and not loop.first -%}",
1317 "{{ raise_exception('System message must be at the beginning.') }}",
1318 "{%- endif -%}",
1319 "<|im_start|>{{ m.role }}\n{{ m.content }}<|im_end|>\n",
1320 "{%- endfor -%}"
1321 );
1322 const ALTERNATION_TMPL: &str = concat!(
1324 "{%- set ns = namespace(prev='') -%}",
1325 "{%- for m in messages -%}",
1326 "{%- if m.role == 'user' and ns.prev == 'user' -%}",
1327 "{{ raise_exception('Conversation roles must alternate.') }}",
1328 "{%- endif -%}",
1329 "<|im_start|>{{ m.role }}\n{{ m.content }}<|im_end|>\n",
1330 "{%- set ns.prev = m.role -%}",
1331 "{%- endfor -%}"
1332 );
1333 const STRICT_BOTH_TMPL: &str = concat!(
1335 "{%- set ns = namespace(prev='') -%}",
1336 "{%- for m in messages -%}",
1337 "{%- if m.role == 'system' and not loop.first -%}",
1338 "{{ raise_exception('System message must be at the beginning.') }}",
1339 "{%- endif -%}",
1340 "{%- if m.role == 'user' and ns.prev == 'user' -%}",
1341 "{{ raise_exception('Conversation roles must alternate.') }}",
1342 "{%- endif -%}",
1343 "<|im_start|>{{ m.role }}\n{{ m.content }}<|im_end|>\n",
1344 "{%- set ns.prev = m.role -%}",
1345 "{%- endfor -%}"
1346 );
1347 const DEFAULT_NONE_GATED_TMPL: &str = concat!(
1350 "{%- set strict = tools is not none -%}",
1351 "{%- for m in messages -%}",
1352 "{%- if strict and m.role == 'system' and not loop.first -%}",
1353 "{{ raise_exception('System message must be at the beginning.') }}",
1354 "{%- endif -%}",
1355 "<|im_start|>{{ m.role }}\n{{ m.content }}<|im_end|>\n",
1356 "{%- endfor -%}"
1357 );
1358 const TOOL_NONEMPTY_GATED_TMPL: &str = concat!(
1359 "{%- set strict = tools|length > 0 -%}",
1360 "{%- for m in messages -%}",
1361 "{%- if strict and m.role == 'system' and not loop.first -%}",
1362 "{{ raise_exception('System message must be at the beginning.') }}",
1363 "{%- endif -%}",
1364 "<|im_start|>{{ m.role }}\n{{ m.content }}<|im_end|>\n",
1365 "{%- endfor -%}"
1366 );
1367 const STRICT_ARRAY_TMPL: &str = concat!(
1369 "{%- for m in messages -%}",
1370 "{%- if m.role == 'system' and not loop.first -%}",
1371 "{{ raise_exception('System message must be at the beginning.') }}",
1372 "{%- endif -%}",
1373 "<|im_start|>{{ m.role }}\n",
1374 "{%- if m.content is not string -%}",
1375 "{%- for part in m.content -%}{{ part.text }}{%- endfor -%}",
1376 "{%- endif -%}",
1377 "<|im_end|>\n",
1378 "{%- endfor -%}"
1379 );
1380
1381 fn claude_shape() -> serde_json::Value {
1383 json!([
1384 {"role": "system", "content": "You are Claude Code."},
1385 {"role": "user", "content": "hello"},
1386 {"role": "system", "content": "mid-conversation reminder"},
1387 ])
1388 }
1389
1390 fn all_restrictions() -> SystemNormalization {
1391 SystemNormalization {
1392 demote_nonleading_system: true,
1393 coalesce_consecutive_users: true,
1394 }
1395 }
1396
1397 #[test]
1398 fn permissive_template_is_not_flagged_and_renders_untouched() {
1399 let f = formatter_for(PERMISSIVE_TMPL);
1400 assert!(!f.default_system_normalization.is_required());
1401 assert!(!f.tool_use_system_normalization.is_required());
1402 let out = render_shape(&f, claude_shape()).unwrap();
1403 assert!(out.contains("<|im_start|>system\nmid-conversation reminder<|im_end|>"));
1404 }
1405
1406 #[test]
1407 fn strict_leading_template_demotes_mid_system_but_keeps_user_turns_apart() {
1408 let f = formatter_for(STRICT_LEADING_TMPL);
1409 assert!(f.default_system_normalization.demote_nonleading_system);
1410 assert!(!f.default_system_normalization.coalesce_consecutive_users);
1412
1413 let out = render_shape(&f, claude_shape()).unwrap();
1415 assert_eq!(out.matches("<|im_start|>system").count(), 1);
1416 assert!(out.contains("<|im_start|>user\nhello<|im_end|>"));
1417 assert!(out.contains("<|im_start|>user\nmid-conversation reminder<|im_end|>"));
1418 }
1419
1420 #[test]
1421 fn alternation_template_coalesces_users_but_keeps_mid_system() {
1422 let f = formatter_for(ALTERNATION_TMPL);
1423 assert!(f.default_system_normalization.coalesce_consecutive_users);
1424 assert!(!f.default_system_normalization.demote_nonleading_system);
1426
1427 let out = render_shape(&f, claude_shape()).unwrap();
1428 assert!(out.contains("<|im_start|>system\nmid-conversation reminder<|im_end|>"));
1429
1430 let out = render_shape(
1431 &f,
1432 json!([
1433 {"role": "system", "content": "s"},
1434 {"role": "user", "content": "hello"},
1435 {"role": "user", "content": "again"},
1436 ]),
1437 )
1438 .unwrap();
1439 assert_eq!(out.matches("<|im_start|>user").count(), 1);
1440 assert!(out.contains("<|im_start|>user\nhello\n\nagain<|im_end|>"));
1441 }
1442
1443 #[test]
1444 fn strict_both_template_demotes_then_coalesces() {
1445 let f = formatter_for(STRICT_BOTH_TMPL);
1446 assert!(f.default_system_normalization.demote_nonleading_system);
1447 assert!(f.default_system_normalization.coalesce_consecutive_users);
1448
1449 let out = render_shape(&f, claude_shape()).unwrap();
1450 assert_eq!(out.matches("<|im_start|>system").count(), 1);
1451 assert!(out.contains("<|im_start|>user\nhello\n\nmid-conversation reminder<|im_end|>"));
1452 }
1453
1454 #[test]
1455 fn system_normalization_flag_is_selected_per_template() {
1456 let f = formatter_for_templates(PERMISSIVE_TMPL, STRICT_LEADING_TMPL);
1457 assert!(!f.default_system_normalization.is_required());
1458 assert!(f.tool_use_system_normalization.is_required());
1459
1460 let no_tools = render_shape(&f, claude_shape()).unwrap();
1461 assert!(no_tools.contains("<|im_start|>system\nmid-conversation reminder<|im_end|>"));
1462 let with_tools = render_shape_with_tools(&f, claude_shape()).unwrap();
1463 assert_eq!(with_tools.matches("<|im_start|>system").count(), 1);
1464 assert!(with_tools.contains("<|im_start|>user\nmid-conversation reminder<|im_end|>"));
1465
1466 let f = formatter_for_templates(STRICT_LEADING_TMPL, PERMISSIVE_TMPL);
1467 assert!(f.default_system_normalization.is_required());
1468 assert!(!f.tool_use_system_normalization.is_required());
1469 let with_tools = render_shape_with_tools(&f, claude_shape()).unwrap();
1470 assert!(with_tools.contains("<|im_start|>system\nmid-conversation reminder<|im_end|>"));
1471 }
1472
1473 #[test]
1474 fn system_normalization_probe_uses_runtime_tools_shape() {
1475 let f = formatter_for_templates(DEFAULT_NONE_GATED_TMPL, TOOL_NONEMPTY_GATED_TMPL);
1476 assert!(!f.default_system_normalization.is_required());
1477 assert!(f.tool_use_system_normalization.is_required());
1478
1479 let no_tools = render_shape(&f, claude_shape()).unwrap();
1480 assert!(no_tools.contains("<|im_start|>system\nmid-conversation reminder<|im_end|>"));
1481
1482 let with_tools = render_shape_with_tools(&f, claude_shape()).unwrap();
1483 assert_eq!(with_tools.matches("<|im_start|>system").count(), 1);
1484 assert!(with_tools.contains("<|im_start|>user\nmid-conversation reminder<|im_end|>"));
1485 }
1486
1487 #[test]
1488 fn system_normalization_precedes_required_content_array_conversion() {
1489 let f = formatter_for(STRICT_ARRAY_TMPL);
1490 assert!(f.requires_content_arrays);
1491 assert!(f.default_system_normalization.demote_nonleading_system);
1492
1493 let out = render_shape(
1494 &f,
1495 json!([
1496 {"role": "system", "content": "A"},
1497 {"role": "system", "content": "B"},
1498 {"role": "user", "content": "hello"},
1499 ]),
1500 )
1501 .unwrap();
1502 assert!(out.contains("A\n\nB"));
1503 }
1504
1505 #[test]
1506 fn normalize_preserves_multimodal_user_content_and_fields() {
1507 let mut m = json!([
1508 {
1509 "role": "user",
1510 "name": "kept",
1511 "content": [
1512 {"type": "text", "text": "look"},
1513 {"type": "image"},
1514 ],
1515 },
1516 {"role": "system", "content": "remember"},
1517 ]);
1518 normalize_system_messages(&mut m, all_restrictions());
1519 assert_eq!(
1520 m,
1521 json!([{
1522 "role": "user",
1523 "name": "kept",
1524 "content": [
1525 {"type": "text", "text": "look"},
1526 {"type": "image"},
1527 {"type": "text", "text": "remember"},
1528 ],
1529 }])
1530 );
1531 }
1532
1533 #[test]
1536 fn coalesce_preserves_multimodal_content_of_the_merged_turn() {
1537 let mut m = json!([
1538 {"role": "user", "content": "look"},
1539 {"role": "user", "content": [
1540 {"type": "text", "text": "at this"},
1541 {"type": "image_url", "image_url": {"url": "http://img"}},
1542 ]},
1543 ]);
1544 normalize_system_messages(&mut m, all_restrictions());
1545 assert_eq!(
1546 m,
1547 json!([{
1548 "role": "user",
1549 "content": [
1550 {"type": "text", "text": "look"},
1551 {"type": "text", "text": "at this"},
1552 {"type": "image_url", "image_url": {"url": "http://img"}},
1553 ],
1554 }])
1555 );
1556 }
1557
1558 #[test]
1559 fn normalize_merges_leading_run_and_coalesces() {
1560 let mut m = json!([
1561 {"role": "system", "content": "A"},
1562 {"role": "system", "content": "B"},
1563 {"role": "user", "content": "hi"},
1564 {"role": "system", "content": "reminder"},
1565 ]);
1566 normalize_system_messages(&mut m, all_restrictions());
1567 assert_eq!(
1568 m,
1569 json!([
1570 {"role": "system", "content": "A\n\nB"},
1571 {"role": "user", "content": "hi\n\nreminder"},
1572 ])
1573 );
1574 }
1575
1576 #[test]
1579 fn each_restriction_applies_only_its_own_rewrite() {
1580 let shape = json!([
1581 {"role": "system", "content": "A"},
1582 {"role": "system", "content": "B"},
1583 {"role": "user", "content": "hi"},
1584 {"role": "system", "content": "reminder"},
1585 ]);
1586
1587 let mut demote_only = shape.clone();
1588 normalize_system_messages(
1589 &mut demote_only,
1590 SystemNormalization {
1591 demote_nonleading_system: true,
1592 coalesce_consecutive_users: false,
1593 },
1594 );
1595 assert_eq!(
1596 demote_only,
1597 json!([
1598 {"role": "system", "content": "A\n\nB"},
1599 {"role": "user", "content": "hi"},
1600 {"role": "user", "content": "reminder"},
1601 ])
1602 );
1603
1604 let mut coalesce_only = shape.clone();
1605 normalize_system_messages(
1606 &mut coalesce_only,
1607 SystemNormalization {
1608 demote_nonleading_system: false,
1609 coalesce_consecutive_users: true,
1610 },
1611 );
1612 assert_eq!(coalesce_only, shape);
1613 }
1614
1615 #[test]
1616 fn normalize_preserves_array_system_content() {
1617 let mut m = json!([
1618 {"role": "user", "content": "hi"},
1619 {"role": "system", "content": [{"type": "text", "text": "one"},
1620 {"type": "text", "text": "two"}]},
1621 ]);
1622 normalize_system_messages(&mut m, all_restrictions());
1623 assert_eq!(
1624 m,
1625 json!([{"role": "user", "content": [
1626 {"type": "text", "text": "hi"},
1627 {"type": "text", "text": "one"},
1628 {"type": "text", "text": "two"},
1629 ]}])
1630 );
1631 }
1632
1633 #[test]
1642 #[ignore]
1643 fn adaptive_system_corpus_audit() {
1644 let dir =
1645 std::env::var("TEMPLATE_CORPUS").expect("set TEMPLATE_CORPUS to the templates dir");
1646 let manifest: serde_json::Value =
1647 serde_json::from_str(&std::fs::read_to_string(format!("{dir}/manifest.json")).unwrap())
1648 .unwrap();
1649
1650 let sys = |c: &str| json!({"role": "system", "content": c});
1653 let usr = |c: &str| json!({"role": "user", "content": c});
1654 let asst = |c: &str| json!({"role": "assistant", "content": c});
1655 let shapes: Vec<(&str, serde_json::Value)> = vec![
1656 ("turn1", json!([sys("s"), usr("u"), sys("mid")])),
1657 (
1658 "multiturn",
1659 json!([sys("s"), usr("u"), sys("mid"), asst("a"), usr("u2")]),
1660 ),
1661 (
1662 "mid_after_asst",
1663 json!([sys("s"), usr("u"), asst("a"), sys("mid"), usr("u2")]),
1664 ),
1665 ("double_leading", json!([sys("s0"), sys("s1"), usr("u")])),
1666 ("consec_user", json!([sys("s"), usr("u0"), usr("u1")])),
1667 (
1668 "tail_reminder",
1669 json!([
1670 sys("s"),
1671 usr("u"),
1672 asst("a"),
1673 usr("u2"),
1674 sys("mid"),
1675 usr("u3")
1676 ]),
1677 ),
1678 ("leading_only_baseline", json!([sys("s"), usr("u")])),
1679 ];
1680
1681 let mut total = 0usize;
1682 let mut flagged = 0usize;
1683 let mut demote_only = 0usize;
1684 let mut coalesce = 0usize;
1685 let mut failures: Vec<String> = Vec::new();
1686 for (file, meta) in manifest.as_object().unwrap() {
1687 let tmpl = std::fs::read_to_string(format!("{dir}/{file}.jinja")).unwrap();
1688 let model = meta["model"].as_str().unwrap_or(file);
1689 let f = match try_formatter_for(&tmpl) {
1692 Some(f) => f,
1693 None => {
1694 eprintln!("[skip-compile] {model}");
1695 continue;
1696 }
1697 };
1698 if render_shape(&f, json!([sys("s"), usr("u")])).is_err() {
1701 eprintln!("[skip-baseline] {model}");
1702 continue;
1703 }
1704 total += 1;
1705 let rules = f.default_system_normalization;
1706 let flag = rules.is_required();
1707 if flag {
1708 flagged += 1;
1709 }
1710 if rules.demote_nonleading_system {
1711 demote_only += usize::from(!rules.coalesce_consecutive_users);
1712 }
1713 if rules.coalesce_consecutive_users {
1714 coalesce += 1;
1715 }
1716 for (name, shape) in &shapes {
1717 if render_shape(&f, shape.clone()).is_err() {
1718 failures.push(format!("{model} | shape={name} | flag={flag}"));
1719 }
1720 }
1721 if flag {
1722 eprintln!(
1723 "[ok] demote={} coalesce={} {model}",
1724 rules.demote_nonleading_system, rules.coalesce_consecutive_users
1725 );
1726 }
1727 }
1728 eprintln!(
1729 "\naudited {total} templates ({flagged} flagged: {demote_only} demote-only, \
1730 {coalesce} coalescing); {} shape failures",
1731 failures.len()
1732 );
1733 for f in &failures {
1734 eprintln!(" FAIL {f}");
1735 }
1736 assert!(
1737 failures.is_empty(),
1738 "{} template/shape combinations did not render (probe insufficient or normalization insufficient)",
1739 failures.len()
1740 );
1741 }
1742
1743 #[test]
1751 fn test_render_long_conversation_does_not_overflow_stack() {
1752 let handle = std::thread::Builder::new()
1753 .stack_size(2 * 1024 * 1024)
1754 .spawn(|| {
1755 let template_string = concat!(
1756 "{%- set ns = namespace(items=[]) -%}",
1757 "{%- for m in messages -%}",
1758 "{%- set ns.items = ns.items + [m] -%}",
1759 "{%- endfor -%}",
1760 "COUNT={{ ns.items | length }}"
1761 );
1762 let chat_template: ChatTemplate =
1763 serde_json::from_value(serde_json::json!({ "chat_template": template_string }))
1764 .unwrap();
1765 let formatter =
1766 HfTokenizerConfigJsonFormatter::new(chat_template, ContextMixins::new(&[]))
1767 .unwrap();
1768
1769 let n = 3000;
1770 let messages: Vec<serde_json::Value> = (0..n)
1771 .map(|i| serde_json::json!({"role": "user", "content": format!("turn {i}")}))
1772 .collect();
1773 let request: NvCreateChatCompletionRequest =
1774 serde_json::from_value(serde_json::json!({
1775 "model": "test",
1776 "messages": messages,
1777 }))
1778 .unwrap();
1779
1780 let rendered = formatter.render(&request).unwrap();
1782 assert_eq!(rendered.trim(), format!("COUNT={n}"));
1783 })
1784 .unwrap();
1785 handle.join().unwrap();
1786 }
1787
1788 #[test]
1799 #[ignore]
1800 fn dump_gptoss_tool_prompt() {
1801 use super::tokcfg::ChatTemplate;
1802 use super::{ContextMixins, HfTokenizerConfigJsonFormatter};
1803
1804 let path = std::env::var("GPTOSS_CHAT_TEMPLATE").expect(
1805 "set GPTOSS_CHAT_TEMPLATE to the tokenizer_config.json, chat_template.jinja, or model dir path",
1806 );
1807 let input_path = std::path::Path::new(&path);
1808 let file_path = if input_path.is_dir() {
1809 input_path.join("tokenizer_config.json")
1811 } else {
1812 input_path.to_path_buf()
1813 };
1814 let raw = std::fs::read_to_string(&file_path).expect("read chat template file");
1815 let template_string: String = match serde_json::from_str::<serde_json::Value>(&raw) {
1822 Ok(v) if v.get("chat_template").is_some() => v["chat_template"]
1823 .as_str()
1824 .expect("chat_template field must be a string")
1825 .to_string(),
1826 _ => {
1827 let sibling = std::path::Path::new(&path)
1828 .parent()
1829 .map(|d| d.join("chat_template.jinja"));
1830 match sibling {
1831 Some(p) if p.exists() => {
1832 eprintln!(
1833 "[info] {path} had no chat_template field; using {}",
1834 p.display()
1835 );
1836 std::fs::read_to_string(&p).expect("read sibling chat_template.jinja")
1837 }
1838 _ => raw,
1839 }
1840 }
1841 };
1842
1843 assert!(
1846 template_string.contains("{%") || template_string.contains("{{"),
1847 "resolved template has no Jinja tags — GPTOSS_CHAT_TEMPLATE ({path}) is probably \
1848 tokenizer_config.json with no chat_template field and no sibling chat_template.jinja. \
1849 Point it at the chat_template.jinja file."
1850 );
1851
1852 let chat_template: ChatTemplate =
1853 serde_json::from_value(serde_json::json!({ "chat_template": template_string }))
1854 .unwrap();
1855
1856 let formatter =
1857 HfTokenizerConfigJsonFormatter::new(chat_template, ContextMixins::new(&[])).unwrap();
1858
1859 let request: NvCreateChatCompletionRequest = serde_json::from_str(
1861 r#"{
1862 "model": "openai/gpt-oss-120b",
1863 "messages": [{"role":"user","content":"Search the repo for the string \"countHook\"."}],
1864 "tools": [
1865 {"type":"function","function":{"name":"grep","description":"search files","parameters":{"type":"object","properties":{"pattern":{"type":"string"},"path":{"type":"string"}},"required":["pattern"]}}},
1866 {"type":"function","function":{"name":"read","description":"read a file","parameters":{"type":"object","properties":{"filePath":{"type":"string"}},"required":["filePath"]}}}
1867 ]
1868 }"#,
1869 )
1870 .unwrap();
1871
1872 let rendered = formatter.render(&request).unwrap();
1873 eprintln!("================ RENDERED gpt-oss PROMPT (tools declared) ================");
1874 eprintln!("{rendered}");
1875 eprintln!("================ END RENDERED PROMPT ================");
1876 eprintln!("[diagnostics] does the rendered prompt contain…");
1877 for needle in [
1878 "commentary",
1879 "Calls to these tools",
1880 "functions",
1881 "# Tools",
1882 "<|channel|>",
1883 "constrain",
1884 "analysis",
1885 ] {
1886 eprintln!(
1887 " {:>22}: {}",
1888 format!("{needle:?}"),
1889 rendered.contains(needle)
1890 );
1891 }
1892 }
1893
1894 #[test]
1896 fn test_convert_media_url_to_placeholder_single_type() {
1897 let mut content_array = vec![
1898 serde_json::json!({"type": "text", "text": "Check this image:"}),
1899 serde_json::json!({"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}),
1900 serde_json::json!({"type": "text", "text": "What do you see?"}),
1901 ];
1902
1903 let conversions = &[("image_url", "image")];
1904 convert_media_url_to_placeholder(&mut content_array, conversions);
1905
1906 assert_eq!(content_array.len(), 3);
1907 assert_eq!(content_array[0]["type"], "text");
1909 assert_eq!(content_array[0]["text"], "Check this image:");
1910 assert_eq!(content_array[1]["type"], "image");
1912 assert!(content_array[1].get("image_url").is_none());
1913 assert_eq!(content_array[2]["type"], "text");
1915 assert_eq!(content_array[2]["text"], "What do you see?");
1916 }
1917
1918 #[test]
1920 fn test_convert_media_url_to_placeholder_multiple_same_type() {
1921 let mut content_array = vec![
1922 serde_json::json!({"type": "image_url", "image_url": {"url": "https://example.com/image1.jpg"}}),
1923 serde_json::json!({"type": "text", "text": "vs"}),
1924 serde_json::json!({"type": "image_url", "image_url": {"url": "https://example.com/image2.jpg"}}),
1925 ];
1926
1927 let conversions = &[("image_url", "image")];
1928 convert_media_url_to_placeholder(&mut content_array, conversions);
1929
1930 assert_eq!(content_array.len(), 3);
1931 assert_eq!(content_array[0]["type"], "image");
1932 assert_eq!(content_array[1]["type"], "text");
1933 assert_eq!(content_array[2]["type"], "image");
1934 }
1935
1936 #[test]
1938 fn test_convert_media_url_to_placeholder_selective_conversion() {
1939 let mut content_array = vec![
1940 serde_json::json!({"type": "audio_url", "audio_url": {"url": "https://example.com/audio.mp3"}}),
1941 serde_json::json!({"type": "video_url", "video_url": {"url": "https://example.com/video.mp4"}}),
1942 serde_json::json!({"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}),
1943 ];
1944
1945 let conversions = &[("image_url", "image")];
1947 convert_media_url_to_placeholder(&mut content_array, conversions);
1948
1949 assert_eq!(content_array.len(), 3);
1950 assert_eq!(content_array[0]["type"], "audio_url");
1952 assert!(content_array[0].get("audio_url").is_some());
1953 assert_eq!(content_array[1]["type"], "video_url");
1954 assert!(content_array[1].get("video_url").is_some());
1955 assert_eq!(content_array[2]["type"], "image");
1957 assert!(content_array[2].get("image_url").is_none());
1958 }
1959
1960 #[test]
1962 fn test_convert_media_url_to_placeholder_multiple_types() {
1963 let mut content_array = vec![
1964 serde_json::json!({"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}),
1965 serde_json::json!({"type": "text", "text": "and listen to"}),
1966 serde_json::json!({"type": "audio_url", "audio_url": {"url": "https://example.com/audio.mp3"}}),
1967 serde_json::json!({"type": "text", "text": "and watch"}),
1968 serde_json::json!({"type": "video_url", "video_url": {"url": "https://example.com/video.mp4"}}),
1969 ];
1970
1971 let conversions = &[
1973 ("image_url", "image"),
1974 ("audio_url", "audio"),
1975 ("video_url", "video"),
1976 ];
1977 convert_media_url_to_placeholder(&mut content_array, conversions);
1978
1979 assert_eq!(content_array.len(), 5);
1980 assert_eq!(content_array[0]["type"], "image");
1981 assert!(content_array[0].get("image_url").is_none());
1982 assert_eq!(content_array[1]["type"], "text");
1983 assert_eq!(content_array[2]["type"], "audio");
1984 assert!(content_array[2].get("audio_url").is_none());
1985 assert_eq!(content_array[3]["type"], "text");
1986 assert_eq!(content_array[4]["type"], "video");
1987 assert!(content_array[4].get("video_url").is_none());
1988 }
1989
1990 #[test]
1992 fn test_convert_media_url_to_placeholder_no_conversions() {
1993 let mut content_array = vec![
1994 serde_json::json!({"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}),
1995 serde_json::json!({"type": "text", "text": "hello"}),
1996 ];
1997
1998 let conversions: &[(&str, &str)] = &[];
1999 convert_media_url_to_placeholder(&mut content_array, conversions);
2000
2001 assert_eq!(content_array.len(), 2);
2002 assert_eq!(content_array[0]["type"], "image_url");
2004 assert!(content_array[0].get("image_url").is_some());
2005 assert_eq!(content_array[1]["type"], "text");
2006 }
2007
2008 #[test]
2011 fn test_default_media_type_conversions_only_converts_image_url() {
2012 let mut content_array = vec![
2013 serde_json::json!({"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}),
2014 serde_json::json!({"type": "video_url", "video_url": {"url": "https://example.com/video.mp4"}}),
2015 serde_json::json!({"type": "audio_url", "audio_url": {"url": "https://example.com/audio.mp3"}}),
2016 serde_json::json!({"type": "text", "text": "hello"}),
2017 ];
2018
2019 convert_media_url_to_placeholder(&mut content_array, DEFAULT_MEDIA_TYPE_CONVERSIONS);
2021
2022 assert_eq!(content_array.len(), 4);
2023
2024 assert_eq!(content_array[0]["type"], "image");
2026 assert!(content_array[0].get("image_url").is_none());
2027
2028 assert_eq!(content_array[1]["type"], "video");
2030 assert!(content_array[1].get("video_url").is_none());
2031
2032 assert_eq!(content_array[2]["type"], "audio");
2034 assert!(content_array[2].get("audio_url").is_none());
2035
2036 assert_eq!(content_array[3]["type"], "text");
2038 assert_eq!(content_array[3]["text"], "hello");
2039 }
2040
2041 #[test]
2042 fn test_may_be_fix_tool_schema_missing_type_and_properties() {
2043 let json_str = r#"{
2044 "model": "gpt-4o",
2045 "messages": [],
2046 "tools": [
2047 {
2048 "type": "function",
2049 "function": {
2050 "name": "get_weather",
2051 "description": "Get the current weather in a given location",
2052 "parameters": {},
2053 "strict": null
2054 }
2055 }
2056 ]
2057 }"#;
2058
2059 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2060 let tools = serde_json::to_value(request.tools()).unwrap();
2061
2062 assert!(tools[0]["function"]["parameters"]["type"] == "object");
2063 assert!(
2064 tools[0]["function"]["parameters"]["properties"]
2065 == serde_json::Value::Object(Default::default())
2066 );
2067 }
2068
2069 #[test]
2070 fn test_may_be_fix_tool_schema_missing_type() {
2071 let json_str = r#"{
2072 "model": "gpt-4o",
2073 "messages": [],
2074 "tools": [
2075 {
2076 "type": "function",
2077 "function": {
2078 "name": "get_weather",
2079 "description": "Get the current weather in a given location",
2080 "parameters": {
2081 "properties": {
2082 "location": {
2083 "type": "string",
2084 "description": "City and state, e.g., 'San Francisco, CA'"
2085 }
2086 }
2087 },
2088 "strict": null
2089 }
2090 }
2091 ]
2092 }"#;
2093 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2094
2095 let tools = serde_json::to_value(request.tools()).unwrap();
2096
2097 assert_eq!(tools[0]["function"]["parameters"]["type"], "object");
2098
2099 let mut expected_properties = serde_json::Map::new();
2100 let mut location = serde_json::Map::new();
2101 location.insert(
2102 "type".to_string(),
2103 serde_json::Value::String("string".to_string()),
2104 );
2105 location.insert(
2106 "description".to_string(),
2107 serde_json::Value::String("City and state, e.g., 'San Francisco, CA'".to_string()),
2108 );
2109 expected_properties.insert("location".to_string(), serde_json::Value::Object(location));
2110
2111 assert_eq!(
2112 tools[0]["function"]["parameters"]["properties"],
2113 serde_json::Value::Object(expected_properties)
2114 );
2115 }
2116
2117 #[test]
2118 fn test_may_be_fix_tool_schema_missing_properties() {
2119 let json_str = r#"{
2120 "model": "gpt-4o",
2121 "messages": [],
2122 "tools": [
2123 {
2124 "type": "function",
2125 "function": {
2126 "name": "get_weather",
2127 "description": "Get the current weather in a given location",
2128 "parameters": {"type": "object"},
2129 "strict": null
2130 }
2131 }
2132 ]
2133 }"#;
2134
2135 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2136 let tools = serde_json::to_value(request.tools()).unwrap();
2137
2138 assert_eq!(
2139 tools[0]["function"]["parameters"]["properties"],
2140 serde_json::Value::Object(Default::default())
2141 );
2142 assert_eq!(tools[0]["function"]["parameters"]["type"], "object");
2143 }
2144
2145 #[test]
2146 fn test_may_be_fix_tool_schema_missing_description() {
2147 let json_str = r#"{
2151 "model": "gpt-4o",
2152 "messages": [],
2153 "tools": [
2154 {
2155 "type": "function",
2156 "function": {
2157 "name": "noop",
2158 "parameters": {
2159 "type": "object",
2160 "properties": { "x": { "type": "string" } },
2161 "required": ["x"],
2162 "additionalProperties": false
2163 },
2164 "strict": null
2165 }
2166 }
2167 ]
2168 }"#;
2169
2170 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2171 let tools = serde_json::to_value(request.tools()).unwrap();
2172
2173 assert_eq!(
2174 tools[0]["function"]["description"],
2175 serde_json::Value::String(String::new())
2176 );
2177 }
2178
2179 #[test]
2180 fn test_may_be_fix_tool_schema_null_description() {
2181 let json_str = r#"{
2183 "model": "gpt-4o",
2184 "messages": [],
2185 "tools": [
2186 {
2187 "type": "function",
2188 "function": {
2189 "name": "noop",
2190 "description": null,
2191 "parameters": {"type": "object", "properties": {}},
2192 "strict": null
2193 }
2194 }
2195 ]
2196 }"#;
2197
2198 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2199 let tools = serde_json::to_value(request.tools()).unwrap();
2200
2201 assert_eq!(
2202 tools[0]["function"]["description"],
2203 serde_json::Value::String(String::new())
2204 );
2205 }
2206
2207 #[test]
2208 fn test_may_be_fix_tool_schema_preserves_description() {
2209 let json_str = r#"{
2211 "model": "gpt-4o",
2212 "messages": [],
2213 "tools": [
2214 {
2215 "type": "function",
2216 "function": {
2217 "name": "get_weather",
2218 "description": "Get the current weather in a given location",
2219 "parameters": {"type": "object", "properties": {}},
2220 "strict": null
2221 }
2222 }
2223 ]
2224 }"#;
2225
2226 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2227 let tools = serde_json::to_value(request.tools()).unwrap();
2228
2229 assert_eq!(
2230 tools[0]["function"]["description"],
2231 "Get the current weather in a given location"
2232 );
2233 }
2234
2235 #[test]
2237 fn test_may_be_fix_msg_content_user_multipart() {
2238 let json_str = r#"{
2239 "model": "gpt-4o",
2240 "messages": [
2241 {
2242 "role": "user",
2243 "content": [
2244 {"type": "text", "text": "part 1"},
2245 {"type": "text", "text": "part 2"}
2246 ]
2247 }
2248 ]
2249 }"#;
2250
2251 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2252 let messages_raw = serde_json::to_value(request.messages()).unwrap();
2253
2254 let messages =
2256 serde_json::to_value(may_be_fix_msg_content(messages_raw, false, &[], &[], None))
2257 .unwrap();
2258
2259 assert_eq!(
2261 messages[0]["content"],
2262 serde_json::Value::String("part 1\npart 2".to_string())
2263 );
2264 }
2265
2266 #[test]
2269 fn test_may_be_fix_msg_content_mixed_messages() {
2270 let json_str = r#"{
2271 "model": "gpt-4o",
2272 "messages": [
2273 {
2274 "role": "system",
2275 "content": "You are a helpful assistant"
2276 },
2277 {
2278 "role": "user",
2279 "content": [
2280 {"type": "text", "text": "Hello"},
2281 {"type": "text", "text": "World"}
2282 ]
2283 },
2284 {
2285 "role": "assistant",
2286 "content": "Hi there!"
2287 },
2288 {
2289 "role": "user",
2290 "content": [
2291 {"type": "text", "text": "Another"},
2292 {"type": "text", "text": "multi-part"},
2293 {"type": "text", "text": "message"}
2294 ]
2295 }
2296 ]
2297 }"#;
2298
2299 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2300 let messages_raw = serde_json::to_value(request.messages()).unwrap();
2301
2302 let messages =
2304 serde_json::to_value(may_be_fix_msg_content(messages_raw, false, &[], &[], None))
2305 .unwrap();
2306
2307 assert_eq!(
2309 messages[0]["content"],
2310 serde_json::Value::String("You are a helpful assistant".to_string())
2311 );
2312
2313 assert_eq!(
2315 messages[1]["content"],
2316 serde_json::Value::String("Hello\nWorld".to_string())
2317 );
2318
2319 assert_eq!(
2321 messages[2]["content"],
2322 serde_json::Value::String("Hi there!".to_string())
2323 );
2324
2325 assert_eq!(
2327 messages[3]["content"],
2328 serde_json::Value::String("Another\nmulti-part\nmessage".to_string())
2329 );
2330 }
2331
2332 #[test]
2334 fn test_may_be_fix_msg_content_empty_array() {
2335 let json_str = r#"{
2336 "model": "gpt-4o",
2337 "messages": [
2338 {
2339 "role": "user",
2340 "content": []
2341 }
2342 ]
2343 }"#;
2344
2345 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2346 let messages_raw = serde_json::to_value(request.messages()).unwrap();
2347
2348 let messages =
2350 serde_json::to_value(may_be_fix_msg_content(messages_raw, false, &[], &[], None))
2351 .unwrap();
2352
2353 assert!(messages[0]["content"].is_array());
2355 assert_eq!(messages[0]["content"].as_array().unwrap().len(), 0);
2356 }
2357
2358 #[test]
2365 fn test_may_be_fix_msg_content_empty_array_with_placeholder_template() {
2366 let json_str = r#"{
2367 "model": "phi-3-vision",
2368 "messages": [
2369 {
2370 "role": "user",
2371 "content": []
2372 }
2373 ]
2374 }"#;
2375
2376 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2377 let messages_raw = serde_json::to_value(request.messages()).unwrap();
2378
2379 let messages = serde_json::to_value(may_be_fix_msg_content(
2382 messages_raw,
2383 false,
2384 &[],
2385 &[],
2386 Some("<|image_{n}|>"),
2387 ))
2388 .unwrap();
2389
2390 assert!(
2391 messages[0]["content"].is_array(),
2392 "empty array should be preserved as `[]`, not flattened to `\"\"`"
2393 );
2394 assert_eq!(messages[0]["content"].as_array().unwrap().len(), 0);
2395 }
2396
2397 #[test]
2399 fn test_may_be_fix_msg_content_single_text() {
2400 let json_str = r#"{
2401 "model": "gpt-4o",
2402 "messages": [
2403 {
2404 "role": "user",
2405 "content": "Simple text message"
2406 }
2407 ]
2408 }"#;
2409
2410 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2411 let messages_raw = serde_json::to_value(request.messages()).unwrap();
2412
2413 let messages =
2415 serde_json::to_value(may_be_fix_msg_content(messages_raw, false, &[], &[], None))
2416 .unwrap();
2417
2418 assert_eq!(
2420 messages[0]["content"],
2421 serde_json::Value::String("Simple text message".to_string())
2422 );
2423 }
2424
2425 #[test]
2428 fn test_may_be_fix_msg_content_mixed_types() {
2429 let json_str = r#"{
2430 "model": "gpt-4o",
2431 "messages": [
2432 {
2433 "role": "user",
2434 "content": [
2435 {"type": "text", "text": "Check this image:"},
2436 {"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}},
2437 {"type": "text", "text": "What do you see?"}
2438 ]
2439 }
2440 ]
2441 }"#;
2442
2443 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2444 let messages_raw = serde_json::to_value(request.messages()).unwrap();
2445
2446 let messages =
2448 serde_json::to_value(may_be_fix_msg_content(messages_raw, false, &[], &[], None))
2449 .unwrap();
2450
2451 assert!(messages[0]["content"].is_array());
2454 let content_array = messages[0]["content"].as_array().unwrap();
2455 assert_eq!(content_array.len(), 3);
2456 assert_eq!(content_array[0]["type"], "text");
2457 assert_eq!(content_array[1]["type"], "image");
2458 assert!(content_array[1].get("image_url").is_none());
2459 assert_eq!(content_array[2]["type"], "text");
2460 }
2461
2462 #[test]
2468 fn test_may_be_fix_msg_content_flattens_phi3_style() {
2469 let json_str = r#"{
2470 "model": "phi-3-vision",
2471 "messages": [
2472 {
2473 "role": "user",
2474 "content": [
2475 {"type": "text", "text": "First "},
2476 {"type": "image_url", "image_url": {"url": "https://example.com/a.jpg"}},
2477 {"type": "text", "text": " then "},
2478 {"type": "image_url", "image_url": {"url": "https://example.com/b.jpg"}},
2479 {"type": "text", "text": "?"}
2480 ]
2481 }
2482 ]
2483 }"#;
2484 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2485 let messages_raw = serde_json::to_value(request.messages()).unwrap();
2486
2487 let messages = serde_json::to_value(may_be_fix_msg_content(
2488 messages_raw,
2489 false,
2490 &[],
2491 &[],
2492 Some("<|image_{n}|>"),
2493 ))
2494 .unwrap();
2495
2496 let content = messages[0]["content"].as_str().expect("content flattened");
2497 assert_eq!(content, "First <|image_1|> then <|image_2|>?");
2498 }
2499
2500 #[test]
2502 fn test_may_be_fix_msg_content_flattens_llava_style() {
2503 let json_str = r#"{
2504 "model": "llava-1.5-7b-hf",
2505 "messages": [
2506 {
2507 "role": "user",
2508 "content": [
2509 {"type": "text", "text": "Describe: "},
2510 {"type": "image_url", "image_url": {"url": "https://example.com/x.jpg"}}
2511 ]
2512 }
2513 ]
2514 }"#;
2515 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2516 let messages_raw = serde_json::to_value(request.messages()).unwrap();
2517
2518 let messages = serde_json::to_value(may_be_fix_msg_content(
2519 messages_raw,
2520 false,
2521 &[],
2522 &[],
2523 Some("<image>"),
2524 ))
2525 .unwrap();
2526
2527 let content = messages[0]["content"].as_str().expect("content flattened");
2528 assert_eq!(content, "Describe: <image>");
2529 }
2530
2531 #[test]
2537 fn test_may_be_fix_msg_content_flattens_empty_placeholder() {
2538 let json_str = r#"{
2539 "model": "nvidia/NVIDIA-Nemotron-Parse-v1.2",
2540 "messages": [
2541 {
2542 "role": "user",
2543 "content": [
2544 {"type": "text", "text": "</s><s><predict_bbox><predict_classes><output_markdown><predict_no_text_in_pic>"},
2545 {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}
2546 ]
2547 }
2548 ]
2549 }"#;
2550 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2551 let messages_raw = serde_json::to_value(request.messages()).unwrap();
2552
2553 let messages = serde_json::to_value(may_be_fix_msg_content(
2554 messages_raw,
2555 false,
2556 &[],
2557 &[],
2558 Some(""),
2559 ))
2560 .unwrap();
2561
2562 let content = messages[0]["content"].as_str().expect("content flattened");
2563 assert_eq!(
2564 content,
2565 "</s><s><predict_bbox><predict_classes><output_markdown><predict_no_text_in_pic>"
2566 );
2567 }
2568
2569 #[test]
2576 fn test_render_nemotron_parse_passthrough() {
2577 use super::super::tokcfg::ChatTemplate;
2578 use super::{ContextMixins, HfTokenizerConfigJsonFormatter};
2579
2580 let chat_template: ChatTemplate = serde_json::from_value(serde_json::json!({
2581 "chat_template": "{% for message in messages %}{{ message['content'] }}{% endfor %}"
2582 }))
2583 .unwrap();
2584 let formatter =
2585 HfTokenizerConfigJsonFormatter::new(chat_template, ContextMixins::new(&[])).unwrap();
2586
2587 for prompt in [
2588 "</s><s><predict_bbox><predict_classes><output_markdown><predict_no_text_in_pic>",
2589 "</s><s><predict_bbox><predict_classes><output_markdown><predict_text_in_pic>",
2590 ] {
2591 let request: NvCreateChatCompletionRequest =
2592 serde_json::from_value(serde_json::json!({
2593 "model": "nvidia/NVIDIA-Nemotron-Parse-v1.2",
2594 "messages": [{
2595 "role": "user",
2596 "content": [
2597 {"type": "text", "text": prompt},
2598 {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}
2599 ]
2600 }]
2601 }))
2602 .unwrap();
2603
2604 let rendered = formatter.render(&request).unwrap();
2605 assert_eq!(
2606 rendered, prompt,
2607 "rendered prompt must be the control tokens only, with no JSON-serialized image array"
2608 );
2609 }
2610 }
2611
2612 #[test]
2615 fn test_may_be_fix_msg_content_non_text_only() {
2616 let json_str = r#"{
2617 "model": "gpt-4o",
2618 "messages": [
2619 {
2620 "role": "user",
2621 "content": [
2622 {"type": "image_url", "image_url": {"url": "https://example.com/image1.jpg"}},
2623 {"type": "image_url", "image_url": {"url": "https://example.com/image2.jpg"}}
2624 ]
2625 }
2626 ]
2627 }"#;
2628
2629 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2630 let messages_raw = serde_json::to_value(request.messages()).unwrap();
2631
2632 let messages =
2634 serde_json::to_value(may_be_fix_msg_content(messages_raw, false, &[], &[], None))
2635 .unwrap();
2636
2637 assert!(messages[0]["content"].is_array());
2639 let content_array = messages[0]["content"].as_array().unwrap();
2640 assert_eq!(content_array.len(), 2);
2641 assert_eq!(content_array[0]["type"], "image");
2642 assert_eq!(content_array[1]["type"], "image");
2643 }
2644
2645 #[test]
2646 fn test_none_tools_safe_for_all_templates() {
2647 use super::tokcfg::ChatTemplate;
2648 use super::{ContextMixins, HfTokenizerConfigJsonFormatter};
2649
2650 let length_template = r#"
2654{%- if tools is iterable and tools | length > 0 %}
2655Tools available: {{ tools | length }}
2656{%- else %}
2657No tools
2658{%- endif %}
2659"#;
2660
2661 let no_tool_template = r#"
2664{%- if tools is not none %}
2665TOOL MODE
2666{%- else %}
2667NORMAL MODE
2668{%- endif %}
2669"#;
2670
2671 let chat_template: ChatTemplate = serde_json::from_value(serde_json::json!({
2672 "chat_template": [
2673 {"safe_length": length_template},
2674 {"no_tool": no_tool_template}
2675 ]
2676 }))
2677 .unwrap();
2678
2679 let formatter =
2680 HfTokenizerConfigJsonFormatter::new(chat_template, ContextMixins::new(&[])).unwrap();
2681
2682 let ctx = context! { tools => Option::<Value>::None };
2683
2684 let result1 = formatter
2685 .env
2686 .get_template("safe_length")
2687 .unwrap()
2688 .render(&ctx);
2689 println!("Safe length template with no tools => None: {:?}", result1);
2690 assert!(
2691 result1.is_ok(),
2692 "Jinja template with and conditional and length filter should handle None: {:?}",
2693 result1
2694 );
2695 assert!(
2696 result1.unwrap().contains("No tools"),
2697 "Should show 'No tools'"
2698 );
2699
2700 let result2 = formatter.env.get_template("no_tool").unwrap().render(&ctx);
2701 println!("Default template with no tools => None: {:?}", result2);
2702 assert!(
2703 result2.is_ok(),
2704 "Jinja template with if tools is not none conditional should handle None: {:?}",
2705 result2
2706 );
2707 assert!(result2.unwrap().contains("NORMAL MODE"));
2708 }
2709
2710 #[test]
2712 fn test_may_be_fix_msg_content_multiple_content_types() {
2713 let json_str = r#"{
2715 "model": "gpt-4o",
2716 "messages": [
2717 {
2718 "role": "user",
2719 "content": [
2720 {"type": "text", "text": "Listen to this:"},
2721 {"type": "audio_url", "audio_url": {"url": "https://example.com/audio.mp3"}},
2722 {"type": "text", "text": "And look at:"},
2723 {"type": "image_url", "image_url": {"url": "https://example.com/img.jpg"}},
2724 {"type": "text", "text": "What do you think?"}
2725 ]
2726 }
2727 ]
2728 }"#;
2729
2730 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2731 let messages_raw = serde_json::to_value(request.messages()).unwrap();
2732 let messages =
2733 serde_json::to_value(may_be_fix_msg_content(messages_raw, false, &[], &[], None))
2734 .unwrap();
2735
2736 assert!(messages[0]["content"].is_array());
2738 let content_array = messages[0]["content"].as_array().unwrap();
2739 assert_eq!(content_array.len(), 5);
2740 assert_eq!(content_array[0]["type"], "text");
2741 assert_eq!(content_array[1]["type"], "audio");
2742 assert_eq!(content_array[2]["type"], "text");
2743 assert_eq!(content_array[3]["type"], "image");
2744 assert_eq!(content_array[4]["type"], "text");
2745
2746 let json_str = r#"{
2748 "model": "gpt-4o",
2749 "messages": [
2750 {
2751 "role": "user",
2752 "content": [
2753 {"type": "text", "text": "Check this:"},
2754 {"type": "video_url", "video_url": {"url": "https://example.com/vid.mp4"}},
2755 {"type": "text", "text": "Interesting?"}
2756 ]
2757 }
2758 ]
2759 }"#;
2760
2761 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2762 let messages_raw = serde_json::to_value(request.messages()).unwrap();
2763 let messages =
2764 serde_json::to_value(may_be_fix_msg_content(messages_raw, false, &[], &[], None))
2765 .unwrap();
2766
2767 assert!(messages[0]["content"].is_array());
2769 assert_eq!(messages[0]["content"].as_array().unwrap().len(), 3);
2770 }
2771
2772 #[test]
2773 fn test_normalize_tool_arguments_tojson() {
2774 let tmpl = r#"{{ messages[0].tool_calls[0].function.arguments | tojson }}"#;
2775
2776 let mut messages = serde_json::Value::Array(vec![serde_json::json!({
2778 "role": "assistant",
2779 "tool_calls": [{
2780 "type": "function",
2781 "function": {
2782 "name": "get_current_weather",
2783 "arguments": "{\"format\":\"celsius\",\"location\":\"San Francisco, CA\"}"
2784 }
2785 }]
2786 })]);
2787
2788 normalize_tool_calls_arguments_in_messages(&mut messages);
2789
2790 let mut env = Environment::new();
2791 env.add_filter("tojson", super::super::tokcfg::tojson);
2792 env.add_template("t", tmpl).unwrap();
2793 let out = env
2794 .get_template("t")
2795 .unwrap()
2796 .render(context! { messages => crate::python::template_value(&messages) })
2797 .unwrap();
2798
2799 assert_eq!(
2802 out,
2803 r#"{"format": "celsius", "location": "San Francisco, CA"}"#
2804 );
2805 }
2806
2807 #[test]
2808 fn test_normalize_tool_arguments_items_loop() {
2809 let tmpl = r#"{% for k, v in messages[0].tool_calls[0].function.arguments|items %}{{k}}={{v}};{% endfor %}"#;
2810
2811 let mut messages = serde_json::Value::Array(vec![serde_json::json!({
2812 "role": "assistant",
2813 "tool_calls": [{
2814 "type": "function",
2815 "function": {
2816 "name": "f",
2817 "arguments": "{\"a\":1,\"b\":\"x\"}"
2818 }
2819 }]
2820 })]);
2821
2822 normalize_tool_calls_arguments_in_messages(&mut messages);
2823
2824 let mut env = Environment::new();
2825 env.add_template("t", tmpl).unwrap();
2826 let out = env
2827 .get_template("t")
2828 .unwrap()
2829 .render(context! { messages => crate::python::template_value(&messages) })
2830 .unwrap();
2831
2832 assert!(out == "a=1;b=x;" || out == "b=x;a=1;");
2833 }
2834
2835 #[test]
2836 fn test_normalize_tool_arguments_legacy_function_call() {
2837 let mut messages = serde_json::Value::Array(vec![serde_json::json!({
2839 "role": "assistant",
2840 "function_call": {
2841 "name": "get_weather",
2842 "arguments": "{\"location\":\"NYC\"}"
2843 }
2844 })]);
2845
2846 normalize_function_call_arguments_in_messages(&mut messages);
2847
2848 assert_eq!(
2849 messages[0]["function_call"]["arguments"],
2850 serde_json::json!({"location": "NYC"})
2851 );
2852 }
2853
2854 #[test]
2855 fn test_normalize_tool_arguments_malformed_json_passthrough() {
2856 let mut messages = serde_json::Value::Array(vec![serde_json::json!({
2858 "role": "assistant",
2859 "tool_calls": [{
2860 "type": "function",
2861 "function": {
2862 "name": "f",
2863 "arguments": "not valid json at all"
2864 }
2865 }]
2866 })]);
2867
2868 normalize_tool_calls_arguments_in_messages(&mut messages);
2869
2870 assert_eq!(
2871 messages[0]["tool_calls"][0]["function"]["arguments"],
2872 serde_json::Value::String("not valid json at all".to_string())
2873 );
2874 }
2875
2876 #[test]
2877 fn test_normalize_tool_arguments_with_multimodal_content() {
2878 let json_str = r#"{
2879 "model": "gpt-4o",
2880 "messages": [
2881 {
2882 "role": "user",
2883 "content": [
2884 {"type": "text", "text": "Check this:"},
2885 {"type": "video_url", "video_url": {"url": "https://example.com/vid.mp4"}},
2886 {"type": "text", "text": "Interesting?"}
2887 ]
2888 },
2889 {
2890 "role": "assistant",
2891 "tool_calls": [{
2892 "id": "call_123",
2893 "type": "function",
2894 "function": {
2895 "name": "analyze_video",
2896 "arguments": "{\"url\":\"https://example.com/vid.mp4\",\"format\":\"mp4\"}"
2897 }
2898 }]
2899 }
2900 ]
2901 }"#;
2902
2903 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2904 let messages_raw = serde_json::to_value(request.messages()).unwrap();
2905
2906 let mut messages =
2908 serde_json::to_value(may_be_fix_msg_content(messages_raw, false, &[], &[], None))
2909 .unwrap();
2910
2911 normalize_tool_calls_arguments_in_messages(&mut messages);
2912
2913 assert!(messages[0]["content"].is_array());
2915 assert_eq!(messages[0]["content"].as_array().unwrap().len(), 3);
2916
2917 assert!(messages[1]["tool_calls"][0]["function"]["arguments"].is_object());
2919 assert_eq!(
2920 messages[1]["tool_calls"][0]["function"]["arguments"]["url"],
2921 "https://example.com/vid.mp4"
2922 );
2923 }
2924
2925 #[test]
2928 fn test_minimax_m3_history_tool_call_float_arguments_match_hf() {
2929 let template = r#"{%- set ns_token = ']<]minimax[>[' -%}
2930{%- macro to_xml(val, ns) -%}
2931{%- if val is mapping -%}
2932{%- for k, v in val.items() if v is not none -%}
2933{{ ns }}<{{ k }}>{{ to_xml(v, ns) }}{{ ns }}</{{ k }}>
2934{%- endfor -%}
2935{%- elif val is iterable and val is not string -%}
2936{%- for item in val -%}
2937{{ ns }}<item>{{ to_xml(item, ns) }}{{ ns }}</item>
2938{%- endfor -%}
2939{%- elif val is none -%}
2940{%- elif val is boolean -%}
2941{{ val | tojson }}
2942{%- else -%}
2943{{ val }}
2944{%- endif -%}
2945{%- endmacro -%}
2946{%- for message in messages if message.tool_calls -%}
2947{%- for tool_call in message.tool_calls -%}
2948{%- if tool_call.function -%}
2949{%- set tool_call = tool_call.function -%}
2950{%- endif -%}
2951{{- ns_token + '<invoke name="' + tool_call.name + '">' }}
2952{%- set _args = tool_call.arguments -%}
2953{%- for k, v in _args.items() if v is not none %}
2954{{- ns_token + '<' + k + '>' -}}
2955{{- to_xml(v, ns_token) -}}
2956{{- ns_token + '</' + k + '>' }}
2957{%- endfor -%}
2958{{- ns_token + '</invoke>' ~ '\n' }}
2959{%- endfor -%}
2960{%- endfor -%}"#;
2961 let rendered = render_shape(
2962 &formatter_for(template),
2963 json!([
2964 {"role": "user", "content": "u"},
2965 {"role": "assistant", "content": "", "tool_calls": [{
2966 "id": "c1",
2967 "type": "function",
2968 "function": {
2969 "name": "fit",
2970 "arguments": r#"{"tolerance": 1e-07, "bounds": [0.00001, 1e16]}"#
2971 }
2972 }]}
2973 ]),
2974 )
2975 .unwrap();
2976 assert_eq!(
2977 rendered,
2978 "]<]minimax[>[<invoke name=\"fit\">]<]minimax[>[<tolerance>1e-07]<]minimax[>[</tolerance>]<]minimax[>[<bounds>]<]minimax[>[<item>1e-05]<]minimax[>[</item>]<]minimax[>[<item>1e+16]<]minimax[>[</item>]<]minimax[>[</bounds>]<]minimax[>[</invoke>\n"
2979 );
2980 }
2981
2982 #[test]
2985 fn test_qwen3_coder_history_tool_call_float_arguments_match_hf() {
2986 let template = r#"{%- for message in messages if message.tool_calls -%}
2987{%- for tool_call in message.tool_calls %}
2988 {%- if tool_call.function is defined %}
2989 {%- set tool_call = tool_call.function %}
2990 {%- endif %}
2991 {%- for args_name, args_value in tool_call.arguments|items %}
2992 {{- '<parameter=' + args_name + '>\n' }}
2993 {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
2994 {{- args_value }}
2995 {{- '\n</parameter>\n' }}
2996 {%- endfor %}
2997{%- endfor %}
2998{%- endfor %}"#;
2999 let rendered = render_shape(
3000 &formatter_for(template),
3001 json!([
3002 {"role": "user", "content": "u"},
3003 {"role": "assistant", "content": "", "tool_calls": [{
3004 "id": "c1",
3005 "type": "function",
3006 "function": {
3007 "name": "fit",
3008 "arguments": r#"{"tolerance": 1e-07, "bounds": [0.00001, 1e16]}"#
3009 }
3010 }]}
3011 ]),
3012 )
3013 .unwrap();
3014 assert_eq!(
3015 rendered,
3016 "<parameter=tolerance>\n1e-07\n</parameter>\n<parameter=bounds>\n[1e-05, 1e+16]\n</parameter>\n"
3017 );
3018 }
3019
3020 #[test]
3022 fn test_may_be_fix_msg_content_string_to_array() {
3023 let json_str = r#"{
3024 "model": "gpt-4o",
3025 "messages": [
3026 {
3027 "role": "user",
3028 "content": "Hello, how are you?"
3029 }
3030 ]
3031 }"#;
3032
3033 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
3034 let messages_raw = serde_json::to_value(request.messages()).unwrap();
3035
3036 let messages =
3038 serde_json::to_value(may_be_fix_msg_content(messages_raw, true, &[], &[], None))
3039 .unwrap();
3040
3041 assert!(messages[0]["content"].is_array());
3043 let content_array = messages[0]["content"].as_array().unwrap();
3044 assert_eq!(content_array.len(), 1);
3045 assert_eq!(content_array[0]["type"], "text");
3046 assert_eq!(content_array[0]["text"], "Hello, how are you?");
3047 }
3048
3049 #[test]
3051 fn test_may_be_fix_msg_content_array_preserved_with_multimodal() {
3052 let json_str = r#"{
3053 "model": "gpt-4o",
3054 "messages": [
3055 {
3056 "role": "user",
3057 "content": [
3058 {"type": "text", "text": "part 1"},
3059 {"type": "text", "text": "part 2"}
3060 ]
3061 }
3062 ]
3063 }"#;
3064
3065 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
3066 let messages_raw = serde_json::to_value(request.messages()).unwrap();
3067
3068 let messages =
3070 serde_json::to_value(may_be_fix_msg_content(messages_raw, true, &[], &[], None))
3071 .unwrap();
3072
3073 assert!(messages[0]["content"].is_array());
3075 let content_array = messages[0]["content"].as_array().unwrap();
3076 assert_eq!(content_array.len(), 2);
3077 assert_eq!(content_array[0]["text"], "part 1");
3078 assert_eq!(content_array[1]["text"], "part 2");
3079 }
3080
3081 fn text_parts(first: &str, second: &str) -> serde_json::Value {
3082 json!([{"type": "text", "text": first}, {"type": "text", "text": second}])
3083 }
3084
3085 const TEXT_PARTS_TMPL: &str = concat!(
3086 "{%- for m in messages -%}<{{ m.role }}>",
3087 "{%- if m.content is string -%}{{ m.content }}{%- else -%}",
3088 "{%- for p in m.content -%}{{ p.text }}",
3089 "{%- if not loop.last -%}{{ ARRAY_SEPARATOR }}{%- endif -%}",
3090 "{%- endfor -%}{%- endif -%}</{{ m.role }}>{%- endfor -%}"
3091 );
3092
3093 #[test]
3094 fn text_part_arrays_keep_template_separators_and_framing() {
3095 let shape = json!([{"role": "user", "content": text_parts("part one", "part two")}]);
3096 for separator in [" ", "", "\n", " :: "] {
3097 let template = TEXT_PARTS_TMPL.replace(
3098 "ARRAY_SEPARATOR",
3099 &serde_json::to_string(separator).unwrap(),
3100 );
3101 let f = formatter_for(&template);
3102 assert!(!f.requires_content_arrays);
3103 assert_eq!(
3104 render_shape(&f, shape.clone()).unwrap(),
3105 format!("<user>part one{separator}part two</user>"),
3106 );
3107 }
3108
3109 let template = TEXT_PARTS_TMPL
3110 .replace("ARRAY_SEPARATOR", "''")
3111 .replace("{{ p.text }}", "<text>{{ p.text }}</text>");
3112 assert_eq!(
3113 render_shape(&formatter_for(&template), shape).unwrap(),
3114 "<user><text>part one</text><text>part two</text></user>",
3115 );
3116 }
3117
3118 #[test]
3119 fn preserved_text_arrays_do_not_fall_through_to_media_flattening() {
3120 let shape = json!([{"role": "user", "content": text_parts("part one", "part two")}]);
3121 assert_eq!(
3122 may_be_fix_msg_content(shape.clone(), false, &["user"], &[], Some("<image>")),
3123 shape,
3124 );
3125 }
3126
3127 #[test]
3128 fn text_array_support_is_selected_per_role_including_array_only_roles() {
3129 let template = concat!(
3130 "{%- for m in messages -%}<{{ m.role }}>",
3131 "{%- if m.role == 'system' -%}{{ m.content }}",
3132 "{%- elif m.role == 'assistant' and m.content is string and m.content|length > 0 -%}",
3133 "{{ raise_exception('Assistant content must be an array.') }}",
3134 "{%- elif m.content is string -%}{{ m.content }}{%- else -%}",
3135 "{%- for p in m.content -%}{{ p.text }}",
3136 "{%- if not loop.last -%}|{%- endif -%}{%- endfor -%}",
3137 "{%- endif -%}</{{ m.role }}>{%- endfor -%}"
3138 );
3139 let f = formatter_for(template);
3140 assert!(!f.requires_content_arrays);
3141 assert!(f.default_native_text_array_roles.contains(&"assistant"));
3142 assert!(!f.default_native_text_array_roles.contains(&"system"));
3143 let parts = text_parts("part one", "part two");
3144 let shape = json!([
3145 {"role": "system", "content": parts},
3146 {"role": "developer", "content": parts},
3147 {"role": "user", "content": parts},
3148 {"role": "assistant", "content": parts, "tool_calls": [{
3149 "id": "c1", "type": "function", "function": {"name": "noop", "arguments": "{}"}
3150 }]},
3151 {"role": "tool", "tool_call_id": "c1", "content": parts},
3152 ]);
3153 let expected = concat!(
3154 "<system>part one\npart two</system>",
3155 "<developer>part one|part two</developer>",
3156 "<user>part one|part two</user>",
3157 "<assistant>part one|part two</assistant>",
3158 "<tool>part one|part two</tool>"
3159 );
3160 assert_eq!(render_shape(&f, shape.clone()).unwrap(), expected);
3161 assert_eq!(render_shape_with_tools(&f, shape).unwrap(), expected);
3162 }
3163
3164 #[test]
3165 fn text_array_support_is_selected_per_template_variant() {
3166 let native = TEXT_PARTS_TMPL.replace("ARRAY_SEPARATOR", "' '");
3167 let string =
3168 "{%- for m in messages -%}<{{ m.role }}>{{ m.content }}</{{ m.role }}>{%- endfor -%}";
3169 let shape = json!([{"role": "user", "content": text_parts("part one", "part two")}]);
3170 for (default, tool_use, default_text, tool_text) in [
3171 (
3172 native.as_str(),
3173 string,
3174 "part one part two",
3175 "part one\npart two",
3176 ),
3177 (
3178 string,
3179 native.as_str(),
3180 "part one\npart two",
3181 "part one part two",
3182 ),
3183 ] {
3184 let f = formatter_for_templates(default, tool_use);
3185 assert_eq!(
3186 render_shape(&f, shape.clone()).unwrap(),
3187 format!("<user>{default_text}</user>"),
3188 );
3189 assert_eq!(
3190 render_shape_with_tools(&f, shape.clone()).unwrap(),
3191 format!("<user>{tool_text}</user>"),
3192 );
3193 }
3194 }
3195
3196 #[test]
3197 fn merged_turns_keep_their_existing_join_with_native_text_arrays() {
3198 let strict = concat!(
3199 "{%- set ns = namespace(prev='') -%}{%- for m in messages -%}",
3200 "{%- if m.role == 'system' and not loop.first -%}",
3201 "{{ raise_exception('System message must be first.') }}",
3202 "{%- elif m.role == 'user' and ns.prev == 'user' -%}",
3203 "{{ raise_exception('Conversation roles must alternate.') }}{%- endif -%}",
3204 "<{{ m.role }}>{%- if m.content is string -%}{{ m.content }}{%- else -%}",
3205 "{%- for p in m.content -%}{{ p.text }}",
3206 "{%- if not loop.last -%}|{%- endif -%}{%- endfor -%}",
3207 "{%- endif -%}</{{ m.role }}>{%- set ns.prev = m.role -%}{%- endfor -%}"
3208 );
3209 let f = formatter_for(strict);
3210 assert!(f.default_system_normalization.demote_nonleading_system);
3211 assert!(f.default_system_normalization.coalesce_consecutive_users);
3212 let shape = json!([
3213 {"role": "system", "content": text_parts("s1", "s2")},
3214 {"role": "system", "content": "rules"},
3215 {"role": "user", "content": text_parts("u1", "u2")},
3216 {"role": "assistant", "content": "a"},
3217 {"role": "user", "content": "first"},
3218 {"role": "user", "content": text_parts("u3", "u4")},
3219 {"role": "user", "content": text_parts("u5", "u6")},
3220 {"role": "assistant", "content": "a2"},
3221 {"role": "user", "content": text_parts("u7", "u8")},
3222 {"role": "system", "content": text_parts("s3", "s4")},
3223 {"role": "assistant", "content": "a3"},
3224 {"role": "system", "content": text_parts("s5", "s6")},
3225 ]);
3226 let mut normalized = shape.clone();
3227 assert_eq!(
3228 normalize_system_messages(&mut normalized, f.default_system_normalization),
3229 [0, 3, 5],
3230 );
3231 assert_eq!(
3232 render_shape(&f, shape).unwrap(),
3233 concat!(
3234 "<system>s1\ns2\nrules</system>",
3235 "<user>u1|u2</user><assistant>a</assistant>",
3236 "<user>first\nu3\nu4\nu5\nu6</user><assistant>a2</assistant>",
3237 "<user>u7\nu8\ns3\ns4</user><assistant>a3</assistant>",
3238 "<user>s5|s6</user>"
3239 ),
3240 );
3241 }
3242
3243 #[test]
3244 fn demoted_unmerged_text_arrays_use_their_new_role() {
3245 let template = concat!(
3246 "{%- for m in messages -%}",
3247 "{%- if m.role == 'system' and not loop.first -%}",
3248 "{{ raise_exception('System message must be first.') }}{%- endif -%}",
3249 "<{{ m.role }}>{%- if m.role == 'system' or m.content is string -%}",
3250 "{{ m.content }}{%- else -%}{%- for p in m.content -%}{{ p.text }}",
3251 "{%- if not loop.last -%}{{ ' ' }}{%- endif -%}{%- endfor -%}",
3252 "{%- endif -%}</{{ m.role }}>{%- endfor -%}"
3253 );
3254 let f = formatter_for(template);
3255 assert!(!f.default_native_text_array_roles.contains(&"system"));
3256 assert!(f.default_native_text_array_roles.contains(&"user"));
3257 assert_eq!(
3258 render_shape(
3259 &f,
3260 json!([
3261 {"role": "user", "content": "first"},
3262 {"role": "assistant", "content": "done"},
3263 {"role": "system", "content": text_parts("part one", "part two")},
3264 ])
3265 )
3266 .unwrap(),
3267 "<user>first</user><assistant>done</assistant><user>part one part two</user>",
3268 );
3269 }
3270
3271 #[test]
3272 fn explicit_content_formats_override_native_text_array_detection() {
3273 let template = TEXT_PARTS_TMPL.replace("ARRAY_SEPARATOR", "' '");
3274 let mut f = formatter_for(&template);
3275 let shape = json!([{"role": "user", "content": text_parts("part one", "part two")}]);
3276 f.content_format = ChatTemplateContentFormat::String(MediaPlaceholders::default());
3277 assert_eq!(
3278 render_shape(&f, shape.clone()).unwrap(),
3279 "<user>part one\npart two</user>",
3280 );
3281 f.content_format = ChatTemplateContentFormat::OpenAi;
3282 assert_eq!(
3283 render_shape(&f, shape).unwrap(),
3284 "<user>part one part two</user>",
3285 );
3286 }
3287
3288 #[test]
3289 fn injected_reasoning_keeps_existing_text_array_boundaries() {
3290 let template = TEXT_PARTS_TMPL.replace("ARRAY_SEPARATOR", "' | '");
3291 for template in [
3292 template.clone(),
3293 template.replace("{{ p.text }}", "<text>{{ p.text }}</text>"),
3294 ] {
3295 let f = formatter_for(&template);
3296 assert!(!f.default_template_handles_reasoning);
3297 assert!(f.default_native_text_array_roles.contains(&"assistant"));
3298 for (reasoning, content, expected) in [
3299 (
3300 json!("r"),
3301 json!([{"type": "text", "text": "part one"}]),
3302 "<assistant><think>r</think>part one</assistant>",
3303 ),
3304 (
3305 json!(["", "r", ""]),
3306 text_parts("part one", "part two"),
3307 "<assistant><think>r</think>part one\npart two</assistant>",
3308 ),
3309 (
3310 json!(["r", "s"]),
3311 text_parts("part one", "part two"),
3312 "<assistant><think>r</think><think>s</think>part one\npart two</assistant>",
3313 ),
3314 (
3315 json!(" "),
3316 text_parts("part one", "part two"),
3317 "<assistant><think> </think>part one\npart two</assistant>",
3318 ),
3319 ] {
3320 let messages = json!([{
3321 "role": "assistant", "content": content, "reasoning_content": reasoning
3322 }]);
3323 assert_eq!(render_shape(&f, messages.clone()).unwrap(), expected);
3324 assert_eq!(render_shape_with_tools(&f, messages).unwrap(), expected);
3325 }
3326 }
3327
3328 let f = formatter_for(&template);
3329 for reasoning in [json!(""), json!([]), json!(["", ""])] {
3330 let messages = json!([{
3331 "role": "assistant", "content": text_parts("part one", "part two"),
3332 "reasoning_content": reasoning
3333 }]);
3334 assert_eq!(
3335 render_shape(&f, messages).unwrap(),
3336 "<assistant>part one | part two</assistant>",
3337 );
3338 }
3339
3340 let native_reasoning = template.replace(
3341 "<{{ m.role }}>",
3342 "<{{ m.role }}>{% if m.reasoning_content is string %}<native>{{ m.reasoning_content }}</native>{% endif %}",
3343 );
3344 let f = formatter_for(&native_reasoning);
3345 assert!(f.default_template_handles_reasoning);
3346 assert_eq!(
3347 render_shape(
3348 &f,
3349 json!([{
3350 "role": "assistant", "content": text_parts("part one", "part two"),
3351 "reasoning_content": "r"
3352 }])
3353 )
3354 .unwrap(),
3355 "<assistant><native>r</native>part one | part two</assistant>",
3356 );
3357 }
3358
3359 fn user() -> Msg {
3360 Msg::User(Default::default())
3361 }
3362 fn tool() -> Msg {
3363 Msg::Tool(Default::default())
3364 }
3365
3366 fn dummy_state(messages: Vec<Msg>) -> NvCreateChatCompletionRequest {
3367 let json = serde_json::json!({
3368 "model": "test-model",
3369 "messages": messages
3370 });
3371 serde_json::from_value(json).unwrap()
3372 }
3373
3374 #[test]
3375 fn add_after_user() {
3376 let s = dummy_state(vec![user()]);
3377 assert!(s.should_add_generation_prompt());
3378 }
3379
3380 #[test]
3381 fn add_after_tool() {
3382 let s = dummy_state(vec![tool()]);
3383 assert!(s.should_add_generation_prompt());
3384 }
3385
3386 #[test]
3387 fn add_when_empty() {
3388 let s = dummy_state(vec![]);
3389 assert!(s.should_add_generation_prompt());
3390 }
3391
3392 fn tool_aware_formatter(
3394 exclude_tools_when_tool_choice_none: bool,
3395 ) -> HfTokenizerConfigJsonFormatter {
3396 let template = r#"
3397{%- if tools is iterable and tools | length > 0 %}
3398TOOL_MODE tools={{ tools | length }}
3399{%- else %}
3400NORMAL_MODE
3401{%- endif %}
3402{{ messages[0].content }}"#;
3403
3404 let chat_template: super::tokcfg::ChatTemplate =
3405 serde_json::from_value(serde_json::json!({ "chat_template": template })).unwrap();
3406
3407 HfTokenizerConfigJsonFormatter::with_options(
3408 chat_template,
3409 ContextMixins::new(&[]),
3410 exclude_tools_when_tool_choice_none,
3411 )
3412 .unwrap()
3413 }
3414
3415 fn gemma4_tool_template_for_tests() -> &'static str {
3416 r#"
3417{{ bos_token }}
3418{%- set loop_messages = messages -%}
3419{%- set ns_turn = namespace(last_user_idx=-1) -%}
3420{%- for i in range(loop_messages | length) -%}
3421 {%- if loop_messages[i]['role'] == 'user' -%}
3422 {%- set ns_turn.last_user_idx = i -%}
3423 {%- endif -%}
3424{%- endfor -%}
3425{%- for message in loop_messages -%}
3426 {%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%}
3427 {{- '<|turn>' + role + '\n' }}
3428
3429 {%- if message.get('reasoning') and loop.index0 > ns_turn.last_user_idx and message.get('tool_calls') -%}
3430 {{- '<|channel>thought\n' + message['reasoning'] + '\n<channel|>'}}
3431 {%- endif -%}
3432
3433 {%- if message['tool_calls'] -%}
3434 {%- for tool_call in message['tool_calls'] -%}
3435 {%- set function = tool_call['function'] -%}
3436 {{- '<|tool_call>call:' + function['name'] + '{' -}}
3437 {%- if function['arguments'] is mapping -%}
3438 {%- set ns_args = namespace(found_first=false) -%}
3439 {%- for key, value in function['arguments'] | dictsort -%}
3440 {%- if ns_args.found_first %},{% endif -%}
3441 {%- set ns_args.found_first = true -%}
3442 {{- key -}}:{{- value -}}
3443 {%- endfor -%}
3444 {%- elif function['arguments'] is string -%}
3445 {{- function['arguments'] -}}
3446 {%- endif -%}
3447 {{- '}<tool_call|>' -}}
3448 {%- endfor -%}
3449 {%- endif -%}
3450
3451 {%- if message['content'] is string -%}
3452 {{- message['content'] -}}
3453 {%- endif -%}
3454 {{- '<turn|>\n' -}}
3455{%- endfor -%}
3456"#
3457 }
3458
3459 fn make_gemma4_tool_formatter_for_tests() -> HfTokenizerConfigJsonFormatter {
3460 let chat_template: ChatTemplate = serde_json::from_value(serde_json::json!({
3461 "chat_template": gemma4_tool_template_for_tests()
3462 }))
3463 .unwrap();
3464 HfTokenizerConfigJsonFormatter::new(chat_template, ContextMixins::new(&[])).unwrap()
3465 }
3466
3467 fn request_with_tool_choice(tool_choice: &str) -> NvCreateChatCompletionRequest {
3469 serde_json::from_value(serde_json::json!({
3470 "model": "test",
3471 "messages": [{"role": "user", "content": "hello"}],
3472 "tools": [{
3473 "type": "function",
3474 "function": {
3475 "name": "get_weather",
3476 "description": "Get weather",
3477 "parameters": {"type": "object", "properties": {"location": {"type": "string"}}}
3478 }
3479 }],
3480 "tool_choice": tool_choice
3481 }))
3482 .unwrap()
3483 }
3484
3485 #[test]
3486 fn test_exclude_tools_strips_when_tool_choice_none() {
3487 let formatter = tool_aware_formatter(true);
3488 let request = request_with_tool_choice("none");
3489 let result = formatter.render(&request).unwrap();
3490 assert!(
3491 result.contains("NORMAL_MODE"),
3492 "With exclude_tools=true and tool_choice=none, tools should be stripped. Got: {}",
3493 result
3494 );
3495 }
3496
3497 #[test]
3498 fn test_exclude_tools_keeps_when_tool_choice_auto() {
3499 let formatter = tool_aware_formatter(true);
3500 let request = request_with_tool_choice("auto");
3501 let result = formatter.render(&request).unwrap();
3502 assert!(
3503 result.contains("TOOL_MODE"),
3504 "With tool_choice=auto, tools should be included. Got: {}",
3505 result
3506 );
3507 }
3508
3509 #[test]
3510 fn test_no_exclude_tools_keeps_when_tool_choice_none() {
3511 let formatter = tool_aware_formatter(false);
3512 let request = request_with_tool_choice("none");
3513 let result = formatter.render(&request).unwrap();
3514 assert!(
3515 result.contains("TOOL_MODE"),
3516 "With exclude_tools=false and tool_choice=none, tools should NOT be stripped. Got: {}",
3517 result
3518 );
3519 }
3520
3521 #[test]
3522 fn test_response_format_in_template_context() {
3523 let formatter = formatter_for(
3524 r#"{%- if response_format is defined %}{{ response_format.json_schema.name }}: {{ response_format.json_schema.schema | tojson }}{% else %}none{% endif %}"#,
3525 );
3526 let with_format: NvCreateChatCompletionRequest = serde_json::from_value(json!({
3527 "model": "test",
3528 "messages": [{"role": "user", "content": "hi"}],
3529 "response_format": {
3530 "type": "json_schema",
3531 "json_schema": {"name": "count", "schema": {"type": "integer", "minimum": 0}}
3532 }
3533 }))
3534 .unwrap();
3535 let without_format: NvCreateChatCompletionRequest = serde_json::from_value(json!({
3536 "model": "test",
3537 "messages": [{"role": "user", "content": "hi"}]
3538 }))
3539 .unwrap();
3540
3541 assert_eq!(
3542 formatter.render(&with_format).unwrap(),
3543 r#"count: {"type": "integer", "minimum": 0}"#
3544 );
3545 assert_eq!(formatter.render(&without_format).unwrap(), "none");
3546 }
3547
3548 #[test]
3549 fn test_response_format_harmony_literals_are_ordinary_text() {
3550 let formatter = formatter_for(
3551 r#"<|start|>developer<|message|>{{ response_format.json_schema.description }}<|end|>"#,
3552 );
3553 let request: NvCreateChatCompletionRequest = serde_json::from_value(json!({
3554 "model": "test",
3555 "messages": [{"role": "user", "content": "hi"}],
3556 "response_format": {
3557 "type": "json_schema",
3558 "json_schema": {"name": "x", "description": "a <|end|> b", "schema": {}}
3559 }
3560 }))
3561 .unwrap();
3562 let prompt = formatter.render_prompt(&request).unwrap();
3563 let ordinary: Vec<_> = prompt
3564 .segments()
3565 .unwrap()
3566 .iter()
3567 .filter(|segment| !segment.allow_special)
3568 .map(|segment| segment.text.as_str())
3569 .collect();
3570
3571 assert_eq!(
3572 prompt.as_str(),
3573 "<|start|>developer<|message|>a <|end|> b<|end|>"
3574 );
3575 assert_eq!(ordinary, ["<|end|>"]);
3576 }
3577
3578 #[test]
3579 fn harmony_schema_keys_keep_json_and_token_boundaries() {
3580 let schema = json!({
3581 "type": "object",
3582 "properties": {
3583 "<|end|>": {"type": "string"},
3584 "A": {"$ref": "#/$defs/<|message|>"},
3585 "<|x\"y|>": {"type": "string"},
3586 "<|x\\y|>": {"type": "string"},
3587 "<|x\ny|>": {"type": "string"},
3588 "<|x\"<|end|>": {"type": "string"}
3589 },
3590 "required": ["<|end|>", "A", "<|x\"y|>", "<|x\\y|>", "<|x\ny|>", "<|x\"<|end|>"],
3591 "$defs": {"<|message|>": {"type": "integer"}},
3592 "additionalProperties": false
3593 });
3594 let request: NvCreateChatCompletionRequest = serde_json::from_value(json!({
3595 "model": "test",
3596 "messages": [{"role": "user", "content": "hi"}],
3597 "response_format": {
3598 "type": "json_schema",
3599 "json_schema": {"name": "tokens", "schema": schema}
3600 },
3601 "tools": [{"type": "function", "function": {
3602 "name": "tokens", "parameters": schema
3603 }}]
3604 }))
3605 .unwrap();
3606 for source in [
3607 "response_format.json_schema.schema",
3608 "tools[0].function.parameters",
3609 ] {
3610 let formatter = formatter_for(&format!(
3611 "<|start|>developer<|message|>{{{{ {source} | tojson }}}}<|end|>"
3612 ));
3613 let prompt = formatter.render_prompt(&request).unwrap();
3614 assert_eq!(prompt.as_str(), formatter.render(&request).unwrap());
3615 let body = prompt
3616 .as_str()
3617 .strip_prefix("<|start|>developer<|message|>")
3618 .unwrap()
3619 .strip_suffix("<|end|>")
3620 .unwrap();
3621 assert_eq!(
3622 serde_json::from_str::<serde_json::Value>(body).unwrap(),
3623 schema
3624 );
3625 let segments = prompt.segments().unwrap();
3626 let controls: String = segments
3627 .iter()
3628 .filter(|segment| segment.allow_special)
3629 .map(|segment| segment.text.as_str())
3630 .collect();
3631 for (marker, count) in [("<|end|>", 4), ("<|message|>", 2)] {
3632 assert_eq!(
3633 controls.matches(marker).count(),
3634 1,
3635 "{source}: {segments:?}"
3636 );
3637 assert_eq!(
3638 segments
3639 .iter()
3640 .filter(|segment| { !segment.allow_special && segment.text == marker })
3641 .count(),
3642 count,
3643 "{source}: {segments:?}"
3644 );
3645 }
3646 }
3647 }
3648
3649 #[test]
3650 fn harmony_schema_key_lookups_match_required_names() {
3651 let formatter = formatter_for(
3652 "{% set schema = response_format.json_schema.schema %}<|start|>developer<|message|>{{ schema.properties[schema.required[0]].type }}:{% for name, property in schema.properties.items() %}{{ name }}={{ name in schema.required }};{% endfor %}<|end|>",
3653 );
3654 let request: NvCreateChatCompletionRequest = serde_json::from_value(json!({
3655 "model": "test",
3656 "messages": [{"role": "user", "content": "hi"}],
3657 "response_format": {"type": "json_schema", "json_schema": {
3658 "name": "tokens", "schema": {
3659 "type": "object",
3660 "properties": {"<|end|>": {"type": "integer"}, "A": {"type": "string"}},
3661 "required": ["<|end|>"]
3662 }
3663 }}
3664 }))
3665 .unwrap();
3666 let prompt = formatter.render_prompt(&request).unwrap();
3667 assert_eq!(prompt.as_str(), formatter.render(&request).unwrap());
3668 assert!(
3669 prompt.as_str().contains("integer:<|end|>=True;A=False;"),
3670 "{prompt:?}"
3671 );
3672 assert!(
3673 prompt
3674 .segments()
3675 .unwrap()
3676 .iter()
3677 .any(|segment| { !segment.allow_special && segment.text == "<|end|>" })
3678 );
3679 }
3680
3681 #[test]
3682 fn test_inject_reasoning_content_segments_with_tool_calls() {
3683 let mut messages = serde_json::json!([
3685 {
3686 "role": "user",
3687 "content": "What is sqrt(144) and sqrt(256)?"
3688 },
3689 {
3690 "role": "assistant",
3691 "content": "Let me calculate those.",
3692 "reasoning_content": ["I need to compute sqrt(144)", "Now sqrt(256)", ""],
3693 "tool_calls": [
3694 {
3695 "id": "call_0",
3696 "type": "function",
3697 "function": {
3698 "name": "calculator",
3699 "arguments": "{\"expr\": \"sqrt(144)\"}"
3700 }
3701 },
3702 {
3703 "id": "call_1",
3704 "type": "function",
3705 "function": {
3706 "name": "calculator",
3707 "arguments": "{\"expr\": \"sqrt(256)\"}"
3708 }
3709 }
3710 ]
3711 }
3712 ]);
3713
3714 inject_reasoning_content_into_messages(&mut messages);
3715
3716 let assistant = &messages[1];
3717
3718 assert!(
3720 assistant.get("reasoning_content").is_none(),
3721 "reasoning_content should be removed after injection"
3722 );
3723
3724 let content = assistant["content"].as_str().unwrap();
3726 assert!(
3727 content.starts_with("<think>I need to compute sqrt(144)</think>"),
3728 "content should start with first reasoning segment, got: {}",
3729 content
3730 );
3731 assert!(
3732 content.contains("<think>Now sqrt(256)</think>"),
3733 "content should contain second reasoning segment"
3734 );
3735 assert!(
3737 !content.contains("<think></think>"),
3738 "empty segments should be skipped"
3739 );
3740 assert!(
3742 content.ends_with("Let me calculate those."),
3743 "original content should be at the end, got: {}",
3744 content
3745 );
3746
3747 assert!(assistant.get("tool_calls").is_some());
3749 assert_eq!(assistant["tool_calls"].as_array().unwrap().len(), 2);
3750 }
3751
3752 #[test]
3753 fn test_gemma4_template_renders_reasoning_content_segments_around_tool_calls() {
3754 let formatter = make_gemma4_tool_formatter_for_tests();
3755 assert!(
3756 formatter.tool_use_template_handles_reasoning,
3757 "Gemma4 template adaptation should make reasoning_content native"
3758 );
3759
3760 let request: NvCreateChatCompletionRequest = serde_json::from_value(serde_json::json!({
3761 "model": "gemma4-test",
3762 "messages": [
3763 {"role": "user", "content": "inspect two things"},
3764 {
3765 "role": "assistant",
3766 "content": null,
3767 "reasoning_content": [
3768 "Think before the first call.",
3769 "Think before the second call.",
3770 "Think after both calls."
3771 ],
3772 "tool_calls": [
3773 {
3774 "id": "call_0",
3775 "type": "function",
3776 "function": {
3777 "name": "first_tool",
3778 "arguments": "{\"path\":\".\"}"
3779 }
3780 },
3781 {
3782 "id": "call_1",
3783 "type": "function",
3784 "function": {
3785 "name": "second_tool",
3786 "arguments": "{\"path\":\"/tmp\"}"
3787 }
3788 }
3789 ]
3790 }
3791 ]
3792 }))
3793 .unwrap();
3794
3795 let rendered = formatter.render(&request).unwrap();
3796
3797 let expected = concat!(
3798 "<|channel>thought\nThink before the first call.\n<channel|>",
3799 "<|tool_call>call:first_tool{path:.}<tool_call|>",
3800 "<|channel>thought\nThink before the second call.\n<channel|>",
3801 "<|tool_call>call:second_tool{path:/tmp}<tool_call|>",
3802 "<|channel>thought\nThink after both calls.\n<channel|>"
3803 );
3804 assert!(
3805 rendered.contains(expected),
3806 "Gemma4 reasoning segments should stay adjacent to their tool calls, got: {rendered}"
3807 );
3808 assert!(!rendered.contains("<think>"));
3809 assert!(!rendered.contains("reasoning_content"));
3810 }
3811
3812 #[test]
3813 fn test_gemma4_template_renders_reasoning_content_without_tool_calls() {
3814 let formatter = make_gemma4_tool_formatter_for_tests();
3815 let request: NvCreateChatCompletionRequest = serde_json::from_value(serde_json::json!({
3816 "model": "gemma4-test",
3817 "messages": [
3818 {"role": "user", "content": "answer directly"},
3819 {
3820 "role": "assistant",
3821 "content": "Direct answer.",
3822 "reasoning_content": "Private thought."
3823 }
3824 ]
3825 }))
3826 .unwrap();
3827
3828 let rendered = formatter.render(&request).unwrap();
3829
3830 assert!(
3831 rendered.contains("<|channel>thought\nPrivate thought.\n<channel|>Direct answer."),
3832 "Gemma4 reasoning_content should render in the thought channel, got: {rendered}"
3833 );
3834 assert!(!rendered.contains("<think>"));
3835 assert!(!rendered.contains("reasoning_content"));
3836 }
3837
3838 #[test]
3844 fn test_reasoning_flag_is_per_template_not_global() {
3845 const PLAIN_DEFAULT: &str = "{{ bos_token }}{%- for message in messages -%}\
3848 {{ message['role'] }}: {{ message['content'] }}\n{%- endfor -%}";
3849
3850 let chat_template: ChatTemplate = serde_json::from_value(serde_json::json!({
3851 "chat_template": [
3852 {"default": PLAIN_DEFAULT},
3853 {"tool_use": gemma4_tool_template_for_tests()},
3854 ]
3855 }))
3856 .unwrap();
3857 let formatter =
3858 HfTokenizerConfigJsonFormatter::new(chat_template, ContextMixins::new(&[])).unwrap();
3859
3860 assert!(
3864 formatter.tool_use_template_handles_reasoning,
3865 "adapted gemma4 tool_use template should handle reasoning natively"
3866 );
3867 assert!(
3868 !formatter.default_template_handles_reasoning,
3869 "plain default template does not reference reasoning_content"
3870 );
3871
3872 let request: NvCreateChatCompletionRequest = serde_json::from_value(serde_json::json!({
3875 "model": "gemma4-test",
3876 "messages": [
3877 {"role": "user", "content": "answer directly"},
3878 {
3879 "role": "assistant",
3880 "content": "Direct answer.",
3881 "reasoning_content": "Private thought."
3882 }
3883 ]
3884 }))
3885 .unwrap();
3886
3887 let rendered = formatter.render(&request).unwrap();
3888 assert!(
3889 rendered.contains("<think>Private thought.</think>Direct answer."),
3890 "reasoning must be injected on the no-tool default path, got: {rendered}"
3891 );
3892 }
3893
3894 fn reasoning_tool_call_turn(reasoning: serde_json::Value) -> serde_json::Value {
3896 let call = |id: &str, expr: &str| {
3897 json!({"id": id, "type": "function",
3898 "function": {"name": "calc", "arguments": json!({"expr": expr}).to_string()}})
3899 };
3900 json!([
3901 {"role": "user", "content": "sqrt(144) + sqrt(256)?"},
3902 {"role": "assistant", "content": null, "reasoning_content": reasoning,
3903 "tool_calls": [call("call_0", "sqrt(144)"), call("call_1", "sqrt(256)")]},
3904 {"role": "tool", "tool_call_id": "call_0", "content": "12"},
3905 {"role": "tool", "tool_call_id": "call_1", "content": "16"}
3906 ])
3907 }
3908
3909 #[test]
3913 fn test_string_reasoning_template_joins_reasoning_content_segments() {
3914 const MINIMAX_REASONING_TMPL: &str = r#"{%- for message in messages -%}
3915{%- if message.role == 'assistant' -%}
3916{{- ']~b]ai' ~ '\n' -}}
3917{%- set reasoning_content = '' -%}
3918{%- if message.reasoning_content is string -%}
3919{%- set reasoning_content = message.reasoning_content -%}
3920{%- endif -%}
3921{%- if reasoning_content -%}
3922{{- '<think>' ~ '\n' ~ reasoning_content ~ '\n' ~ '</think>' ~ '\n\n' -}}
3923{%- endif -%}
3924{%- for tool_call in message.tool_calls -%}
3925{{- '<invoke name="' ~ tool_call.function.name ~ '">' -}}
3926{%- endfor -%}
3927{{- '[e~[\n' -}}
3928{%- else -%}
3929{{- ']~b]' ~ message.role ~ '\n' ~ message.content ~ '[e~[\n' -}}
3930{%- endif -%}
3931{%- endfor -%}"#;
3932 let f = formatter_for(MINIMAX_REASONING_TMPL);
3933 let expected = concat!(
3935 "]~b]user\nsqrt(144) + sqrt(256)?[e~[\n",
3936 "]~b]ai\n<think>\nCheck both.\nThen add.\n</think>\n\n",
3937 "<invoke name=\"calc\"><invoke name=\"calc\">[e~[\n",
3938 "]~b]tool\n12[e~[\n]~b]tool\n16[e~[\n",
3939 );
3940 for render in [render_shape, render_shape_with_tools] {
3942 for reasoning in [
3943 json!(["Check both.", "Then add.", ""]),
3944 json!("Check both.\nThen add."),
3945 ] {
3946 assert_eq!(
3947 render(&f, reasoning_tool_call_turn(reasoning)).unwrap(),
3948 expected
3949 );
3950 }
3951 }
3952 }
3953
3954 #[test]
3955 fn test_inject_reasoning_content_text_variant() {
3956 let mut messages = serde_json::json!([
3957 {
3958 "role": "assistant",
3959 "content": "The answer is 42.",
3960 "reasoning_content": "Let me think about this carefully."
3961 }
3962 ]);
3963
3964 inject_reasoning_content_into_messages(&mut messages);
3965
3966 let assistant = &messages[0];
3967 assert!(assistant.get("reasoning_content").is_none());
3968 let content = assistant["content"].as_str().unwrap();
3969 assert_eq!(
3970 content,
3971 "<think>Let me think about this carefully.</think>The answer is 42."
3972 );
3973 }
3974
3975 #[test]
3976 fn test_inject_reasoning_content_null_content() {
3977 let mut messages = serde_json::json!([
3979 {
3980 "role": "assistant",
3981 "content": null,
3982 "reasoning_content": "Thinking...",
3983 "tool_calls": [{"id": "call_0", "type": "function", "function": {"name": "f", "arguments": "{}"}}]
3984 }
3985 ]);
3986
3987 inject_reasoning_content_into_messages(&mut messages);
3988
3989 let content = messages[0]["content"].as_str().unwrap();
3990 assert_eq!(content, "<think>Thinking...</think>");
3991 assert!(messages[0].get("reasoning_content").is_none());
3992 }
3993
3994 #[test]
3995 fn test_inject_reasoning_content_skips_non_assistant() {
3996 let mut messages = serde_json::json!([
3997 {
3998 "role": "user",
3999 "content": "hello",
4000 "reasoning_content": "should not be touched"
4001 }
4002 ]);
4003
4004 inject_reasoning_content_into_messages(&mut messages);
4005
4006 assert!(messages[0].get("reasoning_content").is_some());
4008 }
4009
4010 fn make_test_formatter() -> HfTokenizerConfigJsonFormatter {
4012 use super::tokcfg::ChatTemplate;
4013 use super::{ContextMixins, HfTokenizerConfigJsonFormatter};
4014
4015 let template = r#"{%- for message in messages %}{{ message.role }}: {{ message.content }}
4018{%- endfor %}
4019{%- if add_generation_prompt %}assistant:{%- endif %}"#;
4020
4021 let chat_template: ChatTemplate = serde_json::from_value(serde_json::json!({
4022 "chat_template": template
4023 }))
4024 .unwrap();
4025
4026 HfTokenizerConfigJsonFormatter::new(chat_template, ContextMixins::new(&[])).unwrap()
4027 }
4028
4029 #[test]
4032 fn test_reasoning_content_text_roundtrip_render() {
4033 use super::OAIPromptFormatter;
4034 let formatter = make_test_formatter();
4035
4036 let request: NvCreateChatCompletionRequest = serde_json::from_value(serde_json::json!({
4037 "model": "test-model",
4038 "messages": [
4039 {"role": "user", "content": "What is sqrt(144)?"},
4040 {
4041 "role": "assistant",
4042 "content": "The answer is 12.",
4043 "reasoning_content": "I need to compute the square root of 144."
4044 },
4045 {"role": "user", "content": "Are you sure?"}
4046 ]
4047 }))
4048 .unwrap();
4049
4050 let rendered = formatter.render(&request).unwrap();
4051
4052 assert!(
4053 rendered.contains("<think>I need to compute the square root of 144.</think>"),
4054 "reasoning_content must appear as <think> block, got: {}",
4055 rendered
4056 );
4057 assert!(
4058 rendered.contains("The answer is 12."),
4059 "original content must be preserved"
4060 );
4061 assert!(
4062 !rendered.contains("reasoning_content"),
4063 "raw reasoning_content field should not leak into prompt"
4064 );
4065 }
4066
4067 #[test]
4071 fn test_reasoning_content_agentic_tool_call_roundtrip_render() {
4072 use super::OAIPromptFormatter;
4073 let formatter = make_test_formatter();
4074
4075 let request: NvCreateChatCompletionRequest = serde_json::from_value(serde_json::json!({
4076 "model": "test-model",
4077 "messages": [
4078 {"role": "user", "content": "What is sqrt(144) + sqrt(256)?"},
4079 {
4080 "role": "assistant",
4081 "content": null,
4082 "reasoning_content": "I need to compute both square roots. Let me start with sqrt(144).",
4083 "tool_calls": [{
4084 "id": "call_0",
4085 "type": "function",
4086 "function": {
4087 "name": "calculator",
4088 "arguments": "{\"expr\": \"sqrt(144)\"}"
4089 }
4090 }]
4091 },
4092 {
4093 "role": "tool",
4094 "tool_call_id": "call_0",
4095 "content": "12"
4096 },
4097 {
4098 "role": "assistant",
4099 "content": "sqrt(144) = 12 and sqrt(256) = 16, so the answer is 28.",
4100 "reasoning_content": "Got 12 for sqrt(144). Now sqrt(256) = 16. Sum is 28."
4101 },
4102 {"role": "user", "content": "Thanks!"}
4103 ]
4104 }))
4105 .unwrap();
4106
4107 let rendered = formatter.render(&request).unwrap();
4108
4109 assert!(
4111 rendered.contains("<think>I need to compute both square roots"),
4112 "first turn reasoning must be in prompt, got: {}",
4113 rendered
4114 );
4115 assert!(
4117 rendered.contains("<think>Got 12 for sqrt(144)"),
4118 "second turn reasoning must be in prompt"
4119 );
4120 assert!(
4121 rendered.contains("the answer is 28"),
4122 "final answer content must be preserved"
4123 );
4124 assert!(
4126 !rendered.contains("reasoning_content"),
4127 "raw reasoning_content field should not leak into prompt"
4128 );
4129 }
4130
4131 #[test]
4133 fn test_reasoning_injected_when_template_ignores_it() {
4134 use super::OAIPromptFormatter;
4135 let formatter = make_test_formatter();
4136
4137 assert!(!formatter.default_template_handles_reasoning);
4139 assert!(!formatter.tool_use_template_handles_reasoning);
4140
4141 let request: NvCreateChatCompletionRequest = serde_json::from_value(serde_json::json!({
4142 "model": "test-model",
4143 "messages": [
4144 {"role": "user", "content": "Hello"},
4145 {
4146 "role": "assistant",
4147 "content": "Hi.",
4148 "reasoning_content": "The user said hello."
4149 },
4150 {"role": "user", "content": "Bye"}
4151 ]
4152 }))
4153 .unwrap();
4154
4155 let rendered = formatter.render(&request).unwrap();
4156 assert!(
4157 rendered.contains("<think>The user said hello.</think>"),
4158 "injection must happen when template ignores reasoning_content, got: {}",
4159 rendered
4160 );
4161 }
4162
4163 #[test]
4165 fn test_reasoning_not_injected_when_template_handles_it() {
4166 use super::tokcfg::ChatTemplate;
4167 use super::{ContextMixins, HfTokenizerConfigJsonFormatter, OAIPromptFormatter};
4168
4169 let template = r#"{%- for message in messages %}{%- if message.role == "assistant" and message.reasoning_content is defined and message.reasoning_content %}<think>{{ message.reasoning_content }}</think>
4171{%- endif %}{{ message.role }}: {{ message.content }}
4172{%- endfor %}
4173{%- if add_generation_prompt %}assistant:{%- endif %}"#;
4174
4175 let chat_template: ChatTemplate = serde_json::from_value(serde_json::json!({
4176 "chat_template": template
4177 }))
4178 .unwrap();
4179
4180 let formatter =
4181 HfTokenizerConfigJsonFormatter::new(chat_template, ContextMixins::new(&[])).unwrap();
4182
4183 assert!(formatter.default_template_handles_reasoning);
4185 assert!(formatter.tool_use_template_handles_reasoning);
4186
4187 let request: NvCreateChatCompletionRequest = serde_json::from_value(serde_json::json!({
4188 "model": "test-model",
4189 "messages": [
4190 {"role": "user", "content": "Hello"},
4191 {
4192 "role": "assistant",
4193 "content": "Hi.",
4194 "reasoning_content": "The user said hello."
4195 },
4196 {"role": "user", "content": "Bye"}
4197 ]
4198 }))
4199 .unwrap();
4200
4201 let rendered = formatter.render(&request).unwrap();
4202
4203 assert!(
4205 rendered.contains("<think>The user said hello.</think>"),
4206 "template must render reasoning_content natively, got: {}",
4207 rendered
4208 );
4209 let think_count = rendered.matches("<think>").count();
4211 assert_eq!(
4212 think_count, 1,
4213 "must have exactly one <think> block (from template), got {} in: {}",
4214 think_count, rendered
4215 );
4216 }
4217
4218 const QWEN3_THINKING_TEMPLATE: &str = r##"{%- if tools %}
4222 {{- '<|im_start|>system\n' }}
4223 {%- if messages[0].role == 'system' %}
4224 {{- messages[0].content + '\n\n' }}
4225 {%- endif %}
4226 {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
4227 {%- for tool in tools %}
4228 {{- "\n" }}
4229 {{- tool | tojson }}
4230 {%- endfor %}
4231 {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
4232{%- else %}
4233 {%- if messages[0].role == 'system' %}
4234 {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
4235 {%- endif %}
4236{%- endif %}
4237{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
4238{%- for message in messages[::-1] %}
4239 {%- set index = (messages|length - 1) - loop.index0 %}
4240 {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
4241 {%- set ns.multi_step_tool = false %}
4242 {%- set ns.last_query_index = index %}
4243 {%- endif %}
4244{%- endfor %}
4245{%- for message in messages %}
4246 {%- if message.content is string %}
4247 {%- set content = message.content %}
4248 {%- else %}
4249 {%- set content = '' %}
4250 {%- endif %}
4251 {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
4252 {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
4253 {%- elif message.role == "assistant" %}
4254 {%- set reasoning_content = '' %}
4255 {%- if message.reasoning_content is string %}
4256 {%- set reasoning_content = message.reasoning_content %}
4257 {%- else %}
4258 {%- if '</think>' in content %}
4259 {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
4260 {%- set content = content.split('</think>')[-1].lstrip('\n') %}
4261 {%- endif %}
4262 {%- endif %}
4263 {%- if loop.index0 > ns.last_query_index %}
4264 {%- if loop.last or (not loop.last and reasoning_content) %}
4265 {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
4266 {%- else %}
4267 {{- '<|im_start|>' + message.role + '\n' + content }}
4268 {%- endif %}
4269 {%- else %}
4270 {{- '<|im_start|>' + message.role + '\n' + content }}
4271 {%- endif %}
4272 {%- if message.tool_calls %}
4273 {%- for tool_call in message.tool_calls %}
4274 {%- if (loop.first and content) or (not loop.first) %}
4275 {{- '\n' }}
4276 {%- endif %}
4277 {%- if tool_call.function %}
4278 {%- set tool_call = tool_call.function %}
4279 {%- endif %}
4280 {{- '<tool_call>\n{"name": "' }}
4281 {{- tool_call.name }}
4282 {{- '", "arguments": ' }}
4283 {%- if tool_call.arguments is string %}
4284 {{- tool_call.arguments }}
4285 {%- else %}
4286 {{- tool_call.arguments | tojson }}
4287 {%- endif %}
4288 {{- '}\n</tool_call>' }}
4289 {%- endfor %}
4290 {%- endif %}
4291 {{- '<|im_end|>\n' }}
4292 {%- elif message.role == "tool" %}
4293 {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
4294 {{- '<|im_start|>user' }}
4295 {%- endif %}
4296 {{- '\n<tool_response>\n' }}
4297 {{- content }}
4298 {{- '\n</tool_response>' }}
4299 {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
4300 {{- '<|im_end|>\n' }}
4301 {%- endif %}
4302 {%- endif %}
4303{%- endfor %}
4304{%- if add_generation_prompt %}
4305 {{- '<|im_start|>assistant\n<think>\n' }}
4306{%- endif %}"##;
4307
4308 fn qwen3_thinking_formatter() -> HfTokenizerConfigJsonFormatter {
4309 let chat_template: ChatTemplate = serde_json::from_value(serde_json::json!({
4310 "chat_template": QWEN3_THINKING_TEMPLATE,
4311 }))
4312 .unwrap();
4313 HfTokenizerConfigJsonFormatter::new(chat_template, ContextMixins::new(&[])).unwrap()
4314 }
4315
4316 #[test]
4317 fn test_qwen3_thinking_template_flags_detected() {
4318 let formatter = qwen3_thinking_formatter();
4319 assert!(
4320 formatter.tool_use_template_handles_reasoning,
4321 "template references reasoning_content directly"
4322 );
4323 assert!(
4326 formatter.default_template_handles_tool_calls_arguments_string,
4327 "default template branches on `arguments is string`"
4328 );
4329 assert!(
4330 formatter.tool_use_template_handles_tool_calls_arguments_string,
4331 "tool_use template branches on `arguments is string`"
4332 );
4333 }
4334
4335 const QWEN38_REJECTS_STRING_ARGS_TEMPLATE: &str = r##"{%- for message in messages %}
4339 {%- if message.role == "assistant" and message.tool_calls %}
4340 {%- for tool_call in message.tool_calls %}
4341 {%- if tool_call.function %}
4342 {%- set tool_call = tool_call.function %}
4343 {%- endif %}
4344 {{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
4345 {%- if tool_call.arguments is mapping %}
4346 {%- for args_name, args_value in tool_call.arguments|items %}
4347 {{- '<parameter=' + args_name + '>\n' + args_value + '\n</parameter>\n' }}
4348 {%- endfor %}
4349 {%- elif tool_call.arguments is string %}
4350 {%- if tool_call.arguments|trim %}
4351 {{- raise_exception('Tool call arguments were passed as a JSON string.') }}
4352 {%- endif %}
4353 {%- endif %}
4354 {{- '</function>\n</tool_call>' }}
4355 {%- endfor %}
4356 {%- else %}
4357 {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>\n' }}
4358 {%- endif %}
4359{%- endfor %}"##;
4360
4361 #[test]
4365 fn test_template_rejecting_string_arguments_gets_objects() {
4366 let chat_template: ChatTemplate = serde_json::from_value(serde_json::json!({
4367 "chat_template": QWEN38_REJECTS_STRING_ARGS_TEMPLATE,
4368 }))
4369 .unwrap();
4370 let formatter =
4371 HfTokenizerConfigJsonFormatter::new(chat_template, ContextMixins::new(&[])).unwrap();
4372 assert!(!formatter.default_template_handles_tool_calls_arguments_string);
4373 assert!(!formatter.tool_use_template_handles_tool_calls_arguments_string);
4374
4375 let request: NvCreateChatCompletionRequest = serde_json::from_value(serde_json::json!({
4376 "model": "qwen3.8",
4377 "messages": [
4378 {"role": "user", "content": "What's the weather in San Francisco?"},
4379 {"role": "assistant", "content": "", "tool_calls": [{
4380 "id": "call_sf",
4381 "type": "function",
4382 "function": {"name": "get_weather", "arguments": "{\"location\": \"San Francisco\"}"}
4383 }]},
4384 {"role": "tool", "tool_call_id": "call_sf", "content": "Foggy"}
4385 ],
4386 }))
4387 .unwrap();
4388 let rendered = formatter.render(&request).unwrap();
4389 assert!(
4390 rendered.contains("<parameter=location>\nSan Francisco\n</parameter>"),
4391 "{rendered}"
4392 );
4393 }
4394
4395 #[test]
4405 fn test_qwen3_thinking_append_only_across_tool_use_turn() {
4406 let formatter = qwen3_thinking_formatter();
4407
4408 let tools = serde_json::json!([{
4409 "type": "function",
4410 "function": {
4411 "name": "get_weather",
4412 "description": "Get the current weather for a location",
4413 "parameters": {
4414 "type": "object",
4415 "properties": {
4416 "location": {"type": "string"},
4417 "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}
4418 },
4419 "required": ["location"]
4420 }
4421 }
4422 }]);
4423
4424 let turn1_request: NvCreateChatCompletionRequest =
4426 serde_json::from_value(serde_json::json!({
4427 "model": "qwen3-thinking",
4428 "messages": [
4429 {"role": "system", "content": "You are a helpful assistant."},
4430 {"role": "user", "content": "What's the weather in San Francisco?"},
4431 ],
4432 "tools": tools,
4433 }))
4434 .unwrap();
4435 let p1 = formatter.render(&turn1_request).unwrap();
4436
4437 let model_emitted = "I'll call get_weather for SF.\n\
4441 </think>\n\n\
4442 <tool_call>\n\
4443 {\"name\": \"get_weather\", \"arguments\": {\"location\": \"San Francisco\", \"unit\": \"celsius\"}}\n\
4444 </tool_call><|im_end|>\n";
4445 let wire_after_t1 = format!("{p1}{model_emitted}");
4446
4447 let turn2_request: NvCreateChatCompletionRequest =
4450 serde_json::from_value(serde_json::json!({
4451 "model": "qwen3-thinking",
4452 "messages": [
4453 {"role": "system", "content": "You are a helpful assistant."},
4454 {"role": "user", "content": "What's the weather in San Francisco?"},
4455 {
4456 "role": "assistant",
4457 "content": "",
4458 "reasoning_content": "I'll call get_weather for SF.",
4459 "tool_calls": [{
4460 "id": "call_sf",
4461 "type": "function",
4462 "function": {
4463 "name": "get_weather",
4464 "arguments": "{\"location\": \"San Francisco\", \"unit\": \"celsius\"}"
4465 }
4466 }]
4467 },
4468 {
4469 "role": "tool",
4470 "tool_call_id": "call_sf",
4471 "content": "{\"temp\": 18, \"conditions\": \"Foggy\"}"
4472 }
4473 ],
4474 "tools": tools,
4475 }))
4476 .unwrap();
4477 let p2 = formatter.render(&turn2_request).unwrap();
4478
4479 if !p2.starts_with(&wire_after_t1) {
4480 let div = wire_after_t1
4482 .as_bytes()
4483 .iter()
4484 .zip(p2.as_bytes())
4485 .position(|(a, b)| a != b)
4486 .unwrap_or_else(|| wire_after_t1.len().min(p2.len()));
4487 let lo = div.saturating_sub(40);
4488 panic!(
4489 "turn-2 prompt is NOT a prefix-extension of [turn-1 + model bytes]\n \
4490 diverges at byte {div}\n \
4491 wire ends: ...{}|{}\n \
4492 t2 has: ...{}|{}",
4493 String::from_utf8_lossy(&wire_after_t1.as_bytes()[lo..div]),
4494 String::from_utf8_lossy(
4495 &wire_after_t1.as_bytes()[div..(div + 60).min(wire_after_t1.len())]
4496 ),
4497 String::from_utf8_lossy(&p2.as_bytes()[lo..div]),
4498 String::from_utf8_lossy(&p2.as_bytes()[div..(div + 60).min(p2.len())]),
4499 );
4500 }
4501
4502 let suffix = &p2[wire_after_t1.len()..];
4505 assert!(
4506 suffix.contains("<tool_response>"),
4507 "appended bytes must include the tool response, got: {suffix}"
4508 );
4509 assert!(
4510 suffix.ends_with("<|im_start|>assistant\n<think>\n"),
4511 "appended bytes must end with the next generation prompt, got: {suffix}"
4512 );
4513 }
4514
4515 #[test]
4519 fn test_qwen3_thinking_renders_reasoning_content_segments_as_string() {
4520 let formatter = qwen3_thinking_formatter();
4521 assert!(formatter.default_template_requires_reasoning_string);
4522 assert!(formatter.tool_use_template_requires_reasoning_string);
4523
4524 for render in [render_shape, render_shape_with_tools] {
4525 let segments = json!(["Check both.", "Then add.", ""]);
4526 let rendered = render(&formatter, reasoning_tool_call_turn(segments)).unwrap();
4527 assert!(
4528 rendered.contains(
4529 "<|im_start|>assistant\n<think>\nCheck both.\nThen add.\n</think>\n\n<tool_call>"
4530 ),
4531 "{rendered}"
4532 );
4533 let string = json!("Check both.\nThen add.");
4534 assert_eq!(
4535 rendered,
4536 render(&formatter, reasoning_tool_call_turn(string)).unwrap()
4537 );
4538 }
4539 }
4540
4541 fn contents(messages: &serde_json::Value) -> serde_json::Value {
4544 messages
4545 .as_array()
4546 .unwrap()
4547 .iter()
4548 .map(|m| m["content"].clone())
4549 .collect()
4550 }
4551
4552 #[test]
4554 fn content_format_string_flattens_like_vllm() {
4555 let placeholders = MediaPlaceholders {
4556 image: "<|image_{n}|>".into(),
4557 video: "<video>".into(),
4558 audio: "<|audio_{n}|>".into(),
4559 };
4560 let image = json!({"type": "image_url", "image_url": {"url": "x"}});
4561 let flatten = |mut messages: serde_json::Value, use_audio_in_video| {
4562 flatten_msg_content_to_string(&mut messages, &placeholders, use_audio_in_video)
4563 .map(|()| contents(&messages))
4564 };
4565
4566 let messages = json!([
4568 {"role": "user", "content": [
4569 {"type": "text", "text": "a"}, &image,
4570 {"type": "video_url", "video_url": {"url": "y"}},
4571 {"type": "text", "text": "b"}, &image
4572 ]},
4573 {"role": "user", "content": [&image]}
4574 ]);
4575 assert_eq!(
4576 flatten(messages, false).unwrap(),
4577 json!(["<|image_1|>\n<|image_2|>\n<video>\na\nb", "<|image_3|>"])
4578 );
4579 let messages = json!([
4581 {"role": "system", "content": "plain"},
4582 {"role": "assistant"},
4583 {"role": "assistant", "content": [
4584 {"type": "text", "text": "a"},
4585 {"type": "text", "text": ""},
4586 {"type": "refusal", "refusal": ""},
4587 {"type": "refusal", "refusal": "b"}
4588 ]}
4589 ]);
4590 assert_eq!(
4591 flatten(messages, false).unwrap(),
4592 json!(["plain", "", "a\nb"])
4593 );
4594 let messages = json!([{"role": "user", "content": [
4596 {"type": "text", "text": "see <|image_1|> here"}, &image, &image
4597 ]}]);
4598 assert_eq!(
4599 flatten(messages, false).unwrap(),
4600 json!(["<|image_2|>\nsee <|image_1|> here"])
4601 );
4602 let messages = json!([{"role": "user", "content": [
4604 {"type": "video_url", "video_url": {"url": "y"}},
4605 {"type": "video_url", "video_url": null, "uuid": "cached"},
4606 {"type": "video_url", "video_url": {"url": ""}, "uuid": "cached"},
4607 {"type": "input_audio", "input_audio": {"data": "", "format": "wav"}},
4608 {"type": "text", "text": "x"}
4609 ]}]);
4610 assert_eq!(
4611 flatten(messages, true).unwrap(),
4612 json!(["<video>\n<video>\n<video>\n<|audio_1|>\n<|audio_2|>\nx"])
4613 );
4614
4615 for content in [
4616 json!([{"type": "text", "text": "<|image_1|><|image_1|>"}, &image]),
4617 json!([{"type": "custom", "kind": "chemistry", "fields": {}}]),
4618 ] {
4619 let error = flatten(json!([{"role": "user", "content": content}]), false).unwrap_err();
4620 assert!(
4621 matches!(
4622 error.downcast_ref::<PromptRenderError>(),
4623 Some(PromptRenderError::InvalidRequest(_))
4624 ),
4625 "{error}"
4626 );
4627 }
4628 }
4629
4630 #[test]
4632 fn content_format_openai_wraps_content_in_parts_like_vllm() {
4633 let mut messages = json!([
4634 {"role": "system", "content": "sys"},
4635 {"role": "user", "content": [
4636 {"type": "text", "text": "look", "source_lang_code": "en"},
4637 {"type": "image_url", "image_url": {"url": "x"}, "uuid": "i", "target_lang_code": "de"},
4638 {"type": "input_audio", "input_audio": {"data": "", "format": "wav"}},
4639 {"type": "custom", "kind": "chemistry", "fields": {"data_base64": "x"}}
4640 ]},
4641 {"role": "assistant", "content": [{"type": "refusal", "refusal": "no"}]},
4642 {"role": "assistant"},
4643 {"role": "tool", "tool_call_id": "1", "content": [
4644 {"type": "text", "text": "a"},
4645 {"type": "text", "text": "b"}
4646 ]},
4647 {"role": "tool", "tool_call_id": "2", "content": "done"}
4648 ]);
4649 wrap_msg_content_in_parts(&mut messages);
4650 assert_eq!(
4651 contents(&messages),
4652 json!([
4653 [{"type": "text", "text": "sys"}],
4654 [
4655 {"type": "text", "text": "look", "source_lang_code": "en"},
4656 {"type": "image", "target_lang_code": "de"},
4657 {"type": "audio"},
4658 {"type": "custom", "kind": "chemistry"}
4659 ],
4660 [{"type": "text", "text": "no"}],
4661 [],
4662 "a\nb",
4663 "done",
4664 ])
4665 );
4666 }
4667
4668 #[test]
4670 fn content_format_reaches_render_through_from_parts() {
4671 let template = "{% for m in messages %}{% if m.content is string %}{{ m.content }}\
4672 {% else %}{% for p in m.content %}[{{ p.type }}{% if p.text %} {{ p.text }}{% endif %}]\
4673 {% endfor %}{% endif %}|{% endfor %}";
4674 let request: NvCreateChatCompletionRequest = serde_json::from_value(json!({
4675 "model": "test",
4676 "messages": [
4677 {"role": "system", "content": "sys"},
4678 {"role": "user", "content": [
4679 {"type": "text", "text": "hi"},
4680 {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}},
4681 {"type": "video_url", "video_url": {"url": "https://example.com/v.mp4"}}
4682 ]}
4683 ],
4684 "mm_processor_kwargs": {"use_audio_in_video": 1}
4686 }))
4687 .unwrap();
4688 let render = |content_format| {
4689 let config: SysChatTemplate =
4690 serde_json::from_value(json!({ "chat_template": template })).unwrap();
4691 let crate::PromptFormatter::OAI(formatter) =
4692 crate::PromptFormatter::from_parts_with_content_format(
4693 config,
4694 SysMixins::new(&[]),
4695 true,
4696 content_format,
4697 )
4698 .unwrap();
4699 formatter.render(&request).unwrap()
4700 };
4701
4702 assert_eq!(
4703 render(ChatTemplateContentFormat::Auto),
4704 "sys|[text hi][image][video]|"
4705 );
4706 assert_eq!(
4707 render(ChatTemplateContentFormat::OpenAi),
4708 "[text sys]|[text hi][image][video]|"
4709 );
4710 let placeholders = MediaPlaceholders {
4711 image: "<image>".into(),
4712 video: "<video>".into(),
4713 audio: "<audio>".into(),
4714 };
4715 assert_eq!(
4716 render(ChatTemplateContentFormat::String(placeholders)),
4717 "sys|<image>\n<video>\n<audio>\nhi|"
4718 );
4719 }
4720}