1use super::*;
5
6use crate::{OAIChatLikeRequest, PromptRenderError, TextInput};
7use minijinja::{context, value::Value};
8use serde_json::json;
9use std::{collections::HashMap, result::Result::Ok};
10
11pub fn may_be_fix_tool_schema(tools: serde_json::Value) -> Option<Value> {
15 let mut updated_tools = Vec::new();
19 if let Some(arr) = tools.as_array() {
20 for tool in arr {
21 let mut tool = tool.clone();
22 if let Some(function) = tool.get_mut("function") {
23 if let Some(obj) = function.as_object_mut()
28 && !matches!(obj.get("description"), Some(serde_json::Value::String(_)))
29 {
30 obj.insert(
31 "description".to_string(),
32 serde_json::Value::String(String::new()),
33 );
34 }
35 }
36 if let Some(function) = tool.get_mut("function")
37 && let Some(parameters) = function.get_mut("parameters")
38 {
39 if parameters.is_object() {
41 let mut needs_type = false;
42 let mut needs_properties = false;
43 let is_empty = parameters
44 .as_object()
45 .map(|o| o.is_empty())
46 .unwrap_or(false);
47
48 if is_empty {
50 needs_type = true;
51 needs_properties = true;
52 } else {
53 if let Some(obj) = parameters.as_object() {
55 if !obj.contains_key("type") {
56 needs_type = true;
57 }
58 if !obj.contains_key("properties") {
59 needs_properties = true;
60 }
61 }
62 }
63
64 if (needs_type || needs_properties)
65 && let Some(obj) = parameters.as_object_mut()
66 {
67 if needs_type {
68 obj.insert(
69 "type".to_string(),
70 serde_json::Value::String("object".to_string()),
71 );
72 }
73 if needs_properties {
74 obj.insert(
75 "properties".to_string(),
76 serde_json::Value::Object(Default::default()),
77 );
78 }
79 }
80 }
81 }
82 updated_tools.push(tool);
83 }
84 }
85 Some(crate::python::template_value(&serde_json::Value::Array(
86 updated_tools,
87 )))
88}
89
90const DEFAULT_MEDIA_TYPE_CONVERSIONS: &[(&str, &str)] = &[
93 ("image_url", "image"),
94 ("video_url", "video"),
95 ("audio_url", "audio"),
96];
97
98fn custom_marker(part: &serde_json::Value) -> serde_json::Value {
100 json!({"type": "custom", "kind": part.get("kind")})
101}
102
103fn convert_media_url_to_placeholder(
105 content_array: &mut [serde_json::Value],
106 conversions: &[(&str, &str)],
107) {
108 for part in content_array {
109 let part_type = part.get("type").and_then(|t| t.as_str()).unwrap_or("");
110 if part_type == "custom" {
111 *part = custom_marker(part);
112 } else if let Some((_, target_type)) = conversions.iter().find(|(src, _)| *src == part_type)
113 {
114 *part = serde_json::json!({"type": target_type});
115 }
116 }
117}
118
119fn may_be_fix_msg_content(
120 mut messages: serde_json::Value,
121 preserve_arrays: bool,
122 image_placeholder_template: Option<&str>,
123) -> serde_json::Value {
124 let Some(arr) = messages.as_array_mut() else {
125 return messages;
126 };
127 for msg in arr {
128 let Some(content) = msg.get_mut("content") else {
129 continue;
130 };
131 match content {
132 serde_json::Value::String(_) if preserve_arrays => {
133 let text = content.take();
134 *content = serde_json::Value::Array(vec![serde_json::Value::Object(
135 serde_json::Map::from_iter([
136 ("type".into(), serde_json::Value::String("text".into())),
137 ("text".into(), text),
138 ]),
139 )]);
140 }
141 serde_json::Value::Array(parts) => {
142 convert_media_url_to_placeholder(parts, DEFAULT_MEDIA_TYPE_CONVERSIONS);
143 let text_only = !parts.is_empty()
146 && parts
147 .iter()
148 .all(|part| part.get("type").and_then(|v| v.as_str()) == Some("text"));
149 if text_only && !preserve_arrays {
150 let text = parts
151 .iter()
152 .filter_map(|part| part.get("text")?.as_str())
153 .collect::<Vec<_>>()
154 .join("\n");
155 *content = serde_json::Value::String(text);
156 } else if !preserve_arrays
157 && !parts.iter().any(|part| part.get("type").and_then(|v| v.as_str()) == Some("custom"))
160 && !parts.is_empty()
161 && let Some(template) = image_placeholder_template
162 {
163 *content = serde_json::Value::String(flatten_mixed_content(parts, template));
164 }
165 }
166 _ => {}
167 }
168 }
169 messages
170}
171
172fn flatten_mixed_content(parts: &[serde_json::Value], placeholder_tpl: &str) -> String {
195 let mut out = String::new();
196 let mut img_idx: u32 = 1;
197 for part in parts {
198 let type_str = part.get("type").and_then(|t| t.as_str()).unwrap_or("");
199 if type_str == "text" {
200 if let Some(text) = part.get("text").and_then(|t| t.as_str()) {
201 out.push_str(text);
202 }
203 } else if !type_str.is_empty() {
204 let placeholder = placeholder_tpl.replace("{n}", &img_idx.to_string());
205 out.push_str(&placeholder);
206 img_idx += 1;
207 }
208 }
209 out
210}
211
212enum ContentPart<'a> {
214 Text(Option<&'a str>),
216 Media(&'static str),
218 Custom,
220 Other,
221}
222
223fn content_part(part: &serde_json::Value) -> ContentPart<'_> {
225 let field = |name: &str| part.get(name).and_then(serde_json::Value::as_str);
226 match part.get("type").and_then(serde_json::Value::as_str) {
227 Some("text") => ContentPart::Text(field("text")),
228 Some("refusal") => ContentPart::Text(field("refusal")),
229 Some("image_url") => ContentPart::Media("image"),
230 Some("video_url") => ContentPart::Media("video"),
231 Some("audio_url" | "input_audio") => ContentPart::Media("audio"),
232 Some("custom") => ContentPart::Custom,
233 _ => ContentPart::Other,
234 }
235}
236
237fn flatten_msg_content_to_string(
241 messages: &mut serde_json::Value,
242 media_placeholders: &MediaPlaceholders,
243 use_audio_in_video: bool,
244) -> Result<()> {
245 let Some(msgs) = messages.as_array_mut() else {
246 return Ok(());
247 };
248 let mut media_counts = HashMap::new();
249 for msg in msgs {
250 let Some(msg) = msg.as_object_mut() else {
251 continue;
252 };
253 let content = match msg.get("content") {
254 Some(serde_json::Value::String(_)) => continue,
255 Some(serde_json::Value::Array(parts)) => flatten_parts_to_string(
256 parts,
257 media_placeholders,
258 use_audio_in_video,
259 &mut media_counts,
260 )?,
261 _ => String::new(),
263 };
264 msg.insert("content".to_string(), serde_json::Value::String(content));
265 }
266 Ok(())
267}
268
269fn flatten_parts_to_string(
274 parts: &[serde_json::Value],
275 media_placeholders: &MediaPlaceholders,
276 use_audio_in_video: bool,
277 media_counts: &mut HashMap<&'static str, usize>,
278) -> Result<String> {
279 let mut texts = Vec::new();
280 let mut by_modality: Vec<(&str, Vec<String>)> = Vec::new();
281 for part in parts {
282 match content_part(part) {
283 ContentPart::Text(text) => texts.extend(text.filter(|text| !text.is_empty())),
284 ContentPart::Media(modality) => {
285 let audio_track = modality == "video"
287 && use_audio_in_video
288 && part["video_url"]["url"]
289 .as_str()
290 .is_some_and(|url| !url.is_empty());
291 for modality in std::iter::once(modality).chain(audio_track.then_some("audio")) {
292 let n = media_counts.entry(modality).or_default();
293 *n += 1;
294 let template = match modality {
295 "image" => &media_placeholders.image,
296 "video" => &media_placeholders.video,
297 _ => &media_placeholders.audio,
298 };
299 if template.is_empty() {
300 continue;
301 }
302 let placeholder = template.replace("{n}", &n.to_string());
303 match by_modality.iter_mut().find(|(m, _)| *m == modality) {
304 Some((_, placeholders)) => placeholders.push(placeholder),
305 None => by_modality.push((modality, vec![placeholder])),
306 }
307 }
308 }
309 ContentPart::Custom | ContentPart::Other => {
310 return Err(PromptRenderError::invalid_request(format!(
311 "content part type {} is not supported with the `string` chat template content format",
312 part["type"]
313 ))
314 .into());
315 }
316 }
317 }
318 let text = texts.join("\n");
319
320 let mut counts: Vec<(String, usize)> = Vec::new();
322 for placeholder in by_modality.into_iter().flat_map(|(_, p)| p) {
323 match counts.iter_mut().find(|(p, _)| *p == placeholder) {
324 Some((_, count)) => *count += 1,
325 None => counts.push((placeholder, 1)),
326 }
327 }
328 let mut lines = Vec::new();
329 for (placeholder, count) in counts {
330 let present = text.matches(placeholder.as_str()).count();
331 if present > count {
332 return Err(PromptRenderError::invalid_request(format!(
333 "found more '{placeholder}' placeholders in the message text than media items"
334 ))
335 .into());
336 }
337 lines.extend(std::iter::repeat_n(placeholder, count - present));
338 }
339 if lines.is_empty() {
340 return Ok(text);
341 }
342 if !text.is_empty() {
343 lines.push(text);
344 }
345 Ok(lines.join("\n"))
346}
347
348const KNOWN_CONTENT_PART_FIELDS: &[&str] = &[
352 "type",
353 "text",
354 "refusal",
355 "image_url",
356 "video_url",
357 "audio_url",
358 "input_audio",
359 "uuid",
360 "audio_embeds",
361 "closed",
362 "data",
363 "file",
364 "image_embeds",
365 "image_pil",
366 "name",
367 "prompt_cache_breakpoint",
368 "thinking",
369 "video_embeds",
370];
371
372fn openai_part(part: serde_json::Value) -> Option<serde_json::Value> {
375 let mut wrapped = match content_part(&part) {
376 ContentPart::Text(text) => json!({"type": "text", "text": text?}),
377 ContentPart::Media(modality) => json!({"type": modality}),
378 ContentPart::Custom => return Some(custom_marker(&part)),
379 ContentPart::Other => return Some(part),
381 };
382 if let serde_json::Value::Object(fields) = part {
383 for (key, value) in fields {
384 if !KNOWN_CONTENT_PART_FIELDS.contains(&key.as_str()) {
385 wrapped[key.as_str()] = value;
386 }
387 }
388 }
389 Some(wrapped)
390}
391
392fn wrap_msg_content_in_parts(messages: &mut serde_json::Value) {
396 let Some(msgs) = messages.as_array_mut() else {
397 return;
398 };
399 for msg in msgs {
400 let is_tool = msg.get("role").and_then(serde_json::Value::as_str) == Some("tool");
401 let parts: Vec<serde_json::Value> = match take_message_content(msg) {
402 serde_json::Value::String(text) => {
403 let mut part = json!({"type": "text"});
404 part["text"] = text.into();
405 vec![part]
406 }
407 serde_json::Value::Array(parts) => parts.into_iter().filter_map(openai_part).collect(),
408 _ => Vec::new(),
409 };
410 let Some(msg) = msg.as_object_mut() else {
411 continue;
412 };
413 let all_text = parts.iter().all(|part| part["type"] == "text");
414 let content = if is_tool && all_text {
415 let texts: Vec<&str> = parts.iter().filter_map(|p| p["text"].as_str()).collect();
416 serde_json::Value::String(texts.join("\n"))
417 } else {
418 serde_json::Value::Array(parts)
419 };
420 msg.insert("content".to_string(), content);
421 }
422}
423
424fn normalize_tool_calls_arguments_in_messages(messages: &mut serde_json::Value) {
425 let Some(msgs) = messages.as_array_mut() else {
430 return;
431 };
432
433 for msg in msgs.iter_mut() {
434 if let Some(tool_calls) = msg.get_mut("tool_calls").and_then(|v| v.as_array_mut()) {
435 for tc in tool_calls {
436 if let Some(function) = tc.get_mut("function").and_then(|v| v.as_object_mut())
437 && let Some(args) = function.get_mut("arguments")
438 && let Some(s) = args.as_str()
439 && let Ok(parsed) = serde_json::from_str(s)
440 {
441 *args = parsed;
442 }
443 }
444 }
445 }
446}
447
448fn normalize_function_call_arguments_in_messages(messages: &mut serde_json::Value) {
449 let Some(msgs) = messages.as_array_mut() else {
454 return;
455 };
456
457 for msg in msgs.iter_mut() {
458 if let Some(function_call) = msg.get_mut("function_call").and_then(|v| v.as_object_mut())
459 && let Some(args) = function_call.get_mut("arguments")
460 && let Some(s) = args.as_str()
461 && let Ok(parsed) = serde_json::from_str(s)
462 {
463 *args = parsed;
464 }
465 }
466}
467
468fn inject_reasoning_content_into_messages(messages: &mut serde_json::Value) {
482 let Some(msgs) = messages.as_array_mut() else {
483 return;
484 };
485
486 for msg in msgs.iter_mut() {
487 if msg.get("role").and_then(|r| r.as_str()) != Some("assistant") {
488 continue;
489 }
490
491 let reasoning = match msg.get("reasoning_content") {
492 Some(serde_json::Value::String(s)) if !s.is_empty() => {
493 format!("<think>{}</think>", s)
494 }
495 Some(serde_json::Value::Array(segments)) => {
496 let mut result = String::new();
497 for seg in segments {
498 if let Some(s) = seg.as_str()
499 && !s.is_empty()
500 {
501 result.push_str("<think>");
502 result.push_str(s);
503 result.push_str("</think>");
504 }
505 }
506 if result.is_empty() {
507 continue;
508 }
509 result
510 }
511 _ => continue,
512 };
513
514 match msg.get("content") {
515 Some(serde_json::Value::String(s)) if !s.is_empty() => {
517 msg["content"] = serde_json::Value::String(format!("{}{}", reasoning, s));
518 }
519 None | Some(serde_json::Value::Null) | Some(serde_json::Value::String(_)) => {
520 msg["content"] = serde_json::Value::String(reasoning);
521 }
522 Some(serde_json::Value::Array(_)) => {
524 let think_part = serde_json::json!({
525 "type": "text",
526 "text": reasoning
527 });
528 if let Some(arr) = msg.get_mut("content").and_then(|v| v.as_array_mut()) {
529 arr.insert(0, think_part);
530 }
531 }
532 _ => continue,
534 }
535
536 if let Some(obj) = msg.as_object_mut() {
539 obj.remove("reasoning_content");
540 }
541 }
542}
543
544fn join_reasoning_content_segments_in_messages(messages: &mut serde_json::Value) {
550 let Some(msgs) = messages.as_array_mut() else {
551 return;
552 };
553
554 for msg in msgs.iter_mut() {
555 if msg.get("role").and_then(|r| r.as_str()) != Some("assistant") {
556 continue;
557 }
558 if let Some(reasoning) = msg.get_mut("reasoning_content")
559 && let Some(segments) = reasoning.as_array()
560 {
561 let joined = segments
562 .iter()
563 .filter_map(|s| s.as_str())
564 .filter(|s| !s.is_empty())
565 .collect::<Vec<_>>()
566 .join("\n");
567 *reasoning = joined.into();
568 }
569 }
570}
571
572impl OAIChatLikeRequest for dynamo_protocols::types::CreateChatCompletionRequest {
578 fn model(&self) -> String {
579 self.model.clone()
580 }
581
582 fn messages(&self) -> Value {
583 let messages_json = self.function_template_messages();
584 Value::from_serialize(&messages_json)
585 }
586
587 fn typed_messages(&self) -> Option<&[dynamo_protocols::types::ChatCompletionRequestMessage]> {
588 Some(self.messages.as_slice())
589 }
590
591 fn tools(&self) -> Option<Value> {
592 let tools = self.tools.as_ref()?;
593 may_be_fix_tool_schema(
594 serde_json::to_value(
595 tools
596 .iter()
597 .map(|tool| tool.function_template_tool())
598 .collect::<Vec<_>>(),
599 )
600 .unwrap(),
601 )
602 }
603
604 fn tool_choice(&self) -> Option<Value> {
605 self.tool_choice
606 .as_ref()
607 .map(|choice| Value::from_serialize(choice.function_template_choice()))
608 }
609
610 fn response_format(&self) -> Option<Value> {
611 self.response_format.as_ref().map(Value::from_serialize)
612 }
613
614 fn reasoning_effort(&self) -> Option<Value> {
615 self.reasoning_effort.as_ref().map(Value::from_serialize)
616 }
617
618 fn should_add_generation_prompt(&self) -> bool {
619 true
621 }
622
623 fn extract_text(&self) -> Option<TextInput> {
624 Some(TextInput::Single(String::new()))
625 }
626
627 fn mm_processor_kwargs(&self) -> Option<&serde_json::Value> {
628 self.mm_processor_kwargs.as_ref()
629 }
630}
631
632fn merge_message_content(
636 target: serde_json::Value,
637 source: serde_json::Value,
638) -> serde_json::Value {
639 use serde_json::Value;
640 let text_part = |text: String| json!({"type": "text", "text": text});
641 match (target, source) {
642 (Value::String(mut target), Value::String(source)) => {
643 if !target.is_empty() && !source.is_empty() {
644 target.push_str("\n\n");
645 }
646 target.push_str(&source);
647 Value::String(target)
648 }
649 (Value::Array(mut target), Value::Array(source)) => {
650 target.extend(source);
651 Value::Array(target)
652 }
653 (Value::Array(mut target), Value::String(source)) => {
654 if !source.is_empty() {
655 target.push(text_part(source));
656 }
657 Value::Array(target)
658 }
659 (Value::String(target), Value::Array(source)) => {
660 let mut parts = Vec::with_capacity(source.len() + 1);
661 if !target.is_empty() {
662 parts.push(text_part(target));
663 }
664 parts.extend(source);
665 Value::Array(parts)
666 }
667 (Value::Null, source) => source,
668 (target, _) => target,
670 }
671}
672
673fn append_message_content(target: &mut serde_json::Value, source: serde_json::Value) {
675 let Some(target) = target.as_object_mut() else {
676 return;
677 };
678 let merged = merge_message_content(
679 target.remove("content").unwrap_or(serde_json::Value::Null),
680 source,
681 );
682 target.insert("content".to_string(), merged);
683}
684
685fn take_message_content(message: &mut serde_json::Value) -> serde_json::Value {
686 message
687 .get_mut("content")
688 .map(serde_json::Value::take)
689 .unwrap_or(serde_json::Value::Null)
690}
691
692fn normalize_system_messages(messages: &mut serde_json::Value, rules: SystemNormalization) {
696 let serde_json::Value::Array(list) = messages else {
697 return;
698 };
699 let role_is =
700 |m: &serde_json::Value, r: &str| m.get("role").and_then(|v| v.as_str()) == Some(r);
701
702 if rules.demote_nonleading_system {
703 let leading = list.iter().take_while(|m| role_is(m, "system")).count();
706 if leading > 1 {
707 for mut trailing in list.drain(1..leading).collect::<Vec<_>>() {
708 let content = take_message_content(&mut trailing);
709 append_message_content(&mut list[0], content);
710 }
711 }
712
713 let leading = list.iter().take_while(|m| role_is(m, "system")).count();
716 for m in list.iter_mut().skip(leading) {
717 if role_is(m, "system")
718 && let Some(m) = m.as_object_mut()
719 {
720 m.insert("role".to_string(), json!("user"));
721 }
722 }
723 }
724
725 if rules.coalesce_consecutive_users {
726 let mut coalesced: Vec<serde_json::Value> = Vec::with_capacity(list.len());
727 for mut m in list.drain(..) {
728 if role_is(&m, "user") && coalesced.last().is_some_and(|p| role_is(p, "user")) {
729 let content = take_message_content(&mut m);
730 append_message_content(coalesced.last_mut().unwrap(), content);
731 } else {
732 coalesced.push(m);
733 }
734 }
735 *list = coalesced;
736 }
737}
738
739impl HfTokenizerConfigJsonFormatter {
740 fn template_handles_native_tools(&self, has_tools: bool) -> bool {
741 if has_tools {
742 self.tool_use_template_handles_native_tools
743 } else {
744 self.default_template_handles_native_tools
745 }
746 }
747
748 fn render_with_token_policy(
749 &self,
750 req: &dyn OAIChatLikeRequest,
751 protect_literals: bool,
752 ) -> Result<crate::RenderedPrompt> {
753 let mixins = Value::from_dyn_object(self.mixins.clone());
754
755 let exclude = self.exclude_tools_when_tool_choice_none
756 && req.tool_choice().as_ref().and_then(Value::as_str) == Some("none");
757 let flat_tools = if exclude { None } else { req.tools() };
758 let has_tools = flat_tools
760 .as_ref()
761 .and_then(|tools| tools.len())
762 .is_some_and(|len| len > 0);
763 let tools = if !exclude && self.template_handles_native_tools(has_tools) {
767 req.template_tools().or(flat_tools)
768 } else {
769 flat_tools
770 };
771 let add_generation_prompt = req.should_add_generation_prompt();
772
773 tracing::trace!(
774 "Rendering prompt with tools: {:?}, add_generation_prompt: {}",
775 has_tools,
776 add_generation_prompt
777 );
778
779 let (
782 template_name,
783 template_handles_tool_calls_args_string,
784 template_handles_reasoning,
785 template_requires_reasoning_string,
786 system_normalization,
787 ) = if has_tools {
788 (
789 "tool_use",
790 self.tool_use_template_handles_tool_calls_arguments_string,
791 self.tool_use_template_handles_reasoning,
792 self.tool_use_template_requires_reasoning_string,
793 self.tool_use_system_normalization,
794 )
795 } else {
796 (
797 "default",
798 self.default_template_handles_tool_calls_arguments_string,
799 self.default_template_handles_reasoning,
800 self.default_template_requires_reasoning_string,
801 self.default_system_normalization,
802 )
803 };
804
805 let mut messages_for_template = crate::messages_to_json(req)?;
806
807 crate::reject_unsupported_partial_assistant(&messages_for_template)?;
808 crate::reject_unsupported_message_tools(&messages_for_template, &[])?;
809
810 if system_normalization.is_required() {
811 normalize_system_messages(&mut messages_for_template, system_normalization);
812 }
813
814 match &self.content_format {
815 ChatTemplateContentFormat::Auto => {
816 messages_for_template = may_be_fix_msg_content(
817 messages_for_template,
818 self.requires_content_arrays,
819 self.image_placeholder_template,
820 );
821 }
822 ChatTemplateContentFormat::String(media_placeholders) => {
823 let use_audio_in_video = req
824 .mm_processor_kwargs()
825 .and_then(|kwargs| kwargs.get("use_audio_in_video"))
826 .is_some_and(|flag| Value::from_serialize(flag).is_true());
828 flatten_msg_content_to_string(
829 &mut messages_for_template,
830 media_placeholders,
831 use_audio_in_video,
832 )?;
833 }
834 ChatTemplateContentFormat::OpenAi => {
835 wrap_msg_content_in_parts(&mut messages_for_template);
836 }
837 }
838
839 if !template_handles_tool_calls_args_string {
846 normalize_tool_calls_arguments_in_messages(&mut messages_for_template);
847 }
848 normalize_function_call_arguments_in_messages(&mut messages_for_template);
852
853 if !template_handles_reasoning {
858 inject_reasoning_content_into_messages(&mut messages_for_template);
859 } else if template_requires_reasoning_string {
860 join_reasoning_content_segments_in_messages(&mut messages_for_template);
861 }
862
863 let tmpl = self.env.get_template(template_name)?;
864 let mut literals = HarmonyLiterals::default();
865 let mut tools = tools;
866 let mut response_format = req
867 .response_format()
868 .as_ref()
869 .map(crate::python::template_json)
870 .transpose()?;
871 let mut placeholder = "__DYNAMO_HARMONY_LITERAL_".to_string();
872 if protect_literals
873 && tmpl.source().contains("<|start|>")
874 && tmpl.source().contains("<|message|>")
875 {
876 let mut tool_json = tools
879 .as_ref()
880 .map(crate::python::template_json)
881 .transpose()?;
882 let collision_source = format!(
883 "{messages_for_template}{tool_json:?}{response_format:?}{:?}{}",
884 req.chat_template_args(),
885 tmpl.source()
886 );
887 while collision_source.contains(&placeholder) {
888 placeholder.push('_');
889 }
890 shield_harmony_literals(&mut messages_for_template, &placeholder, &mut literals);
891 if let Some(value) = &mut tool_json {
892 shield_harmony_literals(value, &placeholder, &mut literals);
893 tools = Some(crate::python::template_value(value));
894 }
895 if let Some(value) = &mut response_format {
896 shield_harmony_literals(value, &placeholder, &mut literals);
897 }
898 }
899
900 let ctx = context! {
901 messages => crate::python::template_value(&messages_for_template),
902 tools => tools.as_ref().map(crate::python::template_json).transpose()?.as_ref().map(crate::python::template_value),
903 response_format => response_format.as_ref()
904 .map(crate::python::template_value)
905 .unwrap_or(Value::UNDEFINED),
906 bos_token => self.config.bos_tok(),
907 eos_token => self.config.eos_tok(),
908 unk_token => self.config.unk_tok(),
909 add_generation_prompt => add_generation_prompt,
910 ..mixins
911 };
912
913 let ctx = if let Some(args) = req.chat_template_args() {
915 let extra = crate::python::template_value(&serde_json::to_value(args)?);
916 context! { ..ctx, ..extra }
917 } else {
918 ctx
919 };
920
921 let rendered = tmpl.render(&ctx)?;
922 if literals.values.is_empty() {
923 return Ok(crate::RenderedPrompt::text(rendered));
924 }
925 let mut segments = Vec::new();
926 let mut remaining = rendered.as_str();
927 while let Some(start) = remaining.find(&placeholder) {
928 segments.push(crate::RenderedSegment::new(&remaining[..start], true));
929 let after = &remaining[start + placeholder.len()..];
930 let (index, suffix) = after
931 .split_once('_')
932 .ok_or_else(|| anyhow::anyhow!("damaged Harmony literal placeholder"))?;
933 let index: usize = index.parse()?;
934 segments.push(crate::RenderedSegment::new(
935 literals
936 .values
937 .get(index)
938 .ok_or_else(|| anyhow::anyhow!("unknown Harmony literal placeholder"))?,
939 false,
940 ));
941 remaining = suffix;
942 }
943 segments.push(crate::RenderedSegment::new(remaining, true));
944 Ok(crate::RenderedPrompt::segmented(segments))
945 }
946}
947
948#[derive(Default)]
951struct HarmonyLiterals {
952 values: Vec<String>,
953 indices: HashMap<String, usize>,
954}
955
956impl HarmonyLiterals {
957 fn index(&mut self, text: &str) -> usize {
958 if let Some(index) = self.indices.get(text) {
959 return *index;
960 }
961 let index = self.values.len();
962 self.values.push(text.to_owned());
963 self.indices.insert(text.to_owned(), index);
964 index
965 }
966}
967
968fn shield_harmony_literals(
969 value: &mut serde_json::Value,
970 prefix: &str,
971 literals: &mut HarmonyLiterals,
972) {
973 match value {
974 serde_json::Value::String(text) => shield_harmony_text(text, prefix, literals),
975 serde_json::Value::Array(values) => {
976 for value in values {
977 shield_harmony_literals(value, prefix, literals);
978 }
979 }
980 serde_json::Value::Object(values) => {
981 if values.keys().any(|key| key.contains("<|")) {
982 for (mut key, mut value) in std::mem::take(values) {
983 shield_harmony_text(&mut key, prefix, literals);
984 shield_harmony_literals(&mut value, prefix, literals);
985 values.insert(key, value);
986 }
987 } else {
988 for value in values.values_mut() {
989 shield_harmony_literals(value, prefix, literals);
990 }
991 }
992 }
993 _ => {}
994 }
995}
996
997fn shield_harmony_text(text: &mut String, prefix: &str, literals: &mut HarmonyLiterals) {
998 let mut protected = String::new();
999 let mut remaining = text.as_str();
1000 while let Some(start) = remaining.find("<|") {
1001 let Some(end) = remaining[start + 2..].find("|>") else {
1002 break;
1003 };
1004 let end = start + 2 + end + 2;
1005 let marker = &remaining[start..end];
1006 if marker
1008 .chars()
1009 .any(|c| c == '"' || c == '\\' || c <= '\u{1f}')
1010 {
1011 protected.push_str(&remaining[..start + 2]);
1012 remaining = &remaining[start + 2..];
1013 continue;
1014 }
1015 protected.push_str(&remaining[..start]);
1016 protected.push_str(&format!("{prefix}{}_", literals.index(marker)));
1017 remaining = &remaining[end..];
1018 }
1019 if remaining.len() != text.len() {
1020 protected.push_str(remaining);
1021 *text = protected;
1022 }
1023}
1024
1025impl OAIPromptFormatter for HfTokenizerConfigJsonFormatter {
1026 fn supports_native_tools(&self, request: &dyn OAIChatLikeRequest) -> bool {
1027 let exclude = self.exclude_tools_when_tool_choice_none
1029 && request.tool_choice().as_ref().and_then(Value::as_str) == Some("none");
1030 let has_tools = !exclude
1031 && request
1032 .tools()
1033 .as_ref()
1034 .and_then(|tools| tools.len())
1035 .is_some_and(|len| len > 0);
1036 self.template_handles_native_tools(has_tools)
1037 }
1038
1039 fn supports_add_generation_prompt(&self) -> bool {
1040 self.supports_add_generation_prompt
1041 }
1042
1043 fn render(&self, req: &dyn OAIChatLikeRequest) -> Result<String> {
1044 Ok(self
1045 .render_with_token_policy(req, false)?
1046 .as_str()
1047 .to_string())
1048 }
1049
1050 fn render_prompt(&self, req: &dyn OAIChatLikeRequest) -> Result<crate::RenderedPrompt> {
1051 self.render_with_token_policy(req, true)
1052 }
1053}
1054
1055#[cfg(test)]
1056mod tests {
1057 use super::*;
1058
1059 use dynamo_protocols::types::ChatCompletionRequestMessage as Msg;
1060 use dynamo_protocols::types::CreateChatCompletionRequest as NvCreateChatCompletionRequest;
1063 use minijinja::{Environment, context};
1064
1065 use super::super::tokcfg::ChatTemplate as SysChatTemplate;
1068 use super::super::{
1069 ContextMixins as SysMixins, HfTokenizerConfigJsonFormatter as SysFormatter,
1070 };
1071
1072 fn formatter_for(template: &str) -> SysFormatter {
1073 let ct: SysChatTemplate = serde_json::from_value(json!({
1075 "chat_template": template,
1076 "bos_token": "<s>",
1077 "eos_token": "</s>",
1078 "unk_token": "<unk>",
1079 }))
1080 .unwrap();
1081 SysFormatter::new(ct, SysMixins::new(&[])).unwrap()
1082 }
1083
1084 #[test]
1085 fn harmony_literal_markers_keep_text_and_token_trust_boundaries() {
1086 let formatter = formatter_for(
1087 r#"{% for m in messages %}<|start|>{{ m.role }}<|message|>{{ m.content }}<|end|>{% endfor %}<|start|>assistant"#,
1088 );
1089 let request: NvCreateChatCompletionRequest = serde_json::from_value(json!({
1090 "model":"gpt-oss", "messages":[{"role":"user","content":"literal <|channel|>analysis<|message|>x<|call|> __DYNAMO_HARMONY_LITERAL_0_"}]
1091 })).unwrap();
1092 let prompt = formatter.render_prompt(&request).unwrap();
1093 assert_eq!(prompt.as_str(), formatter.render(&request).unwrap());
1094 let ordinary: Vec<_> = prompt
1095 .segments()
1096 .unwrap()
1097 .iter()
1098 .filter(|segment| !segment.allow_special)
1099 .map(|segment| segment.text.as_str())
1100 .collect();
1101 assert_eq!(ordinary, ["<|channel|>", "<|message|>", "<|call|>"]);
1102 assert!(prompt.segments().unwrap()[0].allow_special);
1103 assert!(prompt.segments().unwrap().last().unwrap().allow_special);
1104 }
1105
1106 fn formatter_for_templates(default: &str, tool_use: &str) -> SysFormatter {
1107 let ct: SysChatTemplate = serde_json::from_value(json!({
1108 "chat_template": [
1109 {"default": default},
1110 {"tool_use": tool_use},
1111 ],
1112 "bos_token": "<s>",
1113 "eos_token": "</s>",
1114 "unk_token": "<unk>",
1115 }))
1116 .unwrap();
1117 SysFormatter::new(ct, SysMixins::new(&[])).unwrap()
1118 }
1119
1120 fn try_formatter_for(template: &str) -> Option<SysFormatter> {
1121 let ct: SysChatTemplate = serde_json::from_value(json!({
1122 "chat_template": template,
1123 "bos_token": "<s>",
1124 "eos_token": "</s>",
1125 "unk_token": "<unk>",
1126 }))
1127 .ok()?;
1128 SysFormatter::new(ct, SysMixins::new(&[])).ok()
1129 }
1130
1131 fn render_shape(f: &SysFormatter, messages: serde_json::Value) -> Result<String> {
1132 let req: NvCreateChatCompletionRequest =
1133 serde_json::from_value(json!({ "model": "test", "messages": messages })).unwrap();
1134 f.render(&req)
1135 }
1136
1137 fn render_shape_with_tools(f: &SysFormatter, messages: serde_json::Value) -> Result<String> {
1138 let req: NvCreateChatCompletionRequest = serde_json::from_value(json!({
1139 "model": "test",
1140 "messages": messages,
1141 "tools": [{
1142 "type": "function",
1143 "function": {"name": "noop", "parameters": {}}
1144 }]
1145 }))
1146 .unwrap();
1147 f.render(&req)
1148 }
1149
1150 struct RawMessagesRequest(Value);
1151
1152 impl OAIChatLikeRequest for RawMessagesRequest {
1153 fn model(&self) -> String {
1154 "test".to_string()
1155 }
1156
1157 fn messages(&self) -> Value {
1158 self.0.clone()
1159 }
1160
1161 fn should_add_generation_prompt(&self) -> bool {
1162 true
1163 }
1164 }
1165
1166 fn render_raw_shape(f: &SysFormatter, messages: serde_json::Value) -> Result<String> {
1167 f.render(&RawMessagesRequest(Value::from_serialize(&messages)))
1168 }
1169
1170 #[test]
1171 fn content_normalization_preserves_unchanged_values() {
1172 let messages = json!([
1173 {"role": "user", "content": " 中文 <special>\n", "name": "user"},
1174 {"role": "assistant", "content": null},
1175 {"role": "user", "content": []},
1176 {"role": "assistant", "tool_calls": []}
1177 ]);
1178 assert_eq!(
1179 may_be_fix_msg_content(messages.clone(), false, Some("")),
1180 messages
1181 );
1182 }
1183
1184 const PERMISSIVE_TMPL: &str = concat!(
1185 "{%- for m in messages -%}",
1186 "<|im_start|>{{ m.role }}\n{{ m.content }}<|im_end|>\n",
1187 "{%- endfor -%}"
1188 );
1189
1190 #[test]
1191 fn jinja_templates_reject_message_level_tools() {
1192 let f = formatter_for(PERMISSIVE_TMPL);
1193 let error = render_shape(
1194 &f,
1195 json!([
1196 {"role": "system", "tools": [{"name": "lookup", "parameters": {"type": "object"}}]},
1197 {"role": "user", "content": "hi"}
1198 ]),
1199 )
1200 .unwrap_err();
1201 assert!(matches!(
1202 error.downcast_ref::<crate::PromptRenderError>(),
1203 Some(crate::PromptRenderError::InvalidRequest(message))
1204 if message.contains("message-level `tools`")
1205 ));
1206
1207 let error = render_raw_shape(
1208 &f,
1209 json!([{
1210 "role": "user",
1211 "content": "hi",
1212 "tools": [{"name": "lookup", "parameters": {"type": "object"}}]
1213 }]),
1214 )
1215 .unwrap_err();
1216 assert!(matches!(
1217 error.downcast_ref::<crate::PromptRenderError>(),
1218 Some(crate::PromptRenderError::InvalidRequest(message))
1219 if message.contains("message-level `tools`")
1220 ));
1221
1222 let rendered = render_shape(
1223 &f,
1224 json!([
1225 {"role": "system", "content": "You are helpful.", "tools": []},
1226 {"role": "user", "content": "hi"}
1227 ]),
1228 )
1229 .unwrap();
1230 assert!(rendered.contains("<|im_start|>system\nYou are helpful.<|im_end|>"));
1231 }
1232
1233 #[test]
1234 fn jinja_templates_reject_unsupported_partial_assistant() {
1235 let f = formatter_for(PERMISSIVE_TMPL);
1236 let error = render_shape(
1237 &f,
1238 json!([
1239 {"role": "user", "content": "Continue"},
1240 {"role": "assistant", "content": "prefix", "partial": true}
1241 ]),
1242 )
1243 .unwrap_err();
1244 assert!(matches!(
1245 error.downcast_ref::<crate::PromptRenderError>(),
1246 Some(crate::PromptRenderError::InvalidRequest(message))
1247 if message.contains("`partial: true` is not supported")
1248 ));
1249
1250 let rendered = render_shape(
1251 &f,
1252 json!([{"role": "assistant", "content": "ordinary", "partial": false}]),
1253 )
1254 .unwrap();
1255 assert!(rendered.contains("ordinary"));
1256 }
1257 const STRICT_LEADING_TMPL: &str = concat!(
1259 "{%- for m in messages -%}",
1260 "{%- if m.role == 'system' and not loop.first -%}",
1261 "{{ raise_exception('System message must be at the beginning.') }}",
1262 "{%- endif -%}",
1263 "<|im_start|>{{ m.role }}\n{{ m.content }}<|im_end|>\n",
1264 "{%- endfor -%}"
1265 );
1266 const ALTERNATION_TMPL: &str = concat!(
1268 "{%- set ns = namespace(prev='') -%}",
1269 "{%- for m in messages -%}",
1270 "{%- if m.role == 'user' and ns.prev == 'user' -%}",
1271 "{{ raise_exception('Conversation roles must alternate.') }}",
1272 "{%- endif -%}",
1273 "<|im_start|>{{ m.role }}\n{{ m.content }}<|im_end|>\n",
1274 "{%- set ns.prev = m.role -%}",
1275 "{%- endfor -%}"
1276 );
1277 const STRICT_BOTH_TMPL: &str = concat!(
1279 "{%- set ns = namespace(prev='') -%}",
1280 "{%- for m in messages -%}",
1281 "{%- if m.role == 'system' and not loop.first -%}",
1282 "{{ raise_exception('System message must be at the beginning.') }}",
1283 "{%- endif -%}",
1284 "{%- if m.role == 'user' and ns.prev == 'user' -%}",
1285 "{{ raise_exception('Conversation roles must alternate.') }}",
1286 "{%- endif -%}",
1287 "<|im_start|>{{ m.role }}\n{{ m.content }}<|im_end|>\n",
1288 "{%- set ns.prev = m.role -%}",
1289 "{%- endfor -%}"
1290 );
1291 const DEFAULT_NONE_GATED_TMPL: &str = concat!(
1294 "{%- set strict = tools is not none -%}",
1295 "{%- for m in messages -%}",
1296 "{%- if strict and m.role == 'system' and not loop.first -%}",
1297 "{{ raise_exception('System message must be at the beginning.') }}",
1298 "{%- endif -%}",
1299 "<|im_start|>{{ m.role }}\n{{ m.content }}<|im_end|>\n",
1300 "{%- endfor -%}"
1301 );
1302 const TOOL_NONEMPTY_GATED_TMPL: &str = concat!(
1303 "{%- set strict = tools|length > 0 -%}",
1304 "{%- for m in messages -%}",
1305 "{%- if strict and m.role == 'system' and not loop.first -%}",
1306 "{{ raise_exception('System message must be at the beginning.') }}",
1307 "{%- endif -%}",
1308 "<|im_start|>{{ m.role }}\n{{ m.content }}<|im_end|>\n",
1309 "{%- endfor -%}"
1310 );
1311 const STRICT_ARRAY_TMPL: &str = concat!(
1313 "{%- for m in messages -%}",
1314 "{%- if m.role == 'system' and not loop.first -%}",
1315 "{{ raise_exception('System message must be at the beginning.') }}",
1316 "{%- endif -%}",
1317 "<|im_start|>{{ m.role }}\n",
1318 "{%- if m.content is not string -%}",
1319 "{%- for part in m.content -%}{{ part.text }}{%- endfor -%}",
1320 "{%- endif -%}",
1321 "<|im_end|>\n",
1322 "{%- endfor -%}"
1323 );
1324
1325 fn claude_shape() -> serde_json::Value {
1327 json!([
1328 {"role": "system", "content": "You are Claude Code."},
1329 {"role": "user", "content": "hello"},
1330 {"role": "system", "content": "mid-conversation reminder"},
1331 ])
1332 }
1333
1334 fn all_restrictions() -> SystemNormalization {
1335 SystemNormalization {
1336 demote_nonleading_system: true,
1337 coalesce_consecutive_users: true,
1338 }
1339 }
1340
1341 #[test]
1342 fn permissive_template_is_not_flagged_and_renders_untouched() {
1343 let f = formatter_for(PERMISSIVE_TMPL);
1344 assert!(!f.default_system_normalization.is_required());
1345 assert!(!f.tool_use_system_normalization.is_required());
1346 let out = render_shape(&f, claude_shape()).unwrap();
1347 assert!(out.contains("<|im_start|>system\nmid-conversation reminder<|im_end|>"));
1348 }
1349
1350 #[test]
1351 fn strict_leading_template_demotes_mid_system_but_keeps_user_turns_apart() {
1352 let f = formatter_for(STRICT_LEADING_TMPL);
1353 assert!(f.default_system_normalization.demote_nonleading_system);
1354 assert!(!f.default_system_normalization.coalesce_consecutive_users);
1356
1357 let out = render_shape(&f, claude_shape()).unwrap();
1359 assert_eq!(out.matches("<|im_start|>system").count(), 1);
1360 assert!(out.contains("<|im_start|>user\nhello<|im_end|>"));
1361 assert!(out.contains("<|im_start|>user\nmid-conversation reminder<|im_end|>"));
1362 }
1363
1364 #[test]
1365 fn alternation_template_coalesces_users_but_keeps_mid_system() {
1366 let f = formatter_for(ALTERNATION_TMPL);
1367 assert!(f.default_system_normalization.coalesce_consecutive_users);
1368 assert!(!f.default_system_normalization.demote_nonleading_system);
1370
1371 let out = render_shape(&f, claude_shape()).unwrap();
1372 assert!(out.contains("<|im_start|>system\nmid-conversation reminder<|im_end|>"));
1373
1374 let out = render_shape(
1375 &f,
1376 json!([
1377 {"role": "system", "content": "s"},
1378 {"role": "user", "content": "hello"},
1379 {"role": "user", "content": "again"},
1380 ]),
1381 )
1382 .unwrap();
1383 assert_eq!(out.matches("<|im_start|>user").count(), 1);
1384 assert!(out.contains("<|im_start|>user\nhello\n\nagain<|im_end|>"));
1385 }
1386
1387 #[test]
1388 fn strict_both_template_demotes_then_coalesces() {
1389 let f = formatter_for(STRICT_BOTH_TMPL);
1390 assert!(f.default_system_normalization.demote_nonleading_system);
1391 assert!(f.default_system_normalization.coalesce_consecutive_users);
1392
1393 let out = render_shape(&f, claude_shape()).unwrap();
1394 assert_eq!(out.matches("<|im_start|>system").count(), 1);
1395 assert!(out.contains("<|im_start|>user\nhello\n\nmid-conversation reminder<|im_end|>"));
1396 }
1397
1398 #[test]
1399 fn system_normalization_flag_is_selected_per_template() {
1400 let f = formatter_for_templates(PERMISSIVE_TMPL, STRICT_LEADING_TMPL);
1401 assert!(!f.default_system_normalization.is_required());
1402 assert!(f.tool_use_system_normalization.is_required());
1403
1404 let no_tools = render_shape(&f, claude_shape()).unwrap();
1405 assert!(no_tools.contains("<|im_start|>system\nmid-conversation reminder<|im_end|>"));
1406 let with_tools = render_shape_with_tools(&f, claude_shape()).unwrap();
1407 assert_eq!(with_tools.matches("<|im_start|>system").count(), 1);
1408 assert!(with_tools.contains("<|im_start|>user\nmid-conversation reminder<|im_end|>"));
1409
1410 let f = formatter_for_templates(STRICT_LEADING_TMPL, PERMISSIVE_TMPL);
1411 assert!(f.default_system_normalization.is_required());
1412 assert!(!f.tool_use_system_normalization.is_required());
1413 let with_tools = render_shape_with_tools(&f, claude_shape()).unwrap();
1414 assert!(with_tools.contains("<|im_start|>system\nmid-conversation reminder<|im_end|>"));
1415 }
1416
1417 #[test]
1418 fn system_normalization_probe_uses_runtime_tools_shape() {
1419 let f = formatter_for_templates(DEFAULT_NONE_GATED_TMPL, TOOL_NONEMPTY_GATED_TMPL);
1420 assert!(!f.default_system_normalization.is_required());
1421 assert!(f.tool_use_system_normalization.is_required());
1422
1423 let no_tools = render_shape(&f, claude_shape()).unwrap();
1424 assert!(no_tools.contains("<|im_start|>system\nmid-conversation reminder<|im_end|>"));
1425
1426 let with_tools = render_shape_with_tools(&f, claude_shape()).unwrap();
1427 assert_eq!(with_tools.matches("<|im_start|>system").count(), 1);
1428 assert!(with_tools.contains("<|im_start|>user\nmid-conversation reminder<|im_end|>"));
1429 }
1430
1431 #[test]
1432 fn system_normalization_precedes_required_content_array_conversion() {
1433 let f = formatter_for(STRICT_ARRAY_TMPL);
1434 assert!(f.requires_content_arrays);
1435 assert!(f.default_system_normalization.demote_nonleading_system);
1436
1437 let out = render_shape(
1438 &f,
1439 json!([
1440 {"role": "system", "content": "A"},
1441 {"role": "system", "content": "B"},
1442 {"role": "user", "content": "hello"},
1443 ]),
1444 )
1445 .unwrap();
1446 assert!(out.contains("A\n\nB"));
1447 }
1448
1449 #[test]
1450 fn normalize_preserves_multimodal_user_content_and_fields() {
1451 let mut m = json!([
1452 {
1453 "role": "user",
1454 "name": "kept",
1455 "content": [
1456 {"type": "text", "text": "look"},
1457 {"type": "image"},
1458 ],
1459 },
1460 {"role": "system", "content": "remember"},
1461 ]);
1462 normalize_system_messages(&mut m, all_restrictions());
1463 assert_eq!(
1464 m,
1465 json!([{
1466 "role": "user",
1467 "name": "kept",
1468 "content": [
1469 {"type": "text", "text": "look"},
1470 {"type": "image"},
1471 {"type": "text", "text": "remember"},
1472 ],
1473 }])
1474 );
1475 }
1476
1477 #[test]
1480 fn coalesce_preserves_multimodal_content_of_the_merged_turn() {
1481 let mut m = json!([
1482 {"role": "user", "content": "look"},
1483 {"role": "user", "content": [
1484 {"type": "text", "text": "at this"},
1485 {"type": "image_url", "image_url": {"url": "http://img"}},
1486 ]},
1487 ]);
1488 normalize_system_messages(&mut m, all_restrictions());
1489 assert_eq!(
1490 m,
1491 json!([{
1492 "role": "user",
1493 "content": [
1494 {"type": "text", "text": "look"},
1495 {"type": "text", "text": "at this"},
1496 {"type": "image_url", "image_url": {"url": "http://img"}},
1497 ],
1498 }])
1499 );
1500 }
1501
1502 #[test]
1503 fn normalize_merges_leading_run_and_coalesces() {
1504 let mut m = json!([
1505 {"role": "system", "content": "A"},
1506 {"role": "system", "content": "B"},
1507 {"role": "user", "content": "hi"},
1508 {"role": "system", "content": "reminder"},
1509 ]);
1510 normalize_system_messages(&mut m, all_restrictions());
1511 assert_eq!(
1512 m,
1513 json!([
1514 {"role": "system", "content": "A\n\nB"},
1515 {"role": "user", "content": "hi\n\nreminder"},
1516 ])
1517 );
1518 }
1519
1520 #[test]
1523 fn each_restriction_applies_only_its_own_rewrite() {
1524 let shape = json!([
1525 {"role": "system", "content": "A"},
1526 {"role": "system", "content": "B"},
1527 {"role": "user", "content": "hi"},
1528 {"role": "system", "content": "reminder"},
1529 ]);
1530
1531 let mut demote_only = shape.clone();
1532 normalize_system_messages(
1533 &mut demote_only,
1534 SystemNormalization {
1535 demote_nonleading_system: true,
1536 coalesce_consecutive_users: false,
1537 },
1538 );
1539 assert_eq!(
1540 demote_only,
1541 json!([
1542 {"role": "system", "content": "A\n\nB"},
1543 {"role": "user", "content": "hi"},
1544 {"role": "user", "content": "reminder"},
1545 ])
1546 );
1547
1548 let mut coalesce_only = shape.clone();
1549 normalize_system_messages(
1550 &mut coalesce_only,
1551 SystemNormalization {
1552 demote_nonleading_system: false,
1553 coalesce_consecutive_users: true,
1554 },
1555 );
1556 assert_eq!(coalesce_only, shape);
1557 }
1558
1559 #[test]
1560 fn normalize_preserves_array_system_content() {
1561 let mut m = json!([
1562 {"role": "user", "content": "hi"},
1563 {"role": "system", "content": [{"type": "text", "text": "one"},
1564 {"type": "text", "text": "two"}]},
1565 ]);
1566 normalize_system_messages(&mut m, all_restrictions());
1567 assert_eq!(
1568 m,
1569 json!([{"role": "user", "content": [
1570 {"type": "text", "text": "hi"},
1571 {"type": "text", "text": "one"},
1572 {"type": "text", "text": "two"},
1573 ]}])
1574 );
1575 }
1576
1577 #[test]
1586 #[ignore]
1587 fn adaptive_system_corpus_audit() {
1588 let dir =
1589 std::env::var("TEMPLATE_CORPUS").expect("set TEMPLATE_CORPUS to the templates dir");
1590 let manifest: serde_json::Value =
1591 serde_json::from_str(&std::fs::read_to_string(format!("{dir}/manifest.json")).unwrap())
1592 .unwrap();
1593
1594 let sys = |c: &str| json!({"role": "system", "content": c});
1597 let usr = |c: &str| json!({"role": "user", "content": c});
1598 let asst = |c: &str| json!({"role": "assistant", "content": c});
1599 let shapes: Vec<(&str, serde_json::Value)> = vec![
1600 ("turn1", json!([sys("s"), usr("u"), sys("mid")])),
1601 (
1602 "multiturn",
1603 json!([sys("s"), usr("u"), sys("mid"), asst("a"), usr("u2")]),
1604 ),
1605 (
1606 "mid_after_asst",
1607 json!([sys("s"), usr("u"), asst("a"), sys("mid"), usr("u2")]),
1608 ),
1609 ("double_leading", json!([sys("s0"), sys("s1"), usr("u")])),
1610 ("consec_user", json!([sys("s"), usr("u0"), usr("u1")])),
1611 (
1612 "tail_reminder",
1613 json!([
1614 sys("s"),
1615 usr("u"),
1616 asst("a"),
1617 usr("u2"),
1618 sys("mid"),
1619 usr("u3")
1620 ]),
1621 ),
1622 ("leading_only_baseline", json!([sys("s"), usr("u")])),
1623 ];
1624
1625 let mut total = 0usize;
1626 let mut flagged = 0usize;
1627 let mut demote_only = 0usize;
1628 let mut coalesce = 0usize;
1629 let mut failures: Vec<String> = Vec::new();
1630 for (file, meta) in manifest.as_object().unwrap() {
1631 let tmpl = std::fs::read_to_string(format!("{dir}/{file}.jinja")).unwrap();
1632 let model = meta["model"].as_str().unwrap_or(file);
1633 let f = match try_formatter_for(&tmpl) {
1636 Some(f) => f,
1637 None => {
1638 eprintln!("[skip-compile] {model}");
1639 continue;
1640 }
1641 };
1642 if render_shape(&f, json!([sys("s"), usr("u")])).is_err() {
1645 eprintln!("[skip-baseline] {model}");
1646 continue;
1647 }
1648 total += 1;
1649 let rules = f.default_system_normalization;
1650 let flag = rules.is_required();
1651 if flag {
1652 flagged += 1;
1653 }
1654 if rules.demote_nonleading_system {
1655 demote_only += usize::from(!rules.coalesce_consecutive_users);
1656 }
1657 if rules.coalesce_consecutive_users {
1658 coalesce += 1;
1659 }
1660 for (name, shape) in &shapes {
1661 if render_shape(&f, shape.clone()).is_err() {
1662 failures.push(format!("{model} | shape={name} | flag={flag}"));
1663 }
1664 }
1665 if flag {
1666 eprintln!(
1667 "[ok] demote={} coalesce={} {model}",
1668 rules.demote_nonleading_system, rules.coalesce_consecutive_users
1669 );
1670 }
1671 }
1672 eprintln!(
1673 "\naudited {total} templates ({flagged} flagged: {demote_only} demote-only, \
1674 {coalesce} coalescing); {} shape failures",
1675 failures.len()
1676 );
1677 for f in &failures {
1678 eprintln!(" FAIL {f}");
1679 }
1680 assert!(
1681 failures.is_empty(),
1682 "{} template/shape combinations did not render (probe insufficient or normalization insufficient)",
1683 failures.len()
1684 );
1685 }
1686
1687 #[test]
1695 fn test_render_long_conversation_does_not_overflow_stack() {
1696 let handle = std::thread::Builder::new()
1697 .stack_size(2 * 1024 * 1024)
1698 .spawn(|| {
1699 let template_string = concat!(
1700 "{%- set ns = namespace(items=[]) -%}",
1701 "{%- for m in messages -%}",
1702 "{%- set ns.items = ns.items + [m] -%}",
1703 "{%- endfor -%}",
1704 "COUNT={{ ns.items | length }}"
1705 );
1706 let chat_template: ChatTemplate =
1707 serde_json::from_value(serde_json::json!({ "chat_template": template_string }))
1708 .unwrap();
1709 let formatter =
1710 HfTokenizerConfigJsonFormatter::new(chat_template, ContextMixins::new(&[]))
1711 .unwrap();
1712
1713 let n = 3000;
1714 let messages: Vec<serde_json::Value> = (0..n)
1715 .map(|i| serde_json::json!({"role": "user", "content": format!("turn {i}")}))
1716 .collect();
1717 let request: NvCreateChatCompletionRequest =
1718 serde_json::from_value(serde_json::json!({
1719 "model": "test",
1720 "messages": messages,
1721 }))
1722 .unwrap();
1723
1724 let rendered = formatter.render(&request).unwrap();
1726 assert_eq!(rendered.trim(), format!("COUNT={n}"));
1727 })
1728 .unwrap();
1729 handle.join().unwrap();
1730 }
1731
1732 #[test]
1743 #[ignore]
1744 fn dump_gptoss_tool_prompt() {
1745 use super::tokcfg::ChatTemplate;
1746 use super::{ContextMixins, HfTokenizerConfigJsonFormatter};
1747
1748 let path = std::env::var("GPTOSS_CHAT_TEMPLATE").expect(
1749 "set GPTOSS_CHAT_TEMPLATE to the tokenizer_config.json, chat_template.jinja, or model dir path",
1750 );
1751 let input_path = std::path::Path::new(&path);
1752 let file_path = if input_path.is_dir() {
1753 input_path.join("tokenizer_config.json")
1755 } else {
1756 input_path.to_path_buf()
1757 };
1758 let raw = std::fs::read_to_string(&file_path).expect("read chat template file");
1759 let template_string: String = match serde_json::from_str::<serde_json::Value>(&raw) {
1766 Ok(v) if v.get("chat_template").is_some() => v["chat_template"]
1767 .as_str()
1768 .expect("chat_template field must be a string")
1769 .to_string(),
1770 _ => {
1771 let sibling = std::path::Path::new(&path)
1772 .parent()
1773 .map(|d| d.join("chat_template.jinja"));
1774 match sibling {
1775 Some(p) if p.exists() => {
1776 eprintln!(
1777 "[info] {path} had no chat_template field; using {}",
1778 p.display()
1779 );
1780 std::fs::read_to_string(&p).expect("read sibling chat_template.jinja")
1781 }
1782 _ => raw,
1783 }
1784 }
1785 };
1786
1787 assert!(
1790 template_string.contains("{%") || template_string.contains("{{"),
1791 "resolved template has no Jinja tags — GPTOSS_CHAT_TEMPLATE ({path}) is probably \
1792 tokenizer_config.json with no chat_template field and no sibling chat_template.jinja. \
1793 Point it at the chat_template.jinja file."
1794 );
1795
1796 let chat_template: ChatTemplate =
1797 serde_json::from_value(serde_json::json!({ "chat_template": template_string }))
1798 .unwrap();
1799
1800 let formatter =
1801 HfTokenizerConfigJsonFormatter::new(chat_template, ContextMixins::new(&[])).unwrap();
1802
1803 let request: NvCreateChatCompletionRequest = serde_json::from_str(
1805 r#"{
1806 "model": "openai/gpt-oss-120b",
1807 "messages": [{"role":"user","content":"Search the repo for the string \"countHook\"."}],
1808 "tools": [
1809 {"type":"function","function":{"name":"grep","description":"search files","parameters":{"type":"object","properties":{"pattern":{"type":"string"},"path":{"type":"string"}},"required":["pattern"]}}},
1810 {"type":"function","function":{"name":"read","description":"read a file","parameters":{"type":"object","properties":{"filePath":{"type":"string"}},"required":["filePath"]}}}
1811 ]
1812 }"#,
1813 )
1814 .unwrap();
1815
1816 let rendered = formatter.render(&request).unwrap();
1817 eprintln!("================ RENDERED gpt-oss PROMPT (tools declared) ================");
1818 eprintln!("{rendered}");
1819 eprintln!("================ END RENDERED PROMPT ================");
1820 eprintln!("[diagnostics] does the rendered prompt contain…");
1821 for needle in [
1822 "commentary",
1823 "Calls to these tools",
1824 "functions",
1825 "# Tools",
1826 "<|channel|>",
1827 "constrain",
1828 "analysis",
1829 ] {
1830 eprintln!(
1831 " {:>22}: {}",
1832 format!("{needle:?}"),
1833 rendered.contains(needle)
1834 );
1835 }
1836 }
1837
1838 #[test]
1840 fn test_convert_media_url_to_placeholder_single_type() {
1841 let mut content_array = vec![
1842 serde_json::json!({"type": "text", "text": "Check this image:"}),
1843 serde_json::json!({"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}),
1844 serde_json::json!({"type": "text", "text": "What do you see?"}),
1845 ];
1846
1847 let conversions = &[("image_url", "image")];
1848 convert_media_url_to_placeholder(&mut content_array, conversions);
1849
1850 assert_eq!(content_array.len(), 3);
1851 assert_eq!(content_array[0]["type"], "text");
1853 assert_eq!(content_array[0]["text"], "Check this image:");
1854 assert_eq!(content_array[1]["type"], "image");
1856 assert!(content_array[1].get("image_url").is_none());
1857 assert_eq!(content_array[2]["type"], "text");
1859 assert_eq!(content_array[2]["text"], "What do you see?");
1860 }
1861
1862 #[test]
1864 fn test_convert_media_url_to_placeholder_multiple_same_type() {
1865 let mut content_array = vec![
1866 serde_json::json!({"type": "image_url", "image_url": {"url": "https://example.com/image1.jpg"}}),
1867 serde_json::json!({"type": "text", "text": "vs"}),
1868 serde_json::json!({"type": "image_url", "image_url": {"url": "https://example.com/image2.jpg"}}),
1869 ];
1870
1871 let conversions = &[("image_url", "image")];
1872 convert_media_url_to_placeholder(&mut content_array, conversions);
1873
1874 assert_eq!(content_array.len(), 3);
1875 assert_eq!(content_array[0]["type"], "image");
1876 assert_eq!(content_array[1]["type"], "text");
1877 assert_eq!(content_array[2]["type"], "image");
1878 }
1879
1880 #[test]
1882 fn test_convert_media_url_to_placeholder_selective_conversion() {
1883 let mut content_array = vec![
1884 serde_json::json!({"type": "audio_url", "audio_url": {"url": "https://example.com/audio.mp3"}}),
1885 serde_json::json!({"type": "video_url", "video_url": {"url": "https://example.com/video.mp4"}}),
1886 serde_json::json!({"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}),
1887 ];
1888
1889 let conversions = &[("image_url", "image")];
1891 convert_media_url_to_placeholder(&mut content_array, conversions);
1892
1893 assert_eq!(content_array.len(), 3);
1894 assert_eq!(content_array[0]["type"], "audio_url");
1896 assert!(content_array[0].get("audio_url").is_some());
1897 assert_eq!(content_array[1]["type"], "video_url");
1898 assert!(content_array[1].get("video_url").is_some());
1899 assert_eq!(content_array[2]["type"], "image");
1901 assert!(content_array[2].get("image_url").is_none());
1902 }
1903
1904 #[test]
1906 fn test_convert_media_url_to_placeholder_multiple_types() {
1907 let mut content_array = vec![
1908 serde_json::json!({"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}),
1909 serde_json::json!({"type": "text", "text": "and listen to"}),
1910 serde_json::json!({"type": "audio_url", "audio_url": {"url": "https://example.com/audio.mp3"}}),
1911 serde_json::json!({"type": "text", "text": "and watch"}),
1912 serde_json::json!({"type": "video_url", "video_url": {"url": "https://example.com/video.mp4"}}),
1913 ];
1914
1915 let conversions = &[
1917 ("image_url", "image"),
1918 ("audio_url", "audio"),
1919 ("video_url", "video"),
1920 ];
1921 convert_media_url_to_placeholder(&mut content_array, conversions);
1922
1923 assert_eq!(content_array.len(), 5);
1924 assert_eq!(content_array[0]["type"], "image");
1925 assert!(content_array[0].get("image_url").is_none());
1926 assert_eq!(content_array[1]["type"], "text");
1927 assert_eq!(content_array[2]["type"], "audio");
1928 assert!(content_array[2].get("audio_url").is_none());
1929 assert_eq!(content_array[3]["type"], "text");
1930 assert_eq!(content_array[4]["type"], "video");
1931 assert!(content_array[4].get("video_url").is_none());
1932 }
1933
1934 #[test]
1936 fn test_convert_media_url_to_placeholder_no_conversions() {
1937 let mut content_array = vec![
1938 serde_json::json!({"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}),
1939 serde_json::json!({"type": "text", "text": "hello"}),
1940 ];
1941
1942 let conversions: &[(&str, &str)] = &[];
1943 convert_media_url_to_placeholder(&mut content_array, conversions);
1944
1945 assert_eq!(content_array.len(), 2);
1946 assert_eq!(content_array[0]["type"], "image_url");
1948 assert!(content_array[0].get("image_url").is_some());
1949 assert_eq!(content_array[1]["type"], "text");
1950 }
1951
1952 #[test]
1955 fn test_default_media_type_conversions_only_converts_image_url() {
1956 let mut content_array = vec![
1957 serde_json::json!({"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}),
1958 serde_json::json!({"type": "video_url", "video_url": {"url": "https://example.com/video.mp4"}}),
1959 serde_json::json!({"type": "audio_url", "audio_url": {"url": "https://example.com/audio.mp3"}}),
1960 serde_json::json!({"type": "text", "text": "hello"}),
1961 ];
1962
1963 convert_media_url_to_placeholder(&mut content_array, DEFAULT_MEDIA_TYPE_CONVERSIONS);
1965
1966 assert_eq!(content_array.len(), 4);
1967
1968 assert_eq!(content_array[0]["type"], "image");
1970 assert!(content_array[0].get("image_url").is_none());
1971
1972 assert_eq!(content_array[1]["type"], "video");
1974 assert!(content_array[1].get("video_url").is_none());
1975
1976 assert_eq!(content_array[2]["type"], "audio");
1978 assert!(content_array[2].get("audio_url").is_none());
1979
1980 assert_eq!(content_array[3]["type"], "text");
1982 assert_eq!(content_array[3]["text"], "hello");
1983 }
1984
1985 #[test]
1986 fn test_may_be_fix_tool_schema_missing_type_and_properties() {
1987 let json_str = r#"{
1988 "model": "gpt-4o",
1989 "messages": [],
1990 "tools": [
1991 {
1992 "type": "function",
1993 "function": {
1994 "name": "get_weather",
1995 "description": "Get the current weather in a given location",
1996 "parameters": {},
1997 "strict": null
1998 }
1999 }
2000 ]
2001 }"#;
2002
2003 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2004 let tools = serde_json::to_value(request.tools()).unwrap();
2005
2006 assert!(tools[0]["function"]["parameters"]["type"] == "object");
2007 assert!(
2008 tools[0]["function"]["parameters"]["properties"]
2009 == serde_json::Value::Object(Default::default())
2010 );
2011 }
2012
2013 #[test]
2014 fn test_may_be_fix_tool_schema_missing_type() {
2015 let json_str = r#"{
2016 "model": "gpt-4o",
2017 "messages": [],
2018 "tools": [
2019 {
2020 "type": "function",
2021 "function": {
2022 "name": "get_weather",
2023 "description": "Get the current weather in a given location",
2024 "parameters": {
2025 "properties": {
2026 "location": {
2027 "type": "string",
2028 "description": "City and state, e.g., 'San Francisco, CA'"
2029 }
2030 }
2031 },
2032 "strict": null
2033 }
2034 }
2035 ]
2036 }"#;
2037 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2038
2039 let tools = serde_json::to_value(request.tools()).unwrap();
2040
2041 assert_eq!(tools[0]["function"]["parameters"]["type"], "object");
2042
2043 let mut expected_properties = serde_json::Map::new();
2044 let mut location = serde_json::Map::new();
2045 location.insert(
2046 "type".to_string(),
2047 serde_json::Value::String("string".to_string()),
2048 );
2049 location.insert(
2050 "description".to_string(),
2051 serde_json::Value::String("City and state, e.g., 'San Francisco, CA'".to_string()),
2052 );
2053 expected_properties.insert("location".to_string(), serde_json::Value::Object(location));
2054
2055 assert_eq!(
2056 tools[0]["function"]["parameters"]["properties"],
2057 serde_json::Value::Object(expected_properties)
2058 );
2059 }
2060
2061 #[test]
2062 fn test_may_be_fix_tool_schema_missing_properties() {
2063 let json_str = r#"{
2064 "model": "gpt-4o",
2065 "messages": [],
2066 "tools": [
2067 {
2068 "type": "function",
2069 "function": {
2070 "name": "get_weather",
2071 "description": "Get the current weather in a given location",
2072 "parameters": {"type": "object"},
2073 "strict": null
2074 }
2075 }
2076 ]
2077 }"#;
2078
2079 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2080 let tools = serde_json::to_value(request.tools()).unwrap();
2081
2082 assert_eq!(
2083 tools[0]["function"]["parameters"]["properties"],
2084 serde_json::Value::Object(Default::default())
2085 );
2086 assert_eq!(tools[0]["function"]["parameters"]["type"], "object");
2087 }
2088
2089 #[test]
2090 fn test_may_be_fix_tool_schema_missing_description() {
2091 let json_str = r#"{
2095 "model": "gpt-4o",
2096 "messages": [],
2097 "tools": [
2098 {
2099 "type": "function",
2100 "function": {
2101 "name": "noop",
2102 "parameters": {
2103 "type": "object",
2104 "properties": { "x": { "type": "string" } },
2105 "required": ["x"],
2106 "additionalProperties": false
2107 },
2108 "strict": null
2109 }
2110 }
2111 ]
2112 }"#;
2113
2114 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2115 let tools = serde_json::to_value(request.tools()).unwrap();
2116
2117 assert_eq!(
2118 tools[0]["function"]["description"],
2119 serde_json::Value::String(String::new())
2120 );
2121 }
2122
2123 #[test]
2124 fn test_may_be_fix_tool_schema_null_description() {
2125 let json_str = r#"{
2127 "model": "gpt-4o",
2128 "messages": [],
2129 "tools": [
2130 {
2131 "type": "function",
2132 "function": {
2133 "name": "noop",
2134 "description": null,
2135 "parameters": {"type": "object", "properties": {}},
2136 "strict": null
2137 }
2138 }
2139 ]
2140 }"#;
2141
2142 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2143 let tools = serde_json::to_value(request.tools()).unwrap();
2144
2145 assert_eq!(
2146 tools[0]["function"]["description"],
2147 serde_json::Value::String(String::new())
2148 );
2149 }
2150
2151 #[test]
2152 fn test_may_be_fix_tool_schema_preserves_description() {
2153 let json_str = r#"{
2155 "model": "gpt-4o",
2156 "messages": [],
2157 "tools": [
2158 {
2159 "type": "function",
2160 "function": {
2161 "name": "get_weather",
2162 "description": "Get the current weather in a given location",
2163 "parameters": {"type": "object", "properties": {}},
2164 "strict": null
2165 }
2166 }
2167 ]
2168 }"#;
2169
2170 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2171 let tools = serde_json::to_value(request.tools()).unwrap();
2172
2173 assert_eq!(
2174 tools[0]["function"]["description"],
2175 "Get the current weather in a given location"
2176 );
2177 }
2178
2179 #[test]
2181 fn test_may_be_fix_msg_content_user_multipart() {
2182 let json_str = r#"{
2183 "model": "gpt-4o",
2184 "messages": [
2185 {
2186 "role": "user",
2187 "content": [
2188 {"type": "text", "text": "part 1"},
2189 {"type": "text", "text": "part 2"}
2190 ]
2191 }
2192 ]
2193 }"#;
2194
2195 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2196 let messages_raw = serde_json::to_value(request.messages()).unwrap();
2197
2198 let messages =
2200 serde_json::to_value(may_be_fix_msg_content(messages_raw, false, None)).unwrap();
2201
2202 assert_eq!(
2204 messages[0]["content"],
2205 serde_json::Value::String("part 1\npart 2".to_string())
2206 );
2207 }
2208
2209 #[test]
2212 fn test_may_be_fix_msg_content_mixed_messages() {
2213 let json_str = r#"{
2214 "model": "gpt-4o",
2215 "messages": [
2216 {
2217 "role": "system",
2218 "content": "You are a helpful assistant"
2219 },
2220 {
2221 "role": "user",
2222 "content": [
2223 {"type": "text", "text": "Hello"},
2224 {"type": "text", "text": "World"}
2225 ]
2226 },
2227 {
2228 "role": "assistant",
2229 "content": "Hi there!"
2230 },
2231 {
2232 "role": "user",
2233 "content": [
2234 {"type": "text", "text": "Another"},
2235 {"type": "text", "text": "multi-part"},
2236 {"type": "text", "text": "message"}
2237 ]
2238 }
2239 ]
2240 }"#;
2241
2242 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2243 let messages_raw = serde_json::to_value(request.messages()).unwrap();
2244
2245 let messages =
2247 serde_json::to_value(may_be_fix_msg_content(messages_raw, false, None)).unwrap();
2248
2249 assert_eq!(
2251 messages[0]["content"],
2252 serde_json::Value::String("You are a helpful assistant".to_string())
2253 );
2254
2255 assert_eq!(
2257 messages[1]["content"],
2258 serde_json::Value::String("Hello\nWorld".to_string())
2259 );
2260
2261 assert_eq!(
2263 messages[2]["content"],
2264 serde_json::Value::String("Hi there!".to_string())
2265 );
2266
2267 assert_eq!(
2269 messages[3]["content"],
2270 serde_json::Value::String("Another\nmulti-part\nmessage".to_string())
2271 );
2272 }
2273
2274 #[test]
2276 fn test_may_be_fix_msg_content_empty_array() {
2277 let json_str = r#"{
2278 "model": "gpt-4o",
2279 "messages": [
2280 {
2281 "role": "user",
2282 "content": []
2283 }
2284 ]
2285 }"#;
2286
2287 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2288 let messages_raw = serde_json::to_value(request.messages()).unwrap();
2289
2290 let messages =
2292 serde_json::to_value(may_be_fix_msg_content(messages_raw, false, None)).unwrap();
2293
2294 assert!(messages[0]["content"].is_array());
2296 assert_eq!(messages[0]["content"].as_array().unwrap().len(), 0);
2297 }
2298
2299 #[test]
2306 fn test_may_be_fix_msg_content_empty_array_with_placeholder_template() {
2307 let json_str = r#"{
2308 "model": "phi-3-vision",
2309 "messages": [
2310 {
2311 "role": "user",
2312 "content": []
2313 }
2314 ]
2315 }"#;
2316
2317 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2318 let messages_raw = serde_json::to_value(request.messages()).unwrap();
2319
2320 let messages = serde_json::to_value(may_be_fix_msg_content(
2323 messages_raw,
2324 false,
2325 Some("<|image_{n}|>"),
2326 ))
2327 .unwrap();
2328
2329 assert!(
2330 messages[0]["content"].is_array(),
2331 "empty array should be preserved as `[]`, not flattened to `\"\"`"
2332 );
2333 assert_eq!(messages[0]["content"].as_array().unwrap().len(), 0);
2334 }
2335
2336 #[test]
2338 fn test_may_be_fix_msg_content_single_text() {
2339 let json_str = r#"{
2340 "model": "gpt-4o",
2341 "messages": [
2342 {
2343 "role": "user",
2344 "content": "Simple text message"
2345 }
2346 ]
2347 }"#;
2348
2349 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2350 let messages_raw = serde_json::to_value(request.messages()).unwrap();
2351
2352 let messages =
2354 serde_json::to_value(may_be_fix_msg_content(messages_raw, false, None)).unwrap();
2355
2356 assert_eq!(
2358 messages[0]["content"],
2359 serde_json::Value::String("Simple text message".to_string())
2360 );
2361 }
2362
2363 #[test]
2366 fn test_may_be_fix_msg_content_mixed_types() {
2367 let json_str = r#"{
2368 "model": "gpt-4o",
2369 "messages": [
2370 {
2371 "role": "user",
2372 "content": [
2373 {"type": "text", "text": "Check this image:"},
2374 {"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}},
2375 {"type": "text", "text": "What do you see?"}
2376 ]
2377 }
2378 ]
2379 }"#;
2380
2381 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2382 let messages_raw = serde_json::to_value(request.messages()).unwrap();
2383
2384 let messages =
2386 serde_json::to_value(may_be_fix_msg_content(messages_raw, false, None)).unwrap();
2387
2388 assert!(messages[0]["content"].is_array());
2391 let content_array = messages[0]["content"].as_array().unwrap();
2392 assert_eq!(content_array.len(), 3);
2393 assert_eq!(content_array[0]["type"], "text");
2394 assert_eq!(content_array[1]["type"], "image");
2395 assert!(content_array[1].get("image_url").is_none());
2396 assert_eq!(content_array[2]["type"], "text");
2397 }
2398
2399 #[test]
2405 fn test_may_be_fix_msg_content_flattens_phi3_style() {
2406 let json_str = r#"{
2407 "model": "phi-3-vision",
2408 "messages": [
2409 {
2410 "role": "user",
2411 "content": [
2412 {"type": "text", "text": "First "},
2413 {"type": "image_url", "image_url": {"url": "https://example.com/a.jpg"}},
2414 {"type": "text", "text": " then "},
2415 {"type": "image_url", "image_url": {"url": "https://example.com/b.jpg"}},
2416 {"type": "text", "text": "?"}
2417 ]
2418 }
2419 ]
2420 }"#;
2421 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2422 let messages_raw = serde_json::to_value(request.messages()).unwrap();
2423
2424 let messages = serde_json::to_value(may_be_fix_msg_content(
2425 messages_raw,
2426 false,
2427 Some("<|image_{n}|>"),
2428 ))
2429 .unwrap();
2430
2431 let content = messages[0]["content"].as_str().expect("content flattened");
2432 assert_eq!(content, "First <|image_1|> then <|image_2|>?");
2433 }
2434
2435 #[test]
2437 fn test_may_be_fix_msg_content_flattens_llava_style() {
2438 let json_str = r#"{
2439 "model": "llava-1.5-7b-hf",
2440 "messages": [
2441 {
2442 "role": "user",
2443 "content": [
2444 {"type": "text", "text": "Describe: "},
2445 {"type": "image_url", "image_url": {"url": "https://example.com/x.jpg"}}
2446 ]
2447 }
2448 ]
2449 }"#;
2450 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2451 let messages_raw = serde_json::to_value(request.messages()).unwrap();
2452
2453 let messages =
2454 serde_json::to_value(may_be_fix_msg_content(messages_raw, false, Some("<image>")))
2455 .unwrap();
2456
2457 let content = messages[0]["content"].as_str().expect("content flattened");
2458 assert_eq!(content, "Describe: <image>");
2459 }
2460
2461 #[test]
2467 fn test_may_be_fix_msg_content_flattens_empty_placeholder() {
2468 let json_str = r#"{
2469 "model": "nvidia/NVIDIA-Nemotron-Parse-v1.2",
2470 "messages": [
2471 {
2472 "role": "user",
2473 "content": [
2474 {"type": "text", "text": "</s><s><predict_bbox><predict_classes><output_markdown><predict_no_text_in_pic>"},
2475 {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}
2476 ]
2477 }
2478 ]
2479 }"#;
2480 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2481 let messages_raw = serde_json::to_value(request.messages()).unwrap();
2482
2483 let messages =
2484 serde_json::to_value(may_be_fix_msg_content(messages_raw, false, Some(""))).unwrap();
2485
2486 let content = messages[0]["content"].as_str().expect("content flattened");
2487 assert_eq!(
2488 content,
2489 "</s><s><predict_bbox><predict_classes><output_markdown><predict_no_text_in_pic>"
2490 );
2491 }
2492
2493 #[test]
2500 fn test_render_nemotron_parse_passthrough() {
2501 use super::super::tokcfg::ChatTemplate;
2502 use super::{ContextMixins, HfTokenizerConfigJsonFormatter};
2503
2504 let chat_template: ChatTemplate = serde_json::from_value(serde_json::json!({
2505 "chat_template": "{% for message in messages %}{{ message['content'] }}{% endfor %}"
2506 }))
2507 .unwrap();
2508 let formatter =
2509 HfTokenizerConfigJsonFormatter::new(chat_template, ContextMixins::new(&[])).unwrap();
2510
2511 for prompt in [
2512 "</s><s><predict_bbox><predict_classes><output_markdown><predict_no_text_in_pic>",
2513 "</s><s><predict_bbox><predict_classes><output_markdown><predict_text_in_pic>",
2514 ] {
2515 let request: NvCreateChatCompletionRequest =
2516 serde_json::from_value(serde_json::json!({
2517 "model": "nvidia/NVIDIA-Nemotron-Parse-v1.2",
2518 "messages": [{
2519 "role": "user",
2520 "content": [
2521 {"type": "text", "text": prompt},
2522 {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}
2523 ]
2524 }]
2525 }))
2526 .unwrap();
2527
2528 let rendered = formatter.render(&request).unwrap();
2529 assert_eq!(
2530 rendered, prompt,
2531 "rendered prompt must be the control tokens only, with no JSON-serialized image array"
2532 );
2533 }
2534 }
2535
2536 #[test]
2539 fn test_may_be_fix_msg_content_non_text_only() {
2540 let json_str = r#"{
2541 "model": "gpt-4o",
2542 "messages": [
2543 {
2544 "role": "user",
2545 "content": [
2546 {"type": "image_url", "image_url": {"url": "https://example.com/image1.jpg"}},
2547 {"type": "image_url", "image_url": {"url": "https://example.com/image2.jpg"}}
2548 ]
2549 }
2550 ]
2551 }"#;
2552
2553 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2554 let messages_raw = serde_json::to_value(request.messages()).unwrap();
2555
2556 let messages =
2558 serde_json::to_value(may_be_fix_msg_content(messages_raw, false, None)).unwrap();
2559
2560 assert!(messages[0]["content"].is_array());
2562 let content_array = messages[0]["content"].as_array().unwrap();
2563 assert_eq!(content_array.len(), 2);
2564 assert_eq!(content_array[0]["type"], "image");
2565 assert_eq!(content_array[1]["type"], "image");
2566 }
2567
2568 #[test]
2569 fn test_none_tools_safe_for_all_templates() {
2570 use super::tokcfg::ChatTemplate;
2571 use super::{ContextMixins, HfTokenizerConfigJsonFormatter};
2572
2573 let length_template = r#"
2577{%- if tools is iterable and tools | length > 0 %}
2578Tools available: {{ tools | length }}
2579{%- else %}
2580No tools
2581{%- endif %}
2582"#;
2583
2584 let no_tool_template = r#"
2587{%- if tools is not none %}
2588TOOL MODE
2589{%- else %}
2590NORMAL MODE
2591{%- endif %}
2592"#;
2593
2594 let chat_template: ChatTemplate = serde_json::from_value(serde_json::json!({
2595 "chat_template": [
2596 {"safe_length": length_template},
2597 {"no_tool": no_tool_template}
2598 ]
2599 }))
2600 .unwrap();
2601
2602 let formatter =
2603 HfTokenizerConfigJsonFormatter::new(chat_template, ContextMixins::new(&[])).unwrap();
2604
2605 let ctx = context! { tools => Option::<Value>::None };
2606
2607 let result1 = formatter
2608 .env
2609 .get_template("safe_length")
2610 .unwrap()
2611 .render(&ctx);
2612 println!("Safe length template with no tools => None: {:?}", result1);
2613 assert!(
2614 result1.is_ok(),
2615 "Jinja template with and conditional and length filter should handle None: {:?}",
2616 result1
2617 );
2618 assert!(
2619 result1.unwrap().contains("No tools"),
2620 "Should show 'No tools'"
2621 );
2622
2623 let result2 = formatter.env.get_template("no_tool").unwrap().render(&ctx);
2624 println!("Default template with no tools => None: {:?}", result2);
2625 assert!(
2626 result2.is_ok(),
2627 "Jinja template with if tools is not none conditional should handle None: {:?}",
2628 result2
2629 );
2630 assert!(result2.unwrap().contains("NORMAL MODE"));
2631 }
2632
2633 #[test]
2635 fn test_may_be_fix_msg_content_multiple_content_types() {
2636 let json_str = r#"{
2638 "model": "gpt-4o",
2639 "messages": [
2640 {
2641 "role": "user",
2642 "content": [
2643 {"type": "text", "text": "Listen to this:"},
2644 {"type": "audio_url", "audio_url": {"url": "https://example.com/audio.mp3"}},
2645 {"type": "text", "text": "And look at:"},
2646 {"type": "image_url", "image_url": {"url": "https://example.com/img.jpg"}},
2647 {"type": "text", "text": "What do you think?"}
2648 ]
2649 }
2650 ]
2651 }"#;
2652
2653 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2654 let messages_raw = serde_json::to_value(request.messages()).unwrap();
2655 let messages =
2656 serde_json::to_value(may_be_fix_msg_content(messages_raw, false, None)).unwrap();
2657
2658 assert!(messages[0]["content"].is_array());
2660 let content_array = messages[0]["content"].as_array().unwrap();
2661 assert_eq!(content_array.len(), 5);
2662 assert_eq!(content_array[0]["type"], "text");
2663 assert_eq!(content_array[1]["type"], "audio");
2664 assert_eq!(content_array[2]["type"], "text");
2665 assert_eq!(content_array[3]["type"], "image");
2666 assert_eq!(content_array[4]["type"], "text");
2667
2668 let json_str = r#"{
2670 "model": "gpt-4o",
2671 "messages": [
2672 {
2673 "role": "user",
2674 "content": [
2675 {"type": "text", "text": "Check this:"},
2676 {"type": "video_url", "video_url": {"url": "https://example.com/vid.mp4"}},
2677 {"type": "text", "text": "Interesting?"}
2678 ]
2679 }
2680 ]
2681 }"#;
2682
2683 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2684 let messages_raw = serde_json::to_value(request.messages()).unwrap();
2685 let messages =
2686 serde_json::to_value(may_be_fix_msg_content(messages_raw, false, None)).unwrap();
2687
2688 assert!(messages[0]["content"].is_array());
2690 assert_eq!(messages[0]["content"].as_array().unwrap().len(), 3);
2691 }
2692
2693 #[test]
2694 fn test_normalize_tool_arguments_tojson() {
2695 let tmpl = r#"{{ messages[0].tool_calls[0].function.arguments | tojson }}"#;
2696
2697 let mut messages = serde_json::Value::Array(vec![serde_json::json!({
2699 "role": "assistant",
2700 "tool_calls": [{
2701 "type": "function",
2702 "function": {
2703 "name": "get_current_weather",
2704 "arguments": "{\"format\":\"celsius\",\"location\":\"San Francisco, CA\"}"
2705 }
2706 }]
2707 })]);
2708
2709 normalize_tool_calls_arguments_in_messages(&mut messages);
2710
2711 let mut env = Environment::new();
2712 env.add_filter("tojson", super::super::tokcfg::tojson);
2713 env.add_template("t", tmpl).unwrap();
2714 let out = env
2715 .get_template("t")
2716 .unwrap()
2717 .render(context! { messages => crate::python::template_value(&messages) })
2718 .unwrap();
2719
2720 assert_eq!(
2723 out,
2724 r#"{"format": "celsius", "location": "San Francisco, CA"}"#
2725 );
2726 }
2727
2728 #[test]
2729 fn test_normalize_tool_arguments_items_loop() {
2730 let tmpl = r#"{% for k, v in messages[0].tool_calls[0].function.arguments|items %}{{k}}={{v}};{% endfor %}"#;
2731
2732 let mut messages = serde_json::Value::Array(vec![serde_json::json!({
2733 "role": "assistant",
2734 "tool_calls": [{
2735 "type": "function",
2736 "function": {
2737 "name": "f",
2738 "arguments": "{\"a\":1,\"b\":\"x\"}"
2739 }
2740 }]
2741 })]);
2742
2743 normalize_tool_calls_arguments_in_messages(&mut messages);
2744
2745 let mut env = Environment::new();
2746 env.add_template("t", tmpl).unwrap();
2747 let out = env
2748 .get_template("t")
2749 .unwrap()
2750 .render(context! { messages => crate::python::template_value(&messages) })
2751 .unwrap();
2752
2753 assert!(out == "a=1;b=x;" || out == "b=x;a=1;");
2754 }
2755
2756 #[test]
2757 fn test_normalize_tool_arguments_legacy_function_call() {
2758 let mut messages = serde_json::Value::Array(vec![serde_json::json!({
2760 "role": "assistant",
2761 "function_call": {
2762 "name": "get_weather",
2763 "arguments": "{\"location\":\"NYC\"}"
2764 }
2765 })]);
2766
2767 normalize_function_call_arguments_in_messages(&mut messages);
2768
2769 assert_eq!(
2770 messages[0]["function_call"]["arguments"],
2771 serde_json::json!({"location": "NYC"})
2772 );
2773 }
2774
2775 #[test]
2776 fn test_normalize_tool_arguments_malformed_json_passthrough() {
2777 let mut messages = serde_json::Value::Array(vec![serde_json::json!({
2779 "role": "assistant",
2780 "tool_calls": [{
2781 "type": "function",
2782 "function": {
2783 "name": "f",
2784 "arguments": "not valid json at all"
2785 }
2786 }]
2787 })]);
2788
2789 normalize_tool_calls_arguments_in_messages(&mut messages);
2790
2791 assert_eq!(
2792 messages[0]["tool_calls"][0]["function"]["arguments"],
2793 serde_json::Value::String("not valid json at all".to_string())
2794 );
2795 }
2796
2797 #[test]
2798 fn test_normalize_tool_arguments_with_multimodal_content() {
2799 let json_str = r#"{
2800 "model": "gpt-4o",
2801 "messages": [
2802 {
2803 "role": "user",
2804 "content": [
2805 {"type": "text", "text": "Check this:"},
2806 {"type": "video_url", "video_url": {"url": "https://example.com/vid.mp4"}},
2807 {"type": "text", "text": "Interesting?"}
2808 ]
2809 },
2810 {
2811 "role": "assistant",
2812 "tool_calls": [{
2813 "id": "call_123",
2814 "type": "function",
2815 "function": {
2816 "name": "analyze_video",
2817 "arguments": "{\"url\":\"https://example.com/vid.mp4\",\"format\":\"mp4\"}"
2818 }
2819 }]
2820 }
2821 ]
2822 }"#;
2823
2824 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2825 let messages_raw = serde_json::to_value(request.messages()).unwrap();
2826
2827 let mut messages =
2829 serde_json::to_value(may_be_fix_msg_content(messages_raw, false, None)).unwrap();
2830
2831 normalize_tool_calls_arguments_in_messages(&mut messages);
2832
2833 assert!(messages[0]["content"].is_array());
2835 assert_eq!(messages[0]["content"].as_array().unwrap().len(), 3);
2836
2837 assert!(messages[1]["tool_calls"][0]["function"]["arguments"].is_object());
2839 assert_eq!(
2840 messages[1]["tool_calls"][0]["function"]["arguments"]["url"],
2841 "https://example.com/vid.mp4"
2842 );
2843 }
2844
2845 #[test]
2848 fn test_minimax_m3_history_tool_call_float_arguments_match_hf() {
2849 let template = r#"{%- set ns_token = ']<]minimax[>[' -%}
2850{%- macro to_xml(val, ns) -%}
2851{%- if val is mapping -%}
2852{%- for k, v in val.items() if v is not none -%}
2853{{ ns }}<{{ k }}>{{ to_xml(v, ns) }}{{ ns }}</{{ k }}>
2854{%- endfor -%}
2855{%- elif val is iterable and val is not string -%}
2856{%- for item in val -%}
2857{{ ns }}<item>{{ to_xml(item, ns) }}{{ ns }}</item>
2858{%- endfor -%}
2859{%- elif val is none -%}
2860{%- elif val is boolean -%}
2861{{ val | tojson }}
2862{%- else -%}
2863{{ val }}
2864{%- endif -%}
2865{%- endmacro -%}
2866{%- for message in messages if message.tool_calls -%}
2867{%- for tool_call in message.tool_calls -%}
2868{%- if tool_call.function -%}
2869{%- set tool_call = tool_call.function -%}
2870{%- endif -%}
2871{{- ns_token + '<invoke name="' + tool_call.name + '">' }}
2872{%- set _args = tool_call.arguments -%}
2873{%- for k, v in _args.items() if v is not none %}
2874{{- ns_token + '<' + k + '>' -}}
2875{{- to_xml(v, ns_token) -}}
2876{{- ns_token + '</' + k + '>' }}
2877{%- endfor -%}
2878{{- ns_token + '</invoke>' ~ '\n' }}
2879{%- endfor -%}
2880{%- endfor -%}"#;
2881 let rendered = render_shape(
2882 &formatter_for(template),
2883 json!([
2884 {"role": "user", "content": "u"},
2885 {"role": "assistant", "content": "", "tool_calls": [{
2886 "id": "c1",
2887 "type": "function",
2888 "function": {
2889 "name": "fit",
2890 "arguments": r#"{"tolerance": 1e-07, "bounds": [0.00001, 1e16]}"#
2891 }
2892 }]}
2893 ]),
2894 )
2895 .unwrap();
2896 assert_eq!(
2897 rendered,
2898 "]<]minimax[>[<invoke name=\"fit\">]<]minimax[>[<tolerance>1e-07]<]minimax[>[</tolerance>]<]minimax[>[<bounds>]<]minimax[>[<item>1e-05]<]minimax[>[</item>]<]minimax[>[<item>1e+16]<]minimax[>[</item>]<]minimax[>[</bounds>]<]minimax[>[</invoke>\n"
2899 );
2900 }
2901
2902 #[test]
2905 fn test_qwen3_coder_history_tool_call_float_arguments_match_hf() {
2906 let template = r#"{%- for message in messages if message.tool_calls -%}
2907{%- for tool_call in message.tool_calls %}
2908 {%- if tool_call.function is defined %}
2909 {%- set tool_call = tool_call.function %}
2910 {%- endif %}
2911 {%- for args_name, args_value in tool_call.arguments|items %}
2912 {{- '<parameter=' + args_name + '>\n' }}
2913 {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
2914 {{- args_value }}
2915 {{- '\n</parameter>\n' }}
2916 {%- endfor %}
2917{%- endfor %}
2918{%- endfor %}"#;
2919 let rendered = render_shape(
2920 &formatter_for(template),
2921 json!([
2922 {"role": "user", "content": "u"},
2923 {"role": "assistant", "content": "", "tool_calls": [{
2924 "id": "c1",
2925 "type": "function",
2926 "function": {
2927 "name": "fit",
2928 "arguments": r#"{"tolerance": 1e-07, "bounds": [0.00001, 1e16]}"#
2929 }
2930 }]}
2931 ]),
2932 )
2933 .unwrap();
2934 assert_eq!(
2935 rendered,
2936 "<parameter=tolerance>\n1e-07\n</parameter>\n<parameter=bounds>\n[1e-05, 1e+16]\n</parameter>\n"
2937 );
2938 }
2939
2940 #[test]
2942 fn test_may_be_fix_msg_content_string_to_array() {
2943 let json_str = r#"{
2944 "model": "gpt-4o",
2945 "messages": [
2946 {
2947 "role": "user",
2948 "content": "Hello, how are you?"
2949 }
2950 ]
2951 }"#;
2952
2953 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2954 let messages_raw = serde_json::to_value(request.messages()).unwrap();
2955
2956 let messages =
2958 serde_json::to_value(may_be_fix_msg_content(messages_raw, true, None)).unwrap();
2959
2960 assert!(messages[0]["content"].is_array());
2962 let content_array = messages[0]["content"].as_array().unwrap();
2963 assert_eq!(content_array.len(), 1);
2964 assert_eq!(content_array[0]["type"], "text");
2965 assert_eq!(content_array[0]["text"], "Hello, how are you?");
2966 }
2967
2968 #[test]
2970 fn test_may_be_fix_msg_content_array_preserved_with_multimodal() {
2971 let json_str = r#"{
2972 "model": "gpt-4o",
2973 "messages": [
2974 {
2975 "role": "user",
2976 "content": [
2977 {"type": "text", "text": "part 1"},
2978 {"type": "text", "text": "part 2"}
2979 ]
2980 }
2981 ]
2982 }"#;
2983
2984 let request: NvCreateChatCompletionRequest = serde_json::from_str(json_str).unwrap();
2985 let messages_raw = serde_json::to_value(request.messages()).unwrap();
2986
2987 let messages =
2989 serde_json::to_value(may_be_fix_msg_content(messages_raw, true, None)).unwrap();
2990
2991 assert!(messages[0]["content"].is_array());
2993 let content_array = messages[0]["content"].as_array().unwrap();
2994 assert_eq!(content_array.len(), 2);
2995 assert_eq!(content_array[0]["text"], "part 1");
2996 assert_eq!(content_array[1]["text"], "part 2");
2997 }
2998
2999 fn user() -> Msg {
3000 Msg::User(Default::default())
3001 }
3002 fn tool() -> Msg {
3003 Msg::Tool(Default::default())
3004 }
3005
3006 fn dummy_state(messages: Vec<Msg>) -> NvCreateChatCompletionRequest {
3007 let json = serde_json::json!({
3008 "model": "test-model",
3009 "messages": messages
3010 });
3011 serde_json::from_value(json).unwrap()
3012 }
3013
3014 #[test]
3015 fn add_after_user() {
3016 let s = dummy_state(vec![user()]);
3017 assert!(s.should_add_generation_prompt());
3018 }
3019
3020 #[test]
3021 fn add_after_tool() {
3022 let s = dummy_state(vec![tool()]);
3023 assert!(s.should_add_generation_prompt());
3024 }
3025
3026 #[test]
3027 fn add_when_empty() {
3028 let s = dummy_state(vec![]);
3029 assert!(s.should_add_generation_prompt());
3030 }
3031
3032 fn tool_aware_formatter(
3034 exclude_tools_when_tool_choice_none: bool,
3035 ) -> HfTokenizerConfigJsonFormatter {
3036 let template = r#"
3037{%- if tools is iterable and tools | length > 0 %}
3038TOOL_MODE tools={{ tools | length }}
3039{%- else %}
3040NORMAL_MODE
3041{%- endif %}
3042{{ messages[0].content }}"#;
3043
3044 let chat_template: super::tokcfg::ChatTemplate =
3045 serde_json::from_value(serde_json::json!({ "chat_template": template })).unwrap();
3046
3047 HfTokenizerConfigJsonFormatter::with_options(
3048 chat_template,
3049 ContextMixins::new(&[]),
3050 exclude_tools_when_tool_choice_none,
3051 )
3052 .unwrap()
3053 }
3054
3055 fn gemma4_tool_template_for_tests() -> &'static str {
3056 r#"
3057{{ bos_token }}
3058{%- set loop_messages = messages -%}
3059{%- set ns_turn = namespace(last_user_idx=-1) -%}
3060{%- for i in range(loop_messages | length) -%}
3061 {%- if loop_messages[i]['role'] == 'user' -%}
3062 {%- set ns_turn.last_user_idx = i -%}
3063 {%- endif -%}
3064{%- endfor -%}
3065{%- for message in loop_messages -%}
3066 {%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%}
3067 {{- '<|turn>' + role + '\n' }}
3068
3069 {%- if message.get('reasoning') and loop.index0 > ns_turn.last_user_idx and message.get('tool_calls') -%}
3070 {{- '<|channel>thought\n' + message['reasoning'] + '\n<channel|>'}}
3071 {%- endif -%}
3072
3073 {%- if message['tool_calls'] -%}
3074 {%- for tool_call in message['tool_calls'] -%}
3075 {%- set function = tool_call['function'] -%}
3076 {{- '<|tool_call>call:' + function['name'] + '{' -}}
3077 {%- if function['arguments'] is mapping -%}
3078 {%- set ns_args = namespace(found_first=false) -%}
3079 {%- for key, value in function['arguments'] | dictsort -%}
3080 {%- if ns_args.found_first %},{% endif -%}
3081 {%- set ns_args.found_first = true -%}
3082 {{- key -}}:{{- value -}}
3083 {%- endfor -%}
3084 {%- elif function['arguments'] is string -%}
3085 {{- function['arguments'] -}}
3086 {%- endif -%}
3087 {{- '}<tool_call|>' -}}
3088 {%- endfor -%}
3089 {%- endif -%}
3090
3091 {%- if message['content'] is string -%}
3092 {{- message['content'] -}}
3093 {%- endif -%}
3094 {{- '<turn|>\n' -}}
3095{%- endfor -%}
3096"#
3097 }
3098
3099 fn make_gemma4_tool_formatter_for_tests() -> HfTokenizerConfigJsonFormatter {
3100 let chat_template: ChatTemplate = serde_json::from_value(serde_json::json!({
3101 "chat_template": gemma4_tool_template_for_tests()
3102 }))
3103 .unwrap();
3104 HfTokenizerConfigJsonFormatter::new(chat_template, ContextMixins::new(&[])).unwrap()
3105 }
3106
3107 fn request_with_tool_choice(tool_choice: &str) -> NvCreateChatCompletionRequest {
3109 serde_json::from_value(serde_json::json!({
3110 "model": "test",
3111 "messages": [{"role": "user", "content": "hello"}],
3112 "tools": [{
3113 "type": "function",
3114 "function": {
3115 "name": "get_weather",
3116 "description": "Get weather",
3117 "parameters": {"type": "object", "properties": {"location": {"type": "string"}}}
3118 }
3119 }],
3120 "tool_choice": tool_choice
3121 }))
3122 .unwrap()
3123 }
3124
3125 #[test]
3126 fn test_exclude_tools_strips_when_tool_choice_none() {
3127 let formatter = tool_aware_formatter(true);
3128 let request = request_with_tool_choice("none");
3129 let result = formatter.render(&request).unwrap();
3130 assert!(
3131 result.contains("NORMAL_MODE"),
3132 "With exclude_tools=true and tool_choice=none, tools should be stripped. Got: {}",
3133 result
3134 );
3135 }
3136
3137 #[test]
3138 fn test_exclude_tools_keeps_when_tool_choice_auto() {
3139 let formatter = tool_aware_formatter(true);
3140 let request = request_with_tool_choice("auto");
3141 let result = formatter.render(&request).unwrap();
3142 assert!(
3143 result.contains("TOOL_MODE"),
3144 "With tool_choice=auto, tools should be included. Got: {}",
3145 result
3146 );
3147 }
3148
3149 #[test]
3150 fn test_no_exclude_tools_keeps_when_tool_choice_none() {
3151 let formatter = tool_aware_formatter(false);
3152 let request = request_with_tool_choice("none");
3153 let result = formatter.render(&request).unwrap();
3154 assert!(
3155 result.contains("TOOL_MODE"),
3156 "With exclude_tools=false and tool_choice=none, tools should NOT be stripped. Got: {}",
3157 result
3158 );
3159 }
3160
3161 #[test]
3162 fn test_response_format_in_template_context() {
3163 let formatter = formatter_for(
3164 r#"{%- if response_format is defined %}{{ response_format.json_schema.name }}: {{ response_format.json_schema.schema | tojson }}{% else %}none{% endif %}"#,
3165 );
3166 let with_format: NvCreateChatCompletionRequest = serde_json::from_value(json!({
3167 "model": "test",
3168 "messages": [{"role": "user", "content": "hi"}],
3169 "response_format": {
3170 "type": "json_schema",
3171 "json_schema": {"name": "count", "schema": {"type": "integer", "minimum": 0}}
3172 }
3173 }))
3174 .unwrap();
3175 let without_format: NvCreateChatCompletionRequest = serde_json::from_value(json!({
3176 "model": "test",
3177 "messages": [{"role": "user", "content": "hi"}]
3178 }))
3179 .unwrap();
3180
3181 assert_eq!(
3182 formatter.render(&with_format).unwrap(),
3183 r#"count: {"type": "integer", "minimum": 0}"#
3184 );
3185 assert_eq!(formatter.render(&without_format).unwrap(), "none");
3186 }
3187
3188 #[test]
3189 fn test_response_format_harmony_literals_are_ordinary_text() {
3190 let formatter = formatter_for(
3191 r#"<|start|>developer<|message|>{{ response_format.json_schema.description }}<|end|>"#,
3192 );
3193 let request: NvCreateChatCompletionRequest = serde_json::from_value(json!({
3194 "model": "test",
3195 "messages": [{"role": "user", "content": "hi"}],
3196 "response_format": {
3197 "type": "json_schema",
3198 "json_schema": {"name": "x", "description": "a <|end|> b", "schema": {}}
3199 }
3200 }))
3201 .unwrap();
3202 let prompt = formatter.render_prompt(&request).unwrap();
3203 let ordinary: Vec<_> = prompt
3204 .segments()
3205 .unwrap()
3206 .iter()
3207 .filter(|segment| !segment.allow_special)
3208 .map(|segment| segment.text.as_str())
3209 .collect();
3210
3211 assert_eq!(
3212 prompt.as_str(),
3213 "<|start|>developer<|message|>a <|end|> b<|end|>"
3214 );
3215 assert_eq!(ordinary, ["<|end|>"]);
3216 }
3217
3218 #[test]
3219 fn harmony_schema_keys_keep_json_and_token_boundaries() {
3220 let schema = json!({
3221 "type": "object",
3222 "properties": {
3223 "<|end|>": {"type": "string"},
3224 "A": {"$ref": "#/$defs/<|message|>"},
3225 "<|x\"y|>": {"type": "string"},
3226 "<|x\\y|>": {"type": "string"},
3227 "<|x\ny|>": {"type": "string"},
3228 "<|x\"<|end|>": {"type": "string"}
3229 },
3230 "required": ["<|end|>", "A", "<|x\"y|>", "<|x\\y|>", "<|x\ny|>", "<|x\"<|end|>"],
3231 "$defs": {"<|message|>": {"type": "integer"}},
3232 "additionalProperties": false
3233 });
3234 let request: NvCreateChatCompletionRequest = serde_json::from_value(json!({
3235 "model": "test",
3236 "messages": [{"role": "user", "content": "hi"}],
3237 "response_format": {
3238 "type": "json_schema",
3239 "json_schema": {"name": "tokens", "schema": schema}
3240 },
3241 "tools": [{"type": "function", "function": {
3242 "name": "tokens", "parameters": schema
3243 }}]
3244 }))
3245 .unwrap();
3246 for source in [
3247 "response_format.json_schema.schema",
3248 "tools[0].function.parameters",
3249 ] {
3250 let formatter = formatter_for(&format!(
3251 "<|start|>developer<|message|>{{{{ {source} | tojson }}}}<|end|>"
3252 ));
3253 let prompt = formatter.render_prompt(&request).unwrap();
3254 assert_eq!(prompt.as_str(), formatter.render(&request).unwrap());
3255 let body = prompt
3256 .as_str()
3257 .strip_prefix("<|start|>developer<|message|>")
3258 .unwrap()
3259 .strip_suffix("<|end|>")
3260 .unwrap();
3261 assert_eq!(
3262 serde_json::from_str::<serde_json::Value>(body).unwrap(),
3263 schema
3264 );
3265 let segments = prompt.segments().unwrap();
3266 let controls: String = segments
3267 .iter()
3268 .filter(|segment| segment.allow_special)
3269 .map(|segment| segment.text.as_str())
3270 .collect();
3271 for (marker, count) in [("<|end|>", 4), ("<|message|>", 2)] {
3272 assert_eq!(
3273 controls.matches(marker).count(),
3274 1,
3275 "{source}: {segments:?}"
3276 );
3277 assert_eq!(
3278 segments
3279 .iter()
3280 .filter(|segment| { !segment.allow_special && segment.text == marker })
3281 .count(),
3282 count,
3283 "{source}: {segments:?}"
3284 );
3285 }
3286 }
3287 }
3288
3289 #[test]
3290 fn harmony_schema_key_lookups_match_required_names() {
3291 let formatter = formatter_for(
3292 "{% set schema = response_format.json_schema.schema %}<|start|>developer<|message|>{{ schema.properties[schema.required[0]].type }}:{% for name, property in schema.properties.items() %}{{ name }}={{ name in schema.required }};{% endfor %}<|end|>",
3293 );
3294 let request: NvCreateChatCompletionRequest = serde_json::from_value(json!({
3295 "model": "test",
3296 "messages": [{"role": "user", "content": "hi"}],
3297 "response_format": {"type": "json_schema", "json_schema": {
3298 "name": "tokens", "schema": {
3299 "type": "object",
3300 "properties": {"<|end|>": {"type": "integer"}, "A": {"type": "string"}},
3301 "required": ["<|end|>"]
3302 }
3303 }}
3304 }))
3305 .unwrap();
3306 let prompt = formatter.render_prompt(&request).unwrap();
3307 assert_eq!(prompt.as_str(), formatter.render(&request).unwrap());
3308 assert!(
3309 prompt.as_str().contains("integer:<|end|>=True;A=False;"),
3310 "{prompt:?}"
3311 );
3312 assert!(
3313 prompt
3314 .segments()
3315 .unwrap()
3316 .iter()
3317 .any(|segment| { !segment.allow_special && segment.text == "<|end|>" })
3318 );
3319 }
3320
3321 #[test]
3322 fn test_inject_reasoning_content_segments_with_tool_calls() {
3323 let mut messages = serde_json::json!([
3325 {
3326 "role": "user",
3327 "content": "What is sqrt(144) and sqrt(256)?"
3328 },
3329 {
3330 "role": "assistant",
3331 "content": "Let me calculate those.",
3332 "reasoning_content": ["I need to compute sqrt(144)", "Now sqrt(256)", ""],
3333 "tool_calls": [
3334 {
3335 "id": "call_0",
3336 "type": "function",
3337 "function": {
3338 "name": "calculator",
3339 "arguments": "{\"expr\": \"sqrt(144)\"}"
3340 }
3341 },
3342 {
3343 "id": "call_1",
3344 "type": "function",
3345 "function": {
3346 "name": "calculator",
3347 "arguments": "{\"expr\": \"sqrt(256)\"}"
3348 }
3349 }
3350 ]
3351 }
3352 ]);
3353
3354 inject_reasoning_content_into_messages(&mut messages);
3355
3356 let assistant = &messages[1];
3357
3358 assert!(
3360 assistant.get("reasoning_content").is_none(),
3361 "reasoning_content should be removed after injection"
3362 );
3363
3364 let content = assistant["content"].as_str().unwrap();
3366 assert!(
3367 content.starts_with("<think>I need to compute sqrt(144)</think>"),
3368 "content should start with first reasoning segment, got: {}",
3369 content
3370 );
3371 assert!(
3372 content.contains("<think>Now sqrt(256)</think>"),
3373 "content should contain second reasoning segment"
3374 );
3375 assert!(
3377 !content.contains("<think></think>"),
3378 "empty segments should be skipped"
3379 );
3380 assert!(
3382 content.ends_with("Let me calculate those."),
3383 "original content should be at the end, got: {}",
3384 content
3385 );
3386
3387 assert!(assistant.get("tool_calls").is_some());
3389 assert_eq!(assistant["tool_calls"].as_array().unwrap().len(), 2);
3390 }
3391
3392 #[test]
3393 fn test_gemma4_template_renders_reasoning_content_segments_around_tool_calls() {
3394 let formatter = make_gemma4_tool_formatter_for_tests();
3395 assert!(
3396 formatter.tool_use_template_handles_reasoning,
3397 "Gemma4 template adaptation should make reasoning_content native"
3398 );
3399
3400 let request: NvCreateChatCompletionRequest = serde_json::from_value(serde_json::json!({
3401 "model": "gemma4-test",
3402 "messages": [
3403 {"role": "user", "content": "inspect two things"},
3404 {
3405 "role": "assistant",
3406 "content": null,
3407 "reasoning_content": [
3408 "Think before the first call.",
3409 "Think before the second call.",
3410 "Think after both calls."
3411 ],
3412 "tool_calls": [
3413 {
3414 "id": "call_0",
3415 "type": "function",
3416 "function": {
3417 "name": "first_tool",
3418 "arguments": "{\"path\":\".\"}"
3419 }
3420 },
3421 {
3422 "id": "call_1",
3423 "type": "function",
3424 "function": {
3425 "name": "second_tool",
3426 "arguments": "{\"path\":\"/tmp\"}"
3427 }
3428 }
3429 ]
3430 }
3431 ]
3432 }))
3433 .unwrap();
3434
3435 let rendered = formatter.render(&request).unwrap();
3436
3437 let expected = concat!(
3438 "<|channel>thought\nThink before the first call.\n<channel|>",
3439 "<|tool_call>call:first_tool{path:.}<tool_call|>",
3440 "<|channel>thought\nThink before the second call.\n<channel|>",
3441 "<|tool_call>call:second_tool{path:/tmp}<tool_call|>",
3442 "<|channel>thought\nThink after both calls.\n<channel|>"
3443 );
3444 assert!(
3445 rendered.contains(expected),
3446 "Gemma4 reasoning segments should stay adjacent to their tool calls, got: {rendered}"
3447 );
3448 assert!(!rendered.contains("<think>"));
3449 assert!(!rendered.contains("reasoning_content"));
3450 }
3451
3452 #[test]
3453 fn test_gemma4_template_renders_reasoning_content_without_tool_calls() {
3454 let formatter = make_gemma4_tool_formatter_for_tests();
3455 let request: NvCreateChatCompletionRequest = serde_json::from_value(serde_json::json!({
3456 "model": "gemma4-test",
3457 "messages": [
3458 {"role": "user", "content": "answer directly"},
3459 {
3460 "role": "assistant",
3461 "content": "Direct answer.",
3462 "reasoning_content": "Private thought."
3463 }
3464 ]
3465 }))
3466 .unwrap();
3467
3468 let rendered = formatter.render(&request).unwrap();
3469
3470 assert!(
3471 rendered.contains("<|channel>thought\nPrivate thought.\n<channel|>Direct answer."),
3472 "Gemma4 reasoning_content should render in the thought channel, got: {rendered}"
3473 );
3474 assert!(!rendered.contains("<think>"));
3475 assert!(!rendered.contains("reasoning_content"));
3476 }
3477
3478 #[test]
3484 fn test_reasoning_flag_is_per_template_not_global() {
3485 const PLAIN_DEFAULT: &str = "{{ bos_token }}{%- for message in messages -%}\
3488 {{ message['role'] }}: {{ message['content'] }}\n{%- endfor -%}";
3489
3490 let chat_template: ChatTemplate = serde_json::from_value(serde_json::json!({
3491 "chat_template": [
3492 {"default": PLAIN_DEFAULT},
3493 {"tool_use": gemma4_tool_template_for_tests()},
3494 ]
3495 }))
3496 .unwrap();
3497 let formatter =
3498 HfTokenizerConfigJsonFormatter::new(chat_template, ContextMixins::new(&[])).unwrap();
3499
3500 assert!(
3504 formatter.tool_use_template_handles_reasoning,
3505 "adapted gemma4 tool_use template should handle reasoning natively"
3506 );
3507 assert!(
3508 !formatter.default_template_handles_reasoning,
3509 "plain default template does not reference reasoning_content"
3510 );
3511
3512 let request: NvCreateChatCompletionRequest = serde_json::from_value(serde_json::json!({
3515 "model": "gemma4-test",
3516 "messages": [
3517 {"role": "user", "content": "answer directly"},
3518 {
3519 "role": "assistant",
3520 "content": "Direct answer.",
3521 "reasoning_content": "Private thought."
3522 }
3523 ]
3524 }))
3525 .unwrap();
3526
3527 let rendered = formatter.render(&request).unwrap();
3528 assert!(
3529 rendered.contains("<think>Private thought.</think>Direct answer."),
3530 "reasoning must be injected on the no-tool default path, got: {rendered}"
3531 );
3532 }
3533
3534 fn reasoning_tool_call_turn(reasoning: serde_json::Value) -> serde_json::Value {
3536 let call = |id: &str, expr: &str| {
3537 json!({"id": id, "type": "function",
3538 "function": {"name": "calc", "arguments": json!({"expr": expr}).to_string()}})
3539 };
3540 json!([
3541 {"role": "user", "content": "sqrt(144) + sqrt(256)?"},
3542 {"role": "assistant", "content": null, "reasoning_content": reasoning,
3543 "tool_calls": [call("call_0", "sqrt(144)"), call("call_1", "sqrt(256)")]},
3544 {"role": "tool", "tool_call_id": "call_0", "content": "12"},
3545 {"role": "tool", "tool_call_id": "call_1", "content": "16"}
3546 ])
3547 }
3548
3549 #[test]
3553 fn test_string_reasoning_template_joins_reasoning_content_segments() {
3554 const MINIMAX_REASONING_TMPL: &str = r#"{%- for message in messages -%}
3555{%- if message.role == 'assistant' -%}
3556{{- ']~b]ai' ~ '\n' -}}
3557{%- set reasoning_content = '' -%}
3558{%- if message.reasoning_content is string -%}
3559{%- set reasoning_content = message.reasoning_content -%}
3560{%- endif -%}
3561{%- if reasoning_content -%}
3562{{- '<think>' ~ '\n' ~ reasoning_content ~ '\n' ~ '</think>' ~ '\n\n' -}}
3563{%- endif -%}
3564{%- for tool_call in message.tool_calls -%}
3565{{- '<invoke name="' ~ tool_call.function.name ~ '">' -}}
3566{%- endfor -%}
3567{{- '[e~[\n' -}}
3568{%- else -%}
3569{{- ']~b]' ~ message.role ~ '\n' ~ message.content ~ '[e~[\n' -}}
3570{%- endif -%}
3571{%- endfor -%}"#;
3572 let f = formatter_for(MINIMAX_REASONING_TMPL);
3573 let expected = concat!(
3575 "]~b]user\nsqrt(144) + sqrt(256)?[e~[\n",
3576 "]~b]ai\n<think>\nCheck both.\nThen add.\n</think>\n\n",
3577 "<invoke name=\"calc\"><invoke name=\"calc\">[e~[\n",
3578 "]~b]tool\n12[e~[\n]~b]tool\n16[e~[\n",
3579 );
3580 for render in [render_shape, render_shape_with_tools] {
3582 for reasoning in [
3583 json!(["Check both.", "Then add.", ""]),
3584 json!("Check both.\nThen add."),
3585 ] {
3586 assert_eq!(
3587 render(&f, reasoning_tool_call_turn(reasoning)).unwrap(),
3588 expected
3589 );
3590 }
3591 }
3592 }
3593
3594 #[test]
3595 fn test_inject_reasoning_content_text_variant() {
3596 let mut messages = serde_json::json!([
3597 {
3598 "role": "assistant",
3599 "content": "The answer is 42.",
3600 "reasoning_content": "Let me think about this carefully."
3601 }
3602 ]);
3603
3604 inject_reasoning_content_into_messages(&mut messages);
3605
3606 let assistant = &messages[0];
3607 assert!(assistant.get("reasoning_content").is_none());
3608 let content = assistant["content"].as_str().unwrap();
3609 assert_eq!(
3610 content,
3611 "<think>Let me think about this carefully.</think>The answer is 42."
3612 );
3613 }
3614
3615 #[test]
3616 fn test_inject_reasoning_content_null_content() {
3617 let mut messages = serde_json::json!([
3619 {
3620 "role": "assistant",
3621 "content": null,
3622 "reasoning_content": "Thinking...",
3623 "tool_calls": [{"id": "call_0", "type": "function", "function": {"name": "f", "arguments": "{}"}}]
3624 }
3625 ]);
3626
3627 inject_reasoning_content_into_messages(&mut messages);
3628
3629 let content = messages[0]["content"].as_str().unwrap();
3630 assert_eq!(content, "<think>Thinking...</think>");
3631 assert!(messages[0].get("reasoning_content").is_none());
3632 }
3633
3634 #[test]
3635 fn test_inject_reasoning_content_skips_non_assistant() {
3636 let mut messages = serde_json::json!([
3637 {
3638 "role": "user",
3639 "content": "hello",
3640 "reasoning_content": "should not be touched"
3641 }
3642 ]);
3643
3644 inject_reasoning_content_into_messages(&mut messages);
3645
3646 assert!(messages[0].get("reasoning_content").is_some());
3648 }
3649
3650 fn make_test_formatter() -> HfTokenizerConfigJsonFormatter {
3652 use super::tokcfg::ChatTemplate;
3653 use super::{ContextMixins, HfTokenizerConfigJsonFormatter};
3654
3655 let template = r#"{%- for message in messages %}{{ message.role }}: {{ message.content }}
3658{%- endfor %}
3659{%- if add_generation_prompt %}assistant:{%- endif %}"#;
3660
3661 let chat_template: ChatTemplate = serde_json::from_value(serde_json::json!({
3662 "chat_template": template
3663 }))
3664 .unwrap();
3665
3666 HfTokenizerConfigJsonFormatter::new(chat_template, ContextMixins::new(&[])).unwrap()
3667 }
3668
3669 #[test]
3672 fn test_reasoning_content_text_roundtrip_render() {
3673 use super::OAIPromptFormatter;
3674 let formatter = make_test_formatter();
3675
3676 let request: NvCreateChatCompletionRequest = serde_json::from_value(serde_json::json!({
3677 "model": "test-model",
3678 "messages": [
3679 {"role": "user", "content": "What is sqrt(144)?"},
3680 {
3681 "role": "assistant",
3682 "content": "The answer is 12.",
3683 "reasoning_content": "I need to compute the square root of 144."
3684 },
3685 {"role": "user", "content": "Are you sure?"}
3686 ]
3687 }))
3688 .unwrap();
3689
3690 let rendered = formatter.render(&request).unwrap();
3691
3692 assert!(
3693 rendered.contains("<think>I need to compute the square root of 144.</think>"),
3694 "reasoning_content must appear as <think> block, got: {}",
3695 rendered
3696 );
3697 assert!(
3698 rendered.contains("The answer is 12."),
3699 "original content must be preserved"
3700 );
3701 assert!(
3702 !rendered.contains("reasoning_content"),
3703 "raw reasoning_content field should not leak into prompt"
3704 );
3705 }
3706
3707 #[test]
3711 fn test_reasoning_content_agentic_tool_call_roundtrip_render() {
3712 use super::OAIPromptFormatter;
3713 let formatter = make_test_formatter();
3714
3715 let request: NvCreateChatCompletionRequest = serde_json::from_value(serde_json::json!({
3716 "model": "test-model",
3717 "messages": [
3718 {"role": "user", "content": "What is sqrt(144) + sqrt(256)?"},
3719 {
3720 "role": "assistant",
3721 "content": null,
3722 "reasoning_content": "I need to compute both square roots. Let me start with sqrt(144).",
3723 "tool_calls": [{
3724 "id": "call_0",
3725 "type": "function",
3726 "function": {
3727 "name": "calculator",
3728 "arguments": "{\"expr\": \"sqrt(144)\"}"
3729 }
3730 }]
3731 },
3732 {
3733 "role": "tool",
3734 "tool_call_id": "call_0",
3735 "content": "12"
3736 },
3737 {
3738 "role": "assistant",
3739 "content": "sqrt(144) = 12 and sqrt(256) = 16, so the answer is 28.",
3740 "reasoning_content": "Got 12 for sqrt(144). Now sqrt(256) = 16. Sum is 28."
3741 },
3742 {"role": "user", "content": "Thanks!"}
3743 ]
3744 }))
3745 .unwrap();
3746
3747 let rendered = formatter.render(&request).unwrap();
3748
3749 assert!(
3751 rendered.contains("<think>I need to compute both square roots"),
3752 "first turn reasoning must be in prompt, got: {}",
3753 rendered
3754 );
3755 assert!(
3757 rendered.contains("<think>Got 12 for sqrt(144)"),
3758 "second turn reasoning must be in prompt"
3759 );
3760 assert!(
3761 rendered.contains("the answer is 28"),
3762 "final answer content must be preserved"
3763 );
3764 assert!(
3766 !rendered.contains("reasoning_content"),
3767 "raw reasoning_content field should not leak into prompt"
3768 );
3769 }
3770
3771 #[test]
3773 fn test_reasoning_injected_when_template_ignores_it() {
3774 use super::OAIPromptFormatter;
3775 let formatter = make_test_formatter();
3776
3777 assert!(!formatter.default_template_handles_reasoning);
3779 assert!(!formatter.tool_use_template_handles_reasoning);
3780
3781 let request: NvCreateChatCompletionRequest = serde_json::from_value(serde_json::json!({
3782 "model": "test-model",
3783 "messages": [
3784 {"role": "user", "content": "Hello"},
3785 {
3786 "role": "assistant",
3787 "content": "Hi.",
3788 "reasoning_content": "The user said hello."
3789 },
3790 {"role": "user", "content": "Bye"}
3791 ]
3792 }))
3793 .unwrap();
3794
3795 let rendered = formatter.render(&request).unwrap();
3796 assert!(
3797 rendered.contains("<think>The user said hello.</think>"),
3798 "injection must happen when template ignores reasoning_content, got: {}",
3799 rendered
3800 );
3801 }
3802
3803 #[test]
3805 fn test_reasoning_not_injected_when_template_handles_it() {
3806 use super::tokcfg::ChatTemplate;
3807 use super::{ContextMixins, HfTokenizerConfigJsonFormatter, OAIPromptFormatter};
3808
3809 let template = r#"{%- for message in messages %}{%- if message.role == "assistant" and message.reasoning_content is defined and message.reasoning_content %}<think>{{ message.reasoning_content }}</think>
3811{%- endif %}{{ message.role }}: {{ message.content }}
3812{%- endfor %}
3813{%- if add_generation_prompt %}assistant:{%- endif %}"#;
3814
3815 let chat_template: ChatTemplate = serde_json::from_value(serde_json::json!({
3816 "chat_template": template
3817 }))
3818 .unwrap();
3819
3820 let formatter =
3821 HfTokenizerConfigJsonFormatter::new(chat_template, ContextMixins::new(&[])).unwrap();
3822
3823 assert!(formatter.default_template_handles_reasoning);
3825 assert!(formatter.tool_use_template_handles_reasoning);
3826
3827 let request: NvCreateChatCompletionRequest = serde_json::from_value(serde_json::json!({
3828 "model": "test-model",
3829 "messages": [
3830 {"role": "user", "content": "Hello"},
3831 {
3832 "role": "assistant",
3833 "content": "Hi.",
3834 "reasoning_content": "The user said hello."
3835 },
3836 {"role": "user", "content": "Bye"}
3837 ]
3838 }))
3839 .unwrap();
3840
3841 let rendered = formatter.render(&request).unwrap();
3842
3843 assert!(
3845 rendered.contains("<think>The user said hello.</think>"),
3846 "template must render reasoning_content natively, got: {}",
3847 rendered
3848 );
3849 let think_count = rendered.matches("<think>").count();
3851 assert_eq!(
3852 think_count, 1,
3853 "must have exactly one <think> block (from template), got {} in: {}",
3854 think_count, rendered
3855 );
3856 }
3857
3858 const QWEN3_THINKING_TEMPLATE: &str = r##"{%- if tools %}
3862 {{- '<|im_start|>system\n' }}
3863 {%- if messages[0].role == 'system' %}
3864 {{- messages[0].content + '\n\n' }}
3865 {%- endif %}
3866 {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
3867 {%- for tool in tools %}
3868 {{- "\n" }}
3869 {{- tool | tojson }}
3870 {%- endfor %}
3871 {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
3872{%- else %}
3873 {%- if messages[0].role == 'system' %}
3874 {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
3875 {%- endif %}
3876{%- endif %}
3877{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
3878{%- for message in messages[::-1] %}
3879 {%- set index = (messages|length - 1) - loop.index0 %}
3880 {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
3881 {%- set ns.multi_step_tool = false %}
3882 {%- set ns.last_query_index = index %}
3883 {%- endif %}
3884{%- endfor %}
3885{%- for message in messages %}
3886 {%- if message.content is string %}
3887 {%- set content = message.content %}
3888 {%- else %}
3889 {%- set content = '' %}
3890 {%- endif %}
3891 {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
3892 {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
3893 {%- elif message.role == "assistant" %}
3894 {%- set reasoning_content = '' %}
3895 {%- if message.reasoning_content is string %}
3896 {%- set reasoning_content = message.reasoning_content %}
3897 {%- else %}
3898 {%- if '</think>' in content %}
3899 {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
3900 {%- set content = content.split('</think>')[-1].lstrip('\n') %}
3901 {%- endif %}
3902 {%- endif %}
3903 {%- if loop.index0 > ns.last_query_index %}
3904 {%- if loop.last or (not loop.last and reasoning_content) %}
3905 {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
3906 {%- else %}
3907 {{- '<|im_start|>' + message.role + '\n' + content }}
3908 {%- endif %}
3909 {%- else %}
3910 {{- '<|im_start|>' + message.role + '\n' + content }}
3911 {%- endif %}
3912 {%- if message.tool_calls %}
3913 {%- for tool_call in message.tool_calls %}
3914 {%- if (loop.first and content) or (not loop.first) %}
3915 {{- '\n' }}
3916 {%- endif %}
3917 {%- if tool_call.function %}
3918 {%- set tool_call = tool_call.function %}
3919 {%- endif %}
3920 {{- '<tool_call>\n{"name": "' }}
3921 {{- tool_call.name }}
3922 {{- '", "arguments": ' }}
3923 {%- if tool_call.arguments is string %}
3924 {{- tool_call.arguments }}
3925 {%- else %}
3926 {{- tool_call.arguments | tojson }}
3927 {%- endif %}
3928 {{- '}\n</tool_call>' }}
3929 {%- endfor %}
3930 {%- endif %}
3931 {{- '<|im_end|>\n' }}
3932 {%- elif message.role == "tool" %}
3933 {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
3934 {{- '<|im_start|>user' }}
3935 {%- endif %}
3936 {{- '\n<tool_response>\n' }}
3937 {{- content }}
3938 {{- '\n</tool_response>' }}
3939 {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
3940 {{- '<|im_end|>\n' }}
3941 {%- endif %}
3942 {%- endif %}
3943{%- endfor %}
3944{%- if add_generation_prompt %}
3945 {{- '<|im_start|>assistant\n<think>\n' }}
3946{%- endif %}"##;
3947
3948 fn qwen3_thinking_formatter() -> HfTokenizerConfigJsonFormatter {
3949 let chat_template: ChatTemplate = serde_json::from_value(serde_json::json!({
3950 "chat_template": QWEN3_THINKING_TEMPLATE,
3951 }))
3952 .unwrap();
3953 HfTokenizerConfigJsonFormatter::new(chat_template, ContextMixins::new(&[])).unwrap()
3954 }
3955
3956 #[test]
3957 fn test_qwen3_thinking_template_flags_detected() {
3958 let formatter = qwen3_thinking_formatter();
3959 assert!(
3960 formatter.tool_use_template_handles_reasoning,
3961 "template references reasoning_content directly"
3962 );
3963 assert!(
3966 formatter.default_template_handles_tool_calls_arguments_string,
3967 "default template branches on `arguments is string`"
3968 );
3969 assert!(
3970 formatter.tool_use_template_handles_tool_calls_arguments_string,
3971 "tool_use template branches on `arguments is string`"
3972 );
3973 }
3974
3975 const QWEN38_REJECTS_STRING_ARGS_TEMPLATE: &str = r##"{%- for message in messages %}
3979 {%- if message.role == "assistant" and message.tool_calls %}
3980 {%- for tool_call in message.tool_calls %}
3981 {%- if tool_call.function %}
3982 {%- set tool_call = tool_call.function %}
3983 {%- endif %}
3984 {{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
3985 {%- if tool_call.arguments is mapping %}
3986 {%- for args_name, args_value in tool_call.arguments|items %}
3987 {{- '<parameter=' + args_name + '>\n' + args_value + '\n</parameter>\n' }}
3988 {%- endfor %}
3989 {%- elif tool_call.arguments is string %}
3990 {%- if tool_call.arguments|trim %}
3991 {{- raise_exception('Tool call arguments were passed as a JSON string.') }}
3992 {%- endif %}
3993 {%- endif %}
3994 {{- '</function>\n</tool_call>' }}
3995 {%- endfor %}
3996 {%- else %}
3997 {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>\n' }}
3998 {%- endif %}
3999{%- endfor %}"##;
4000
4001 #[test]
4005 fn test_template_rejecting_string_arguments_gets_objects() {
4006 let chat_template: ChatTemplate = serde_json::from_value(serde_json::json!({
4007 "chat_template": QWEN38_REJECTS_STRING_ARGS_TEMPLATE,
4008 }))
4009 .unwrap();
4010 let formatter =
4011 HfTokenizerConfigJsonFormatter::new(chat_template, ContextMixins::new(&[])).unwrap();
4012 assert!(!formatter.default_template_handles_tool_calls_arguments_string);
4013 assert!(!formatter.tool_use_template_handles_tool_calls_arguments_string);
4014
4015 let request: NvCreateChatCompletionRequest = serde_json::from_value(serde_json::json!({
4016 "model": "qwen3.8",
4017 "messages": [
4018 {"role": "user", "content": "What's the weather in San Francisco?"},
4019 {"role": "assistant", "content": "", "tool_calls": [{
4020 "id": "call_sf",
4021 "type": "function",
4022 "function": {"name": "get_weather", "arguments": "{\"location\": \"San Francisco\"}"}
4023 }]},
4024 {"role": "tool", "tool_call_id": "call_sf", "content": "Foggy"}
4025 ],
4026 }))
4027 .unwrap();
4028 let rendered = formatter.render(&request).unwrap();
4029 assert!(
4030 rendered.contains("<parameter=location>\nSan Francisco\n</parameter>"),
4031 "{rendered}"
4032 );
4033 }
4034
4035 #[test]
4045 fn test_qwen3_thinking_append_only_across_tool_use_turn() {
4046 let formatter = qwen3_thinking_formatter();
4047
4048 let tools = serde_json::json!([{
4049 "type": "function",
4050 "function": {
4051 "name": "get_weather",
4052 "description": "Get the current weather for a location",
4053 "parameters": {
4054 "type": "object",
4055 "properties": {
4056 "location": {"type": "string"},
4057 "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}
4058 },
4059 "required": ["location"]
4060 }
4061 }
4062 }]);
4063
4064 let turn1_request: NvCreateChatCompletionRequest =
4066 serde_json::from_value(serde_json::json!({
4067 "model": "qwen3-thinking",
4068 "messages": [
4069 {"role": "system", "content": "You are a helpful assistant."},
4070 {"role": "user", "content": "What's the weather in San Francisco?"},
4071 ],
4072 "tools": tools,
4073 }))
4074 .unwrap();
4075 let p1 = formatter.render(&turn1_request).unwrap();
4076
4077 let model_emitted = "I'll call get_weather for SF.\n\
4081 </think>\n\n\
4082 <tool_call>\n\
4083 {\"name\": \"get_weather\", \"arguments\": {\"location\": \"San Francisco\", \"unit\": \"celsius\"}}\n\
4084 </tool_call><|im_end|>\n";
4085 let wire_after_t1 = format!("{p1}{model_emitted}");
4086
4087 let turn2_request: NvCreateChatCompletionRequest =
4090 serde_json::from_value(serde_json::json!({
4091 "model": "qwen3-thinking",
4092 "messages": [
4093 {"role": "system", "content": "You are a helpful assistant."},
4094 {"role": "user", "content": "What's the weather in San Francisco?"},
4095 {
4096 "role": "assistant",
4097 "content": "",
4098 "reasoning_content": "I'll call get_weather for SF.",
4099 "tool_calls": [{
4100 "id": "call_sf",
4101 "type": "function",
4102 "function": {
4103 "name": "get_weather",
4104 "arguments": "{\"location\": \"San Francisco\", \"unit\": \"celsius\"}"
4105 }
4106 }]
4107 },
4108 {
4109 "role": "tool",
4110 "tool_call_id": "call_sf",
4111 "content": "{\"temp\": 18, \"conditions\": \"Foggy\"}"
4112 }
4113 ],
4114 "tools": tools,
4115 }))
4116 .unwrap();
4117 let p2 = formatter.render(&turn2_request).unwrap();
4118
4119 if !p2.starts_with(&wire_after_t1) {
4120 let div = wire_after_t1
4122 .as_bytes()
4123 .iter()
4124 .zip(p2.as_bytes())
4125 .position(|(a, b)| a != b)
4126 .unwrap_or_else(|| wire_after_t1.len().min(p2.len()));
4127 let lo = div.saturating_sub(40);
4128 panic!(
4129 "turn-2 prompt is NOT a prefix-extension of [turn-1 + model bytes]\n \
4130 diverges at byte {div}\n \
4131 wire ends: ...{}|{}\n \
4132 t2 has: ...{}|{}",
4133 String::from_utf8_lossy(&wire_after_t1.as_bytes()[lo..div]),
4134 String::from_utf8_lossy(
4135 &wire_after_t1.as_bytes()[div..(div + 60).min(wire_after_t1.len())]
4136 ),
4137 String::from_utf8_lossy(&p2.as_bytes()[lo..div]),
4138 String::from_utf8_lossy(&p2.as_bytes()[div..(div + 60).min(p2.len())]),
4139 );
4140 }
4141
4142 let suffix = &p2[wire_after_t1.len()..];
4145 assert!(
4146 suffix.contains("<tool_response>"),
4147 "appended bytes must include the tool response, got: {suffix}"
4148 );
4149 assert!(
4150 suffix.ends_with("<|im_start|>assistant\n<think>\n"),
4151 "appended bytes must end with the next generation prompt, got: {suffix}"
4152 );
4153 }
4154
4155 #[test]
4159 fn test_qwen3_thinking_renders_reasoning_content_segments_as_string() {
4160 let formatter = qwen3_thinking_formatter();
4161 assert!(formatter.default_template_requires_reasoning_string);
4162 assert!(formatter.tool_use_template_requires_reasoning_string);
4163
4164 for render in [render_shape, render_shape_with_tools] {
4165 let segments = json!(["Check both.", "Then add.", ""]);
4166 let rendered = render(&formatter, reasoning_tool_call_turn(segments)).unwrap();
4167 assert!(
4168 rendered.contains(
4169 "<|im_start|>assistant\n<think>\nCheck both.\nThen add.\n</think>\n\n<tool_call>"
4170 ),
4171 "{rendered}"
4172 );
4173 let string = json!("Check both.\nThen add.");
4174 assert_eq!(
4175 rendered,
4176 render(&formatter, reasoning_tool_call_turn(string)).unwrap()
4177 );
4178 }
4179 }
4180
4181 fn contents(messages: &serde_json::Value) -> serde_json::Value {
4184 messages
4185 .as_array()
4186 .unwrap()
4187 .iter()
4188 .map(|m| m["content"].clone())
4189 .collect()
4190 }
4191
4192 #[test]
4194 fn content_format_string_flattens_like_vllm() {
4195 let placeholders = MediaPlaceholders {
4196 image: "<|image_{n}|>".into(),
4197 video: "<video>".into(),
4198 audio: "<|audio_{n}|>".into(),
4199 };
4200 let image = json!({"type": "image_url", "image_url": {"url": "x"}});
4201 let flatten = |mut messages: serde_json::Value, use_audio_in_video| {
4202 flatten_msg_content_to_string(&mut messages, &placeholders, use_audio_in_video)
4203 .map(|()| contents(&messages))
4204 };
4205
4206 let messages = json!([
4208 {"role": "user", "content": [
4209 {"type": "text", "text": "a"}, &image,
4210 {"type": "video_url", "video_url": {"url": "y"}},
4211 {"type": "text", "text": "b"}, &image
4212 ]},
4213 {"role": "user", "content": [&image]}
4214 ]);
4215 assert_eq!(
4216 flatten(messages, false).unwrap(),
4217 json!(["<|image_1|>\n<|image_2|>\n<video>\na\nb", "<|image_3|>"])
4218 );
4219 let messages = json!([
4221 {"role": "system", "content": "plain"},
4222 {"role": "assistant"},
4223 {"role": "assistant", "content": [
4224 {"type": "text", "text": "a"},
4225 {"type": "text", "text": ""},
4226 {"type": "refusal", "refusal": ""},
4227 {"type": "refusal", "refusal": "b"}
4228 ]}
4229 ]);
4230 assert_eq!(
4231 flatten(messages, false).unwrap(),
4232 json!(["plain", "", "a\nb"])
4233 );
4234 let messages = json!([{"role": "user", "content": [
4236 {"type": "text", "text": "see <|image_1|> here"}, &image, &image
4237 ]}]);
4238 assert_eq!(
4239 flatten(messages, false).unwrap(),
4240 json!(["<|image_2|>\nsee <|image_1|> here"])
4241 );
4242 let messages = json!([{"role": "user", "content": [
4244 {"type": "video_url", "video_url": {"url": "y"}},
4245 {"type": "video_url", "video_url": null, "uuid": "cached"},
4246 {"type": "video_url", "video_url": {"url": ""}, "uuid": "cached"},
4247 {"type": "input_audio", "input_audio": {"data": "", "format": "wav"}},
4248 {"type": "text", "text": "x"}
4249 ]}]);
4250 assert_eq!(
4251 flatten(messages, true).unwrap(),
4252 json!(["<video>\n<video>\n<video>\n<|audio_1|>\n<|audio_2|>\nx"])
4253 );
4254
4255 for content in [
4256 json!([{"type": "text", "text": "<|image_1|><|image_1|>"}, &image]),
4257 json!([{"type": "custom", "kind": "chemistry", "fields": {}}]),
4258 ] {
4259 let error = flatten(json!([{"role": "user", "content": content}]), false).unwrap_err();
4260 assert!(
4261 matches!(
4262 error.downcast_ref::<PromptRenderError>(),
4263 Some(PromptRenderError::InvalidRequest(_))
4264 ),
4265 "{error}"
4266 );
4267 }
4268 }
4269
4270 #[test]
4272 fn content_format_openai_wraps_content_in_parts_like_vllm() {
4273 let mut messages = json!([
4274 {"role": "system", "content": "sys"},
4275 {"role": "user", "content": [
4276 {"type": "text", "text": "look", "source_lang_code": "en"},
4277 {"type": "image_url", "image_url": {"url": "x"}, "uuid": "i", "target_lang_code": "de"},
4278 {"type": "input_audio", "input_audio": {"data": "", "format": "wav"}},
4279 {"type": "custom", "kind": "chemistry", "fields": {"data_base64": "x"}}
4280 ]},
4281 {"role": "assistant", "content": [{"type": "refusal", "refusal": "no"}]},
4282 {"role": "assistant"},
4283 {"role": "tool", "tool_call_id": "1", "content": [
4284 {"type": "text", "text": "a"},
4285 {"type": "text", "text": "b"}
4286 ]},
4287 {"role": "tool", "tool_call_id": "2", "content": "done"}
4288 ]);
4289 wrap_msg_content_in_parts(&mut messages);
4290 assert_eq!(
4291 contents(&messages),
4292 json!([
4293 [{"type": "text", "text": "sys"}],
4294 [
4295 {"type": "text", "text": "look", "source_lang_code": "en"},
4296 {"type": "image", "target_lang_code": "de"},
4297 {"type": "audio"},
4298 {"type": "custom", "kind": "chemistry"}
4299 ],
4300 [{"type": "text", "text": "no"}],
4301 [],
4302 "a\nb",
4303 "done",
4304 ])
4305 );
4306 }
4307
4308 #[test]
4310 fn content_format_reaches_render_through_from_parts() {
4311 let template = "{% for m in messages %}{% if m.content is string %}{{ m.content }}\
4312 {% else %}{% for p in m.content %}[{{ p.type }}{% if p.text %} {{ p.text }}{% endif %}]\
4313 {% endfor %}{% endif %}|{% endfor %}";
4314 let request: NvCreateChatCompletionRequest = serde_json::from_value(json!({
4315 "model": "test",
4316 "messages": [
4317 {"role": "system", "content": "sys"},
4318 {"role": "user", "content": [
4319 {"type": "text", "text": "hi"},
4320 {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}},
4321 {"type": "video_url", "video_url": {"url": "https://example.com/v.mp4"}}
4322 ]}
4323 ],
4324 "mm_processor_kwargs": {"use_audio_in_video": 1}
4326 }))
4327 .unwrap();
4328 let render = |content_format| {
4329 let config: SysChatTemplate =
4330 serde_json::from_value(json!({ "chat_template": template })).unwrap();
4331 let crate::PromptFormatter::OAI(formatter) =
4332 crate::PromptFormatter::from_parts_with_content_format(
4333 config,
4334 SysMixins::new(&[]),
4335 true,
4336 content_format,
4337 )
4338 .unwrap();
4339 formatter.render(&request).unwrap()
4340 };
4341
4342 assert_eq!(
4343 render(ChatTemplateContentFormat::Auto),
4344 "sys|[text hi][image][video]|"
4345 );
4346 assert_eq!(
4347 render(ChatTemplateContentFormat::OpenAi),
4348 "[text sys]|[text hi][image][video]|"
4349 );
4350 let placeholders = MediaPlaceholders {
4351 image: "<image>".into(),
4352 video: "<video>".into(),
4353 audio: "<audio>".into(),
4354 };
4355 assert_eq!(
4356 render(ChatTemplateContentFormat::String(placeholders)),
4357 "sys|<image>\n<video>\n<audio>\nhi|"
4358 );
4359 }
4360}