dynamo_renderer/template.rs
1// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4use std::{collections::HashSet, sync::Arc};
5
6use anyhow::{Ok, Result};
7use minijinja::Environment;
8
9use super::PromptContextMixin;
10
11mod context;
12mod formatters;
13mod minimax_m2;
14mod oai;
15mod tokcfg;
16
17use super::{OAIPromptFormatter, PromptFormatter};
18pub use oai::may_be_fix_tool_schema;
19pub use tokcfg::{ChatTemplate, ChatTemplateValue};
20
21/// If the model is a DeepSeek family whose HF repo doesn't ship a Jinja
22/// `chat_template`, return the native Rust formatter for it. Returns `None`
23/// for everything else (the caller then loads the HF `tokenizer_config.json`
24/// template via [`PromptFormatter::from_parts`]).
25///
26/// `model_type_lower` is the lowercased `config.json` `model_type` (authoritative,
27/// survives `--served-model-name` renames); `display_name_lower` is the
28/// lowercased served name, used only as a fallback when `model_type` is absent.
29pub fn deepseek_formatter_for(
30 model_type_lower: &Option<String>,
31 display_name_lower: &str,
32) -> Option<PromptFormatter> {
33 if is_deepseek_v41(model_type_lower, display_name_lower) {
34 return Some(PromptFormatter::OAI(Arc::new(
35 super::deepseek::v41::DeepSeekV41Formatter,
36 )));
37 }
38 if is_deepseek_v4(model_type_lower, display_name_lower) {
39 tracing::info!(
40 model_type = ?model_type_lower,
41 display_name = %display_name_lower,
42 "Detected DeepSeek V4 model, using native Rust formatter",
43 );
44 return Some(PromptFormatter::OAI(Arc::new(
45 super::deepseek::v4::DeepSeekV4Formatter::new_thinking(),
46 )));
47 }
48 if is_deepseek_v3_2_non_exp(model_type_lower, display_name_lower) {
49 tracing::info!("Detected DeepSeek V3.2 model (non-Exp), using native Rust formatter");
50 return Some(PromptFormatter::OAI(Arc::new(
51 super::deepseek::v32::DeepSeekV32Formatter::new_thinking(),
52 )));
53 }
54 None
55}
56
57/// If the model is Kimi K3, return its native XTML formatter. K3 ships no
58/// Jinja chat template and must preserve special-vs-ordinary segment boundaries
59/// until tokenization.
60pub fn kimi_k3_formatter_for(
61 model_type_lower: &Option<String>,
62 display_name_lower: &str,
63 exclude_tools_when_tool_choice_none: bool,
64) -> Option<PromptFormatter> {
65 if !is_kimi_k3(model_type_lower, display_name_lower) {
66 return None;
67 }
68
69 tracing::info!(
70 model_type = ?model_type_lower,
71 display_name = %display_name_lower,
72 "Detected Kimi K3 model, using native Rust XTML formatter",
73 );
74 Some(PromptFormatter::OAI(Arc::new(
75 super::kimi_k3::KimiK3Formatter::new(exclude_tools_when_tool_choice_none),
76 )))
77}
78
79fn is_kimi_k3(model_type_lower: &Option<String>, display_name_lower: &str) -> bool {
80 match model_type_lower.as_deref() {
81 Some("kimi_k3") => true,
82 Some(_) => false,
83 None => ["kimi-k3", "kimi_k3", "kimik3"]
84 .iter()
85 .any(|needle| display_name_lower.contains(needle)),
86 }
87}
88
89/// Select a native formatter for model families that do not ship a usable HF
90/// `chat_template`.
91///
92/// Inkling is selected only from the authoritative `config.json` model type;
93/// unlike display-name substring matching, this remains stable under
94/// `--served-model-name` aliases. DeepSeek keeps its existing fallback for
95/// older model cards that do not publish `model_type`.
96pub fn native_formatter_for(
97 model_type_lower: &Option<String>,
98 display_name_lower: &str,
99) -> Option<PromptFormatter> {
100 if model_type_lower.as_deref() == Some("inkling_mm_model") {
101 tracing::info!(
102 model_type = ?model_type_lower,
103 "Detected Inkling model, using native Rust formatter",
104 );
105 return Some(PromptFormatter::OAI(Arc::new(
106 super::inkling::InklingFormatter,
107 )));
108 }
109
110 deepseek_formatter_for(model_type_lower, display_name_lower)
111}
112
113/// How message `content` is shaped before the HF chat template sees it, with
114/// the semantics of vLLM's `--chat-template-content-format`.
115#[derive(Debug, Clone, Default, PartialEq, Eq)]
116pub enum ChatTemplateContentFormat {
117 /// Probe the template at load and pick a shape for it (Dynamo's default).
118 #[default]
119 Auto,
120 /// Flatten every message's content to one string, as vLLM's `string`
121 /// format does with its default `interleave_mm_strings=False`: one
122 /// placeholder per media item first, then the text parts, all joined
123 /// with `\n`.
124 String(MediaPlaceholders),
125 /// Pass every message's content as a list of parts, as vLLM's `openai`
126 /// format does.
127 OpenAi,
128}
129
130/// The model's placeholder for each modality, as vLLM's `get_placeholder_str`
131/// returns it. `{n}` is replaced with the item's 1-based index among that
132/// modality's items in the whole conversation. An empty placeholder adds none.
133#[derive(Debug, Clone, Default, PartialEq, Eq)]
134pub struct MediaPlaceholders {
135 pub image: String,
136 pub video: String,
137 pub audio: String,
138}
139
140impl PromptFormatter {
141 pub fn from_parts(
142 config: ChatTemplate,
143 context: ContextMixins,
144 exclude_tools_when_tool_choice_none: bool,
145 ) -> Result<PromptFormatter> {
146 Self::from_parts_with_content_format(
147 config,
148 context,
149 exclude_tools_when_tool_choice_none,
150 ChatTemplateContentFormat::Auto,
151 )
152 }
153
154 /// [`PromptFormatter::from_parts`] with an explicit content format.
155 pub fn from_parts_with_content_format(
156 config: ChatTemplate,
157 context: ContextMixins,
158 exclude_tools_when_tool_choice_none: bool,
159 content_format: ChatTemplateContentFormat,
160 ) -> Result<PromptFormatter> {
161 let mut formatter = HfTokenizerConfigJsonFormatter::with_options(
162 config,
163 context,
164 exclude_tools_when_tool_choice_none,
165 )?;
166 // Like vLLM, warn when the forced format disagrees with the template.
167 if matches!(content_format, ChatTemplateContentFormat::String(_))
168 && formatter.requires_content_arrays
169 {
170 tracing::warn!(
171 "chat template content format is `string`, but the template renders text only \
172 from content arrays, so message text may be dropped"
173 );
174 }
175 formatter.content_format = content_format;
176 Ok(Self::OAI(Arc::new(formatter)))
177 }
178}
179
180/// Chat Template Jinja Renderer
181///
182/// Manages a Jinja environment with registered templates for chat formatting.
183/// Handles two types of ChatTemplateValue templates:
184///
185/// 1. String template: Registered as the 'default' template
186/// 2. Map template: Contains 'tool_use' and/or 'default' templates
187/// - tool_use: Template for tool-based interactions
188/// - default: Template for standard chat interactions
189///
190/// If the map contains both keys, the `tool_use` template is registered as the `tool_use` template
191/// and the `default` template is registered as the `default` template.
192struct JinjaEnvironment {
193 env: Environment<'static>,
194}
195
196/// Which message-shape restrictions one chat template enforces, probed once at
197/// load. Each rewrite in `normalize_system_messages` is gated on its own flag:
198/// the restrictions are independent, so a template that rejects a non-leading
199/// `system` (Qwen3.5) but accepts consecutive `user` turns keeps those turns
200/// separate instead of being reshaped for a rule it does not have.
201#[derive(Debug, Default, Clone, Copy)]
202struct SystemNormalization {
203 /// Template raises on a `system` turn that is not first.
204 demote_nonleading_system: bool,
205 /// Template raises on two adjacent `user` turns.
206 coalesce_consecutive_users: bool,
207}
208
209impl SystemNormalization {
210 fn is_required(&self) -> bool {
211 self.demote_nonleading_system || self.coalesce_consecutive_users
212 }
213}
214
215/// Formatter for HuggingFace tokenizer config JSON templates
216///
217/// Implements chat template rendering based on HuggingFace's tokenizer_config.json format.
218/// Supports:
219/// - Tool usage templates
220/// - Generation prompts
221/// - Context mixins for template customization
222#[derive(Debug)]
223struct HfTokenizerConfigJsonFormatter {
224 env: Environment<'static>,
225 config: ChatTemplate,
226 mixins: Arc<ContextMixins>,
227 supports_add_generation_prompt: bool,
228 /// How message content is shaped for the template. `Auto` uses
229 /// `requires_content_arrays`, per-role text-array capabilities, and
230 /// `image_placeholder_template`.
231 content_format: ChatTemplateContentFormat,
232 requires_content_arrays: bool,
233 /// Roles whose text parts the selected template renders without printing
234 /// their containers. Keep these arrays so the template owns separators.
235 default_native_text_array_roles: Vec<&'static str>,
236 tool_use_native_text_array_roles: Vec<&'static str>,
237 /// When true, strip tool definitions from the chat template when tool_choice is "none".
238 /// This prevents models from generating raw XML tool calls in the content field.
239 exclude_tools_when_tool_choice_none: bool,
240 /// Cached per selected template variant; native namespace rendering was
241 /// introduced by the template's `deferred_render_mode` argument.
242 /// A capable default must not opt an older tool-use variant into native tools.
243 default_template_handles_native_tools: bool,
244 tool_use_template_handles_native_tools: bool,
245 /// True if the `default` template natively references `reasoning_content`.
246 /// When true and rendering through `default`, skip injection — the template
247 /// handles it. Tracked separately for `default` and `tool_use` because HF
248 /// configs may register different sources for each: Gemma4's `tool_use`
249 /// template is adapted by `normalize_chat_template_source` to read
250 /// `reasoning_content`, while its `default` template is not. A single global
251 /// flag would wrongly suppress injection on the untouched `default` path and
252 /// silently drop prior assistant reasoning on no-tool renders.
253 default_template_handles_reasoning: bool,
254 /// True if the `tool_use` template natively references `reasoning_content`.
255 /// See `default_template_handles_reasoning` for rationale.
256 tool_use_template_handles_reasoning: bool,
257 /// True if the `default` template renders a string `reasoning_content` but
258 /// not the segment array sent with interleaved tool calls (MiniMax-M2 and
259 /// Qwen3 read it only when it `is string`). When true and rendering through
260 /// `default`, join the segments into the string the template expects.
261 default_template_requires_reasoning_string: bool,
262 /// See `default_template_requires_reasoning_string` for rationale.
263 tool_use_template_requires_reasoning_string: bool,
264 /// Per-family placeholder template for image content parts when flattening
265 /// mixed text+image content arrays into a single string (`preserve_arrays`
266 /// = false path). `{n}` in the template is substituted with the 1-based
267 /// image index. `None` when the model's chat template handles content
268 /// arrays natively (Qwen-VL family) or when we have no flatten strategy
269 /// for it (no MM-aware routing benefit either way).
270 image_placeholder_template: Option<&'static str>,
271 /// True if the `default` template branches on `tool_call.arguments is string`
272 /// and renders string arguments verbatim (Qwen3, Hermes, etc.; templates that
273 /// only use the test to reject strings don't count). When true and rendering
274 /// through `default`, skip pre-parsing the JSON-string
275 /// `tool_calls[].function.arguments` into an object — the template wants the
276 /// raw string verbatim. Pre-parsing forces the `tojson`-with-object branch and
277 /// re-emits with minijinja's compact separators, which breaks append-only
278 /// prefix matching across multi-step tool-use turns. Tracked separately for
279 /// `default` and `tool_use` because HF configs may register different sources
280 /// for each, and because `arguments is string` is tool_calls-specific — legacy
281 /// `function_call.arguments` lives outside that branch and is still
282 /// normalized unconditionally.
283 default_template_handles_tool_calls_arguments_string: bool,
284 /// True if the `tool_use` template branches on `tool_call.arguments is string`.
285 /// See `default_template_handles_tool_calls_arguments_string` for rationale.
286 tool_use_template_handles_tool_calls_arguments_string: bool,
287 /// Message-shape restrictions the `default` template enforces.
288 default_system_normalization: SystemNormalization,
289 /// Message-shape restrictions the `tool_use` template enforces.
290 /// Kept separate because dict-form HF configs may register templates with
291 /// different constraints.
292 tool_use_system_normalization: SystemNormalization,
293}
294
295// /// OpenAI Standard Prompt Formatter
296// pub trait StandardPromptFormatter {
297// fn render(&self, context: &impl StandardPromptContext) -> Result<String>;
298// }
299
300// pub trait StandardPromptContext {
301// fn messages(&self) -> Value;
302// fn tools(&self) -> Option<Value>;
303// }
304
305#[derive(Debug, Clone, Default)]
306pub struct ContextMixins {
307 context_mixins: HashSet<PromptContextMixin>,
308}
309
310fn is_deepseek_v41(model_type_lower: &Option<String>, display_name_lower: &str) -> bool {
311 match model_type_lower.as_deref() {
312 Some("deepseek_v41") => true,
313 Some(_) => false,
314 None => [
315 "deepseek-v4.1",
316 "deepseek_v4.1",
317 "deepseek.v4.1",
318 "deepseekv4.1",
319 ]
320 .iter()
321 .any(|prefix| {
322 display_name_lower
323 .strip_prefix(prefix)
324 .is_some_and(|suffix| suffix.is_empty() || suffix.starts_with(['-', '_', '.']))
325 }),
326 }
327}
328
329/// Decides whether to activate the DeepSeek-V4 native formatter.
330///
331/// Primary signal: config.json `model_type`. DeepSeek-V4-Pro and V4-Flash both
332/// ship `"model_type": "deepseek_v4"`, set by the model author — this survives
333/// any `--served-model-name` rename.
334///
335/// Fallback: `display_name`, tight-matched against
336/// `^deepseek(?:[-_.])?v4(?:[-_.]|$)`. Only consulted when config.json is
337/// absent (tokenizer-only MDCs) or unreadable; a concrete config.json value
338/// that is *not* `deepseek_v4` is authoritative and suppresses the fallback.
339fn is_deepseek_v4(model_type_lower: &Option<String>, display_name_lower: &str) -> bool {
340 match model_type_lower.as_deref() {
341 Some("deepseek_v4") => true,
342 Some(_) => false, // config.json says something else — trust it
343 None => is_deepseek_v4_name(display_name_lower),
344 }
345}
346
347/// Decides whether to activate the DeepSeek-V3.2 (non-Exp) native formatter.
348/// Same config-primary / name-fallback rule as V4.
349fn is_deepseek_v3_2_non_exp(model_type_lower: &Option<String>, display_name_lower: &str) -> bool {
350 let name_match = display_name_lower.contains("deepseek")
351 && display_name_lower.contains("v3.2")
352 && !display_name_lower.contains("exp");
353 match model_type_lower.as_deref() {
354 // HF ships `deepseek_v32` (no underscore between 3 and 2); Dynamo's
355 // internal/tool-parser key is `deepseek_v3_2`. Accept both.
356 Some("deepseek_v3_2" | "deepseek_v32") => !display_name_lower.contains("exp"),
357 Some(_) => false,
358 None => name_match,
359 }
360}
361
362/// Tight, anchored match for DeepSeek-V4 display names. Equivalent to the
363/// regex `^deepseek(?:[-_.])?v4(?:[-_.]|$)` over an already-lowercased string.
364/// Written with string ops to avoid pulling in the `regex` crate.
365///
366/// Rejects composite names that previously short-circuited the V4 branch:
367/// - `deepseek-v3.2-v4-foo` (the `v3.2` variant is the real one)
368/// - `deepseek-v40` / `deepseek-v4pro` (no separator after `v4`)
369/// - `my-deepseek-v4` (prefix must be at the start)
370fn is_deepseek_v4_name(name_lower: &str) -> bool {
371 let Some(rest) = name_lower.strip_prefix("deepseek") else {
372 return false;
373 };
374 // Optional single separator between "deepseek" and "v4".
375 let rest = rest
376 .strip_prefix(|c: char| matches!(c, '-' | '_' | '.'))
377 .unwrap_or(rest);
378 let Some(after_v4) = rest.strip_prefix("v4") else {
379 return false;
380 };
381 // `v4` must end the name or be followed by a separator — anything else
382 // (e.g. `v40`, `v4pro`) is a different model family.
383 after_v4.is_empty() || after_v4.starts_with(['-', '_', '.'])
384}
385
386#[cfg(test)]
387mod detection_tests {
388 use super::{is_deepseek_v3_2_non_exp, is_deepseek_v4, is_deepseek_v4_name, is_kimi_k3};
389
390 #[test]
391 fn kimi_k3_detection_prefers_config_model_type() {
392 assert!(is_kimi_k3(&Some("kimi_k3".to_string()), "served-name"));
393 assert!(!is_kimi_k3(
394 &Some("kimi_k2".to_string()),
395 "moonshot-kimi-k3"
396 ));
397 assert!(is_kimi_k3(&None, "moonshot-kimi-k3"));
398 assert!(is_kimi_k3(&None, "kimi_k3-instruct"));
399 assert!(!is_kimi_k3(&None, "kimi-k2.5"));
400 }
401
402 #[test]
403 fn v4_name_matches_canonical_variants() {
404 for name in [
405 "deepseek-v4",
406 "deepseek_v4",
407 "deepseek.v4",
408 "deepseekv4",
409 "deepseek-v4-pro",
410 "deepseek-v4-flash",
411 "deepseek-v4-flash-2507",
412 "deepseek-v4.1",
413 "deepseek_v4_thinking",
414 ] {
415 assert!(is_deepseek_v4_name(name), "expected {name} to match V4");
416 }
417 }
418
419 #[test]
420 fn v4_name_rejects_non_v4() {
421 // Composite names that previously short-circuited to V4 before the
422 // V3.2 branch — now correctly rejected.
423 for name in [
424 "deepseek-v3.2-v4-foo",
425 "my-deepseek-v4",
426 "deepseek-v40",
427 "deepseek-v4pro",
428 "deepseekv40",
429 "deepseek-v3",
430 "deepseek-v3.2",
431 "deepseek-r1",
432 "qwen3-v4", // only deepseek-prefixed names qualify
433 "dsflash",
434 "",
435 ] {
436 assert!(
437 !is_deepseek_v4_name(name),
438 "expected {name} to NOT match V4",
439 );
440 }
441 }
442
443 #[test]
444 fn v4_detection_prefers_config_model_type() {
445 // config.json `model_type = "deepseek_v4"` wins regardless of what
446 // the operator calls the model via --served-model-name.
447 let v4 = Some("deepseek_v4".to_string());
448 for display in ["dsflash", "my-pet-model", "llama-3-8b", ""] {
449 assert!(
450 is_deepseek_v4(&v4, display),
451 "config says deepseek_v4, display {display:?} — expected V4",
452 );
453 }
454
455 // A concrete non-V4 config.json suppresses the display-name fallback.
456 // Even if the operator names the served model "deepseek-v4", a model
457 // with `model_type = "llama"` is NOT DeepSeek-V4.
458 let llama = Some("llama".to_string());
459 for display in ["deepseek-v4", "deepseek-v4-flash", "anything"] {
460 assert!(
461 !is_deepseek_v4(&llama, display),
462 "config says llama, display {display:?} — expected NOT V4",
463 );
464 }
465
466 // No config.json — fall back to display-name match.
467 assert!(is_deepseek_v4(&None, "deepseek-v4-flash"));
468 assert!(!is_deepseek_v4(&None, "dsflash"));
469
470 // A config.json with `"model_type": ""` is treated as "no signal" at
471 // the call site (normalized to None before is_deepseek_v4 is called),
472 // so the display-name fallback still runs — pin that contract.
473 let empty: Option<String> = None;
474 assert!(is_deepseek_v4(&empty, "deepseek-v4-flash"));
475 assert!(!is_deepseek_v4(&empty, "dsflash"));
476 }
477
478 #[test]
479 fn v3_2_detection_prefers_config_model_type() {
480 // config says deepseek_v3_2, any non-"exp" display name triggers.
481 let v3_2 = Some("deepseek_v3_2".to_string());
482 assert!(is_deepseek_v3_2_non_exp(&v3_2, "whatever"));
483 assert!(is_deepseek_v3_2_non_exp(&v3_2, "deepseek-v3.2"));
484 // V3.2-Exp is a separate model family; suppress even via config.
485 assert!(!is_deepseek_v3_2_non_exp(&v3_2, "deepseek-v3.2-exp"));
486
487 // The actual HF config.json spelling has no underscore between 3 and 2
488 // (`deepseek_v32`). It must trigger identically to the internal key.
489 let hf_real = Some("deepseek_v32".to_string());
490 assert!(is_deepseek_v3_2_non_exp(&hf_real, "whatever"));
491 assert!(is_deepseek_v3_2_non_exp(&hf_real, "deepseek-v3.2-nvfp4"));
492 assert!(!is_deepseek_v3_2_non_exp(&hf_real, "deepseek-v3.2-exp"));
493
494 // Other config types lose regardless of display name.
495 let other = Some("deepseek_v4".to_string());
496 assert!(!is_deepseek_v3_2_non_exp(&other, "deepseek-v3.2"));
497
498 // No config — fall back to the original display-name heuristic.
499 assert!(is_deepseek_v3_2_non_exp(&None, "deepseek-v3.2-pro"));
500 assert!(!is_deepseek_v3_2_non_exp(&None, "deepseek-v3.2-exp"));
501 assert!(!is_deepseek_v3_2_non_exp(&None, "deepseek-v4"));
502 }
503}