Skip to main content

dynamo_renderer/
template.rs

1// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4use std::{collections::HashSet, sync::Arc};
5
6use anyhow::{Ok, Result};
7use minijinja::Environment;
8
9use super::PromptContextMixin;
10
11mod context;
12mod formatters;
13mod minimax_m2;
14mod oai;
15mod tokcfg;
16
17use super::{OAIPromptFormatter, PromptFormatter};
18pub use oai::may_be_fix_tool_schema;
19pub use tokcfg::{ChatTemplate, ChatTemplateValue};
20
21/// If the model is a DeepSeek family whose HF repo doesn't ship a Jinja
22/// `chat_template`, return the native Rust formatter for it. Returns `None`
23/// for everything else (the caller then loads the HF `tokenizer_config.json`
24/// template via [`PromptFormatter::from_parts`]).
25///
26/// `model_type_lower` is the lowercased `config.json` `model_type` (authoritative,
27/// survives `--served-model-name` renames); `display_name_lower` is the
28/// lowercased served name, used only as a fallback when `model_type` is absent.
29pub fn deepseek_formatter_for(
30    model_type_lower: &Option<String>,
31    display_name_lower: &str,
32) -> Option<PromptFormatter> {
33    if is_deepseek_v41(model_type_lower, display_name_lower) {
34        return Some(PromptFormatter::OAI(Arc::new(
35            super::deepseek::v41::DeepSeekV41Formatter,
36        )));
37    }
38    if is_deepseek_v4(model_type_lower, display_name_lower) {
39        tracing::info!(
40            model_type = ?model_type_lower,
41            display_name = %display_name_lower,
42            "Detected DeepSeek V4 model, using native Rust formatter",
43        );
44        return Some(PromptFormatter::OAI(Arc::new(
45            super::deepseek::v4::DeepSeekV4Formatter::new_thinking(),
46        )));
47    }
48    if is_deepseek_v3_2_non_exp(model_type_lower, display_name_lower) {
49        tracing::info!("Detected DeepSeek V3.2 model (non-Exp), using native Rust formatter");
50        return Some(PromptFormatter::OAI(Arc::new(
51            super::deepseek::v32::DeepSeekV32Formatter::new_thinking(),
52        )));
53    }
54    None
55}
56
57/// If the model is Kimi K3 or K3.1, return its native XTML formatter. Neither ships a
58/// Jinja chat template and must preserve special-vs-ordinary segment boundaries
59/// until tokenization.
60pub fn kimi_k3_formatter_for(
61    model_type_lower: &Option<String>,
62    display_name_lower: &str,
63    exclude_tools_when_tool_choice_none: bool,
64) -> Option<PromptFormatter> {
65    if !is_kimi_k3(model_type_lower, display_name_lower) {
66        return None;
67    }
68
69    // Configuration is authoritative; only infer the version from the display
70    // name when the model card does not provide a model type.
71    let is_k31 = match model_type_lower.as_deref() {
72        Some(model_type) => model_type == "kimi_k31",
73        None => [
74            "kimi-k3.1",
75            "kimi_k3.1",
76            "kimik3.1",
77            "kimi-k31",
78            "kimi_k31",
79            "kimik31",
80        ]
81        .iter()
82        .any(|needle| display_name_lower.contains(needle)),
83    };
84    let formatter = if is_k31 {
85        super::kimi_k3::KimiK3Formatter::new_k31(exclude_tools_when_tool_choice_none)
86    } else {
87        super::kimi_k3::KimiK3Formatter::new(exclude_tools_when_tool_choice_none)
88    };
89
90    tracing::info!(
91        model_type = ?model_type_lower,
92        display_name = %display_name_lower,
93        "Detected Kimi K3 family model, using native Rust XTML formatter",
94    );
95    Some(PromptFormatter::OAI(Arc::new(formatter)))
96}
97
98fn is_kimi_k3(model_type_lower: &Option<String>, display_name_lower: &str) -> bool {
99    match model_type_lower.as_deref() {
100        Some("kimi_k3" | "kimi_k31") => true,
101        Some(_) => false,
102        None => ["kimi-k3", "kimi_k3", "kimik3"]
103            .iter()
104            .any(|needle| display_name_lower.contains(needle)),
105    }
106}
107
108/// Select a native formatter for model families that do not ship a usable HF
109/// `chat_template`.
110///
111/// Inkling is selected only from the authoritative `config.json` model type;
112/// unlike display-name substring matching, this remains stable under
113/// `--served-model-name` aliases. DeepSeek keeps its existing fallback for
114/// older model cards that do not publish `model_type`.
115pub fn native_formatter_for(
116    model_type_lower: &Option<String>,
117    display_name_lower: &str,
118) -> Option<PromptFormatter> {
119    if model_type_lower.as_deref() == Some("inkling_mm_model") {
120        tracing::info!(
121            model_type = ?model_type_lower,
122            "Detected Inkling model, using native Rust formatter",
123        );
124        return Some(PromptFormatter::OAI(Arc::new(
125            super::inkling::InklingFormatter,
126        )));
127    }
128
129    deepseek_formatter_for(model_type_lower, display_name_lower)
130}
131
132/// How message `content` is shaped before the HF chat template sees it, with
133/// the semantics of vLLM's `--chat-template-content-format`.
134#[derive(Debug, Clone, Default, PartialEq, Eq)]
135pub enum ChatTemplateContentFormat {
136    /// Probe the template at load and pick a shape for it (Dynamo's default).
137    #[default]
138    Auto,
139    /// Flatten every message's content to one string, as vLLM's `string`
140    /// format does with its default `interleave_mm_strings=False`: one
141    /// placeholder per media item first, then the text parts, all joined
142    /// with `\n`.
143    String(MediaPlaceholders),
144    /// Pass every message's content as a list of parts, as vLLM's `openai`
145    /// format does.
146    OpenAi,
147}
148
149/// The model's placeholder for each modality, as vLLM's `get_placeholder_str`
150/// returns it. `{n}` is replaced with the item's 1-based index among that
151/// modality's items in the whole conversation. An empty placeholder adds none.
152#[derive(Debug, Clone, Default, PartialEq, Eq)]
153pub struct MediaPlaceholders {
154    pub image: String,
155    pub video: String,
156    pub audio: String,
157}
158
159impl PromptFormatter {
160    pub fn from_parts(
161        config: ChatTemplate,
162        context: ContextMixins,
163        exclude_tools_when_tool_choice_none: bool,
164    ) -> Result<PromptFormatter> {
165        Self::from_parts_with_content_format(
166            config,
167            context,
168            exclude_tools_when_tool_choice_none,
169            ChatTemplateContentFormat::Auto,
170        )
171    }
172
173    /// [`PromptFormatter::from_parts`] with an explicit content format.
174    pub fn from_parts_with_content_format(
175        config: ChatTemplate,
176        context: ContextMixins,
177        exclude_tools_when_tool_choice_none: bool,
178        content_format: ChatTemplateContentFormat,
179    ) -> Result<PromptFormatter> {
180        let mut formatter = HfTokenizerConfigJsonFormatter::with_options(
181            config,
182            context,
183            exclude_tools_when_tool_choice_none,
184        )?;
185        // Like vLLM, warn when the forced format disagrees with the template.
186        if matches!(content_format, ChatTemplateContentFormat::String(_))
187            && formatter.requires_content_arrays
188        {
189            tracing::warn!(
190                "chat template content format is `string`, but the template renders text only \
191                 from content arrays, so message text may be dropped"
192            );
193        }
194        formatter.content_format = content_format;
195        Ok(Self::OAI(Arc::new(formatter)))
196    }
197}
198
199/// Chat Template Jinja Renderer
200///
201/// Manages a Jinja environment with registered templates for chat formatting.
202/// Handles two types of ChatTemplateValue templates:
203///
204/// 1. String template: Registered as the 'default' template
205/// 2. Map template: Contains 'tool_use' and/or 'default' templates
206///    - tool_use: Template for tool-based interactions
207///    - default: Template for standard chat interactions
208///
209///   If the map contains both keys, the `tool_use` template is registered as the `tool_use` template
210///   and the `default` template is registered as the `default` template.
211struct JinjaEnvironment {
212    env: Environment<'static>,
213}
214
215/// Which message-shape restrictions one chat template enforces, probed once at
216/// load. Each rewrite in `normalize_system_messages` is gated on its own flag:
217/// the restrictions are independent, so a template that rejects a non-leading
218/// `system` (Qwen3.5) but accepts consecutive `user` turns keeps those turns
219/// separate instead of being reshaped for a rule it does not have.
220#[derive(Debug, Default, Clone, Copy)]
221struct SystemNormalization {
222    /// Template accepts `system` but rejects the equivalent `developer` turn.
223    fallback_developer: bool,
224    /// Template raises on a `system` turn that is not first.
225    demote_nonleading_system: bool,
226    /// Template raises on two adjacent `user` turns.
227    coalesce_consecutive_users: bool,
228}
229
230impl SystemNormalization {
231    fn is_required(&self) -> bool {
232        self.fallback_developer || self.demote_nonleading_system || self.coalesce_consecutive_users
233    }
234}
235
236/// Formatter for HuggingFace tokenizer config JSON templates
237///
238/// Implements chat template rendering based on HuggingFace's tokenizer_config.json format.
239/// Supports:
240/// - Tool usage templates
241/// - Generation prompts
242/// - Context mixins for template customization
243#[derive(Debug)]
244struct HfTokenizerConfigJsonFormatter {
245    env: Environment<'static>,
246    config: ChatTemplate,
247    /// The model's control-token spellings. They are shielded in request text
248    /// so only template-owned markers encode as control tokens, unless
249    /// [`ChatTemplate::with_request_control_tokens_as_text`] turns that off
250    /// for a template without Harmony framing.
251    control_markers: oai::ControlMarkers,
252    mixins: Arc<ContextMixins>,
253    supports_add_generation_prompt: bool,
254    /// How message content is shaped for the template. `Auto` uses
255    /// `requires_content_arrays`, per-role text-array capabilities, and
256    /// `image_placeholder_template`.
257    content_format: ChatTemplateContentFormat,
258    requires_content_arrays: bool,
259    /// Roles whose text parts the selected template renders without printing
260    /// their containers. Keep these arrays so the template owns separators.
261    default_native_text_array_roles: Vec<&'static str>,
262    tool_use_native_text_array_roles: Vec<&'static str>,
263    /// When true, strip tool definitions from the chat template when tool_choice is "none".
264    /// This prevents models from generating raw XML tool calls in the content field.
265    exclude_tools_when_tool_choice_none: bool,
266    /// Cached per selected template variant; native namespace rendering was
267    /// introduced by the template's `deferred_render_mode` argument.
268    /// A capable default must not opt an older tool-use variant into native tools.
269    default_template_handles_native_tools: bool,
270    tool_use_template_handles_native_tools: bool,
271    /// True if the `default` template natively references `reasoning_content`.
272    /// When true and rendering through `default`, skip injection — the template
273    /// handles it. Tracked separately for `default` and `tool_use` because HF
274    /// configs may register different sources for each: Gemma4's `tool_use`
275    /// template is adapted by `normalize_chat_template_source` to read
276    /// `reasoning_content`, while its `default` template is not. A single global
277    /// flag would wrongly suppress injection on the untouched `default` path and
278    /// silently drop prior assistant reasoning on no-tool renders.
279    default_template_handles_reasoning: bool,
280    /// True if the `tool_use` template natively references `reasoning_content`.
281    /// See `default_template_handles_reasoning` for rationale.
282    tool_use_template_handles_reasoning: bool,
283    /// True if the `default` template renders a string `reasoning_content` but
284    /// not the segment array sent with interleaved tool calls (MiniMax-M2 and
285    /// Qwen3 read it only when it `is string`). When true and rendering through
286    /// `default`, join the segments into the string the template expects.
287    default_template_requires_reasoning_string: bool,
288    /// See `default_template_requires_reasoning_string` for rationale.
289    tool_use_template_requires_reasoning_string: bool,
290    /// Per-family placeholder template for image content parts when flattening
291    /// mixed text+image content arrays into a single string (`preserve_arrays`
292    /// = false path). `{n}` in the template is substituted with the 1-based
293    /// image index. `None` when the model's chat template handles content
294    /// arrays natively (Qwen-VL family) or when we have no flatten strategy
295    /// for it (no MM-aware routing benefit either way).
296    image_placeholder_template: Option<&'static str>,
297    /// True if the `default` template branches on `tool_call.arguments is string`
298    /// and renders string arguments verbatim (Qwen3, Hermes, etc.; templates that
299    /// only use the test to reject strings don't count). When true and rendering
300    /// through `default`, skip pre-parsing the JSON-string
301    /// `tool_calls[].function.arguments` into an object — the template wants the
302    /// raw string verbatim. Pre-parsing forces the `tojson`-with-object branch and
303    /// re-emits with minijinja's compact separators, which breaks append-only
304    /// prefix matching across multi-step tool-use turns. Tracked separately for
305    /// `default` and `tool_use` because HF configs may register different sources
306    /// for each, and because `arguments is string` is tool_calls-specific — legacy
307    /// `function_call.arguments` lives outside that branch and is still
308    /// normalized unconditionally.
309    default_template_handles_tool_calls_arguments_string: bool,
310    /// True if the `tool_use` template branches on `tool_call.arguments is string`.
311    /// See `default_template_handles_tool_calls_arguments_string` for rationale.
312    tool_use_template_handles_tool_calls_arguments_string: bool,
313    /// Message-shape restrictions the `default` template enforces.
314    default_system_normalization: SystemNormalization,
315    /// Message-shape restrictions the `tool_use` template enforces.
316    /// Kept separate because dict-form HF configs may register templates with
317    /// different constraints.
318    tool_use_system_normalization: SystemNormalization,
319}
320
321// /// OpenAI Standard Prompt Formatter
322// pub trait StandardPromptFormatter {
323//     fn render(&self, context: &impl StandardPromptContext) -> Result<String>;
324// }
325
326// pub trait StandardPromptContext {
327//     fn messages(&self) -> Value;
328//     fn tools(&self) -> Option<Value>;
329// }
330
331#[derive(Debug, Clone, Default)]
332pub struct ContextMixins {
333    context_mixins: HashSet<PromptContextMixin>,
334}
335
336fn is_deepseek_v41(model_type_lower: &Option<String>, display_name_lower: &str) -> bool {
337    match model_type_lower.as_deref() {
338        Some("deepseek_v41") => true,
339        Some(_) => false,
340        None => [
341            "deepseek-v4.1",
342            "deepseek_v4.1",
343            "deepseek.v4.1",
344            "deepseekv4.1",
345        ]
346        .iter()
347        .any(|prefix| {
348            display_name_lower
349                .strip_prefix(prefix)
350                .is_some_and(|suffix| suffix.is_empty() || suffix.starts_with(['-', '_', '.']))
351        }),
352    }
353}
354
355/// Decides whether to activate the DeepSeek-V4 native formatter.
356///
357/// Primary signal: config.json `model_type`. DeepSeek-V4-Pro and V4-Flash both
358/// ship `"model_type": "deepseek_v4"`, set by the model author — this survives
359/// any `--served-model-name` rename.
360///
361/// Fallback: `display_name`, tight-matched against
362/// `^deepseek(?:[-_.])?v4(?:[-_.]|$)`. Only consulted when config.json is
363/// absent (tokenizer-only MDCs) or unreadable; a concrete config.json value
364/// that is *not* `deepseek_v4` is authoritative and suppresses the fallback.
365fn is_deepseek_v4(model_type_lower: &Option<String>, display_name_lower: &str) -> bool {
366    match model_type_lower.as_deref() {
367        Some("deepseek_v4") => true,
368        Some(_) => false, // config.json says something else — trust it
369        None => is_deepseek_v4_name(display_name_lower),
370    }
371}
372
373/// Decides whether to activate the DeepSeek-V3.2 (non-Exp) native formatter.
374/// Same config-primary / name-fallback rule as V4.
375fn is_deepseek_v3_2_non_exp(model_type_lower: &Option<String>, display_name_lower: &str) -> bool {
376    let name_match = display_name_lower.contains("deepseek")
377        && display_name_lower.contains("v3.2")
378        && !display_name_lower.contains("exp");
379    match model_type_lower.as_deref() {
380        // HF ships `deepseek_v32` (no underscore between 3 and 2); Dynamo's
381        // internal/tool-parser key is `deepseek_v3_2`. Accept both.
382        Some("deepseek_v3_2" | "deepseek_v32") => !display_name_lower.contains("exp"),
383        Some(_) => false,
384        None => name_match,
385    }
386}
387
388/// Tight, anchored match for DeepSeek-V4 display names. Equivalent to the
389/// regex `^deepseek(?:[-_.])?v4(?:[-_.]|$)` over an already-lowercased string.
390/// Written with string ops to avoid pulling in the `regex` crate.
391///
392/// Rejects composite names that previously short-circuited the V4 branch:
393/// - `deepseek-v3.2-v4-foo` (the `v3.2` variant is the real one)
394/// - `deepseek-v40` / `deepseek-v4pro` (no separator after `v4`)
395/// - `my-deepseek-v4` (prefix must be at the start)
396fn is_deepseek_v4_name(name_lower: &str) -> bool {
397    let Some(rest) = name_lower.strip_prefix("deepseek") else {
398        return false;
399    };
400    // Optional single separator between "deepseek" and "v4".
401    let rest = rest
402        .strip_prefix(|c: char| matches!(c, '-' | '_' | '.'))
403        .unwrap_or(rest);
404    let Some(after_v4) = rest.strip_prefix("v4") else {
405        return false;
406    };
407    // `v4` must end the name or be followed by a separator — anything else
408    // (e.g. `v40`, `v4pro`) is a different model family.
409    after_v4.is_empty() || after_v4.starts_with(['-', '_', '.'])
410}
411
412#[cfg(test)]
413mod detection_tests {
414    use super::{is_deepseek_v3_2_non_exp, is_deepseek_v4, is_deepseek_v4_name, is_kimi_k3};
415
416    #[test]
417    fn kimi_k3_detection_prefers_config_model_type() {
418        for model_type in ["kimi_k3", "kimi_k31"] {
419            assert!(is_kimi_k3(&Some(model_type.to_string()), "served-name"));
420            assert!(
421                super::kimi_k3_formatter_for(&Some(model_type.to_string()), "served-name", false)
422                    .is_some()
423            );
424        }
425        assert!(!is_kimi_k3(
426            &Some("kimi_k2".to_string()),
427            "moonshot-kimi-k3"
428        ));
429        assert!(is_kimi_k3(&None, "moonshot-kimi-k3"));
430        assert!(is_kimi_k3(&None, "kimi_k3-instruct"));
431        assert!(!is_kimi_k3(&None, "kimi-k2.5"));
432    }
433
434    #[test]
435    fn v4_name_matches_canonical_variants() {
436        for name in [
437            "deepseek-v4",
438            "deepseek_v4",
439            "deepseek.v4",
440            "deepseekv4",
441            "deepseek-v4-pro",
442            "deepseek-v4-flash",
443            "deepseek-v4-flash-2507",
444            "deepseek-v4.1",
445            "deepseek_v4_thinking",
446        ] {
447            assert!(is_deepseek_v4_name(name), "expected {name} to match V4");
448        }
449    }
450
451    #[test]
452    fn v4_name_rejects_non_v4() {
453        // Composite names that previously short-circuited to V4 before the
454        // V3.2 branch — now correctly rejected.
455        for name in [
456            "deepseek-v3.2-v4-foo",
457            "my-deepseek-v4",
458            "deepseek-v40",
459            "deepseek-v4pro",
460            "deepseekv40",
461            "deepseek-v3",
462            "deepseek-v3.2",
463            "deepseek-r1",
464            "qwen3-v4", // only deepseek-prefixed names qualify
465            "dsflash",
466            "",
467        ] {
468            assert!(
469                !is_deepseek_v4_name(name),
470                "expected {name} to NOT match V4",
471            );
472        }
473    }
474
475    #[test]
476    fn v4_detection_prefers_config_model_type() {
477        // config.json `model_type = "deepseek_v4"` wins regardless of what
478        // the operator calls the model via --served-model-name.
479        let v4 = Some("deepseek_v4".to_string());
480        for display in ["dsflash", "my-pet-model", "llama-3-8b", ""] {
481            assert!(
482                is_deepseek_v4(&v4, display),
483                "config says deepseek_v4, display {display:?} — expected V4",
484            );
485        }
486
487        // A concrete non-V4 config.json suppresses the display-name fallback.
488        // Even if the operator names the served model "deepseek-v4", a model
489        // with `model_type = "llama"` is NOT DeepSeek-V4.
490        let llama = Some("llama".to_string());
491        for display in ["deepseek-v4", "deepseek-v4-flash", "anything"] {
492            assert!(
493                !is_deepseek_v4(&llama, display),
494                "config says llama, display {display:?} — expected NOT V4",
495            );
496        }
497
498        // No config.json — fall back to display-name match.
499        assert!(is_deepseek_v4(&None, "deepseek-v4-flash"));
500        assert!(!is_deepseek_v4(&None, "dsflash"));
501
502        // A config.json with `"model_type": ""` is treated as "no signal" at
503        // the call site (normalized to None before is_deepseek_v4 is called),
504        // so the display-name fallback still runs — pin that contract.
505        let empty: Option<String> = None;
506        assert!(is_deepseek_v4(&empty, "deepseek-v4-flash"));
507        assert!(!is_deepseek_v4(&empty, "dsflash"));
508    }
509
510    #[test]
511    fn v3_2_detection_prefers_config_model_type() {
512        // config says deepseek_v3_2, any non-"exp" display name triggers.
513        let v3_2 = Some("deepseek_v3_2".to_string());
514        assert!(is_deepseek_v3_2_non_exp(&v3_2, "whatever"));
515        assert!(is_deepseek_v3_2_non_exp(&v3_2, "deepseek-v3.2"));
516        // V3.2-Exp is a separate model family; suppress even via config.
517        assert!(!is_deepseek_v3_2_non_exp(&v3_2, "deepseek-v3.2-exp"));
518
519        // The actual HF config.json spelling has no underscore between 3 and 2
520        // (`deepseek_v32`). It must trigger identically to the internal key.
521        let hf_real = Some("deepseek_v32".to_string());
522        assert!(is_deepseek_v3_2_non_exp(&hf_real, "whatever"));
523        assert!(is_deepseek_v3_2_non_exp(&hf_real, "deepseek-v3.2-nvfp4"));
524        assert!(!is_deepseek_v3_2_non_exp(&hf_real, "deepseek-v3.2-exp"));
525
526        // Other config types lose regardless of display name.
527        let other = Some("deepseek_v4".to_string());
528        assert!(!is_deepseek_v3_2_non_exp(&other, "deepseek-v3.2"));
529
530        // No config — fall back to the original display-name heuristic.
531        assert!(is_deepseek_v3_2_non_exp(&None, "deepseek-v3.2-pro"));
532        assert!(!is_deepseek_v3_2_non_exp(&None, "deepseek-v3.2-exp"));
533        assert!(!is_deepseek_v3_2_non_exp(&None, "deepseek-v4"));
534    }
535}