Skip to main content

dynamo_renderer/
lib.rs

1// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Prompt Formatting
5//!
6//! Standalone, runtime-free chat-template / prompt formatting for
7//! OpenAI-compatible inference frontends. Renders HuggingFace `chat_template`
8//! jinja2 (via `minijinja` + `minijinja-contrib` pycompat), handles tool
9//! usage formatting and generation-prompt handling.
10//!
11//! Consumers implement [`OAIChatLikeRequest`] for their request type (or use
12//! the ready-made impl for `dynamo-protocols`' OpenAI chat request) and render
13//! with a [`PromptFormatter`] built from a HuggingFace `tokenizer_config.json`
14//! ([`ChatTemplate`]).
15//!
16//! This crate is a *bridge* between OpenAI request types ([`dynamo_protocols`])
17//! and prompt rendering. Most formatters return text; segment-sensitive native
18//! formats can preserve tokenizer policy through [`RenderedPrompt`].
19
20// TODO:
21// 1. Query if `add_generation_prompt` is present in the prompt template
22// 2. Support for models with add_generation_prompt:
23//    - PALS (Prefix-Assisted Language Sampling)
24//    - Continuation - Detected on user turns, where we can return
25//      partial assistant responses without add_generation_prompt
26
27use anyhow::{Context, Result};
28use minijinja::value::Value;
29use std::collections::HashMap;
30use std::sync::Arc;
31
32/// Re-export of `dynamo-tokenizers` as a one-import convenience: consumers that
33/// want both tokenization and chat templating can reach the tokenizer types via
34/// `dynamo_renderer::dynamo_tokenizers::*` without adding a second dependency.
35pub use dynamo_tokenizers;
36
37pub mod beam;
38pub mod deepseek;
39pub mod inkling;
40pub mod kimi_k3;
41mod python;
42mod template;
43
44pub use template::{
45    ChatTemplate, ChatTemplateContentFormat, ChatTemplateValue, ContextMixins, MediaPlaceholders,
46    deepseek_formatter_for, kimi_k3_formatter_for, may_be_fix_tool_schema, native_formatter_for,
47};
48
49/// Selects which context-mixin behaviors a template renders with.
50///
51/// Carried on the model deployment card (`prompt_context`) and consumed by the
52/// chat-template renderer via [`ContextMixins`].
53#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq, Eq, Hash)]
54#[serde(rename_all = "snake_case")]
55pub enum PromptContextMixin {
56    /// Support OAI Chat Messages and Tools
57    OaiChat,
58
59    /// Enables templates with `{{datetime}}` to be rendered with the current date and time.
60    Llama3DateTime,
61}
62
63/// Shared helper: extract a boolean thinking toggle from `chat_template_args`.
64///
65/// Reads the two equivalent keys (`thinking`, `enable_thinking` — vLLM's
66/// canonical kwarg) in order and returns the first bool value found, or `None`
67/// if neither key is present (or neither carries a bool). Used by the V4
68/// formatter's `resolve_thinking_mode` and by reasoning-parser gating in
69/// consumers so both paths agree on the signal interpretation.
70pub fn thinking_bool_from_args(args: Option<&HashMap<String, serde_json::Value>>) -> Option<bool> {
71    let args = args?;
72    for key in ["thinking", "enable_thinking"] {
73        if let Some(v) = args.get(key).and_then(|x| x.as_bool()) {
74            return Some(v);
75        }
76    }
77    None
78}
79
80#[derive(Debug)]
81pub enum TokenInput {
82    Single(Vec<u32>),
83    Batch(Vec<Vec<u32>>),
84}
85
86#[derive(Debug)]
87pub enum TextInput {
88    Single(String),
89    Batch(Vec<String>),
90}
91
92#[derive(Debug)]
93pub enum PromptInput {
94    Tokens(TokenInput),
95    Text(TextInput),
96}
97
98/// One owned prompt segment with an explicit special-token trust boundary.
99#[derive(Debug, Clone, PartialEq, Eq)]
100pub struct RenderedSegment {
101    pub text: String,
102    pub allow_special: bool,
103}
104
105impl RenderedSegment {
106    pub fn new(text: impl Into<String>, allow_special: bool) -> Self {
107        Self {
108            text: text.into(),
109            allow_special,
110        }
111    }
112
113    pub fn as_encode_segment(&self) -> dynamo_tokenizers::EncodeSegment<'_> {
114        dynamo_tokenizers::EncodeSegment::new(&self.text, self.allow_special)
115    }
116}
117
118/// A rendered prompt plus its optional tokenization boundaries.
119///
120/// The prompt owns its segment text while `dynamo-tokenizers` borrows that text
121/// during encoding. Keeping the types separate preserves the tokenizer crate's
122/// published zero-copy `EncodeSegment<'_>` API.
123#[derive(Debug, Clone, PartialEq, Eq)]
124pub struct RenderedPrompt {
125    text: String,
126    segments: Option<Vec<RenderedSegment>>,
127    /// Number of trailing segments that form the assistant generation stub.
128    ///
129    /// Some reference APIs (Moonshot's Kimi K3) render a channel-opening stub
130    /// such as `<|open|>response<|sep|>` after the conversation so the model
131    /// starts inside the right channel, but exclude those tokens from the
132    /// reported `usage.prompt_tokens`. The renderer records how many trailing
133    /// segments belong to that stub so callers can apply the same convention
134    /// without hardcoding a token count. Zero for every other prompt.
135    pending_segments: usize,
136}
137
138impl RenderedPrompt {
139    pub fn text(text: String) -> Self {
140        Self {
141            text,
142            segments: None,
143            pending_segments: 0,
144        }
145    }
146
147    pub fn segmented(segments: Vec<RenderedSegment>) -> Self {
148        Self::segmented_with_pending(segments, 0)
149    }
150
151    /// Build a segmented prompt whose last `pending_segments` segments are the
152    /// assistant generation stub (see [`RenderedPrompt::pending_segments`]).
153    ///
154    /// # Panics
155    ///
156    /// Panics if `pending_segments` exceeds the number of segments; a renderer
157    /// that claims more pending segments than it produced is a bug.
158    pub fn segmented_with_pending(segments: Vec<RenderedSegment>, pending_segments: usize) -> Self {
159        assert!(
160            pending_segments <= segments.len(),
161            "pending_segments ({pending_segments}) exceeds rendered segments ({})",
162            segments.len()
163        );
164        let text = segments
165            .iter()
166            .map(|segment| segment.text.as_str())
167            .collect();
168        Self {
169            text,
170            segments: Some(segments),
171            pending_segments,
172        }
173    }
174
175    pub fn as_str(&self) -> &str {
176        &self.text
177    }
178
179    pub fn segments(&self) -> Option<&[RenderedSegment]> {
180        self.segments.as_deref()
181    }
182
183    pub fn encode_segments(&self) -> Option<Vec<dynamo_tokenizers::EncodeSegment<'_>>> {
184        Some(
185            self.segments()?
186                .iter()
187                .map(RenderedSegment::as_encode_segment)
188                .collect(),
189        )
190    }
191
192    /// Number of trailing segments that form the assistant generation stub.
193    ///
194    /// Zero when the prompt is unsegmented, when no generation prompt was
195    /// added, or when the renderer has no stub convention to report.
196    pub fn pending_segments(&self) -> usize {
197        self.pending_segments
198    }
199
200    /// The trailing generation-stub segments, or `None` for unsegmented prompts.
201    ///
202    /// Callers that report usage under the reference API's convention encode
203    /// these and subtract their token count from the physical prompt length.
204    pub fn pending_encode_segments(&self) -> Option<Vec<dynamo_tokenizers::EncodeSegment<'_>>> {
205        let segments = self.segments()?;
206        let start = segments.len() - self.pending_segments;
207        Some(
208            segments[start..]
209                .iter()
210                .map(RenderedSegment::as_encode_segment)
211                .collect(),
212        )
213    }
214
215    pub fn into_text(self) -> String {
216        self.text
217    }
218}
219
220/// A prompt-rendering failure caused by the request rather than server state.
221///
222/// Callers can downcast an [`anyhow::Error`] to this type and map it to their
223/// protocol's invalid-request status without treating every template failure as
224/// a client error.
225#[derive(Debug, Clone, PartialEq, Eq)]
226pub enum PromptRenderError {
227    InvalidRequest(String),
228}
229
230impl PromptRenderError {
231    pub fn invalid_request(message: impl Into<String>) -> Self {
232        Self::InvalidRequest(message.into())
233    }
234}
235
236impl std::fmt::Display for PromptRenderError {
237    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
238        match self {
239            Self::InvalidRequest(message) => f.write_str(message),
240        }
241    }
242}
243
244impl std::error::Error for PromptRenderError {}
245
246/// Trait that defines a request that can map to an OpenAI-like request.
247///
248/// Implement this for your request type to render it through a
249/// [`PromptFormatter`]. Media/multimodal IO config is intentionally *not* part
250/// of this trait — it is a preprocessing concern owned by the consumer, kept
251/// off the rendering surface so this crate stays runtime-free.
252pub trait OAIChatLikeRequest {
253    fn model(&self) -> String;
254    fn messages(&self) -> Value;
255    /// Preserve Responses replay metadata that typed Chat messages cannot express.
256    /// Typed messages remain available for media collection.
257    fn template_messages_json_override(&self) -> Option<serde_json::Value> {
258        None
259    }
260
261    /// Optional fast path for rendering without calling [`Self::messages`].
262    ///
263    /// Returning `Some` requires the same serialized messages, including all
264    /// transformations, as `messages()`. Adapters that rewrite messages must
265    /// return `None` unless this slice includes those same changes.
266    fn typed_messages(&self) -> Option<&[dynamo_protocols::types::ChatCompletionRequestMessage]> {
267        None
268    }
269    fn tools(&self) -> Option<Value> {
270        None
271    }
272    /// Namespace/deferred tool definitions retained by a Responses adapter.
273    /// The formatter uses these only when its selected template supports them.
274    fn template_tools(&self) -> Option<Value> {
275        None
276    }
277    fn tool_choice(&self) -> Option<Value> {
278        None
279    }
280    fn response_format(&self) -> Option<Value> {
281        None
282    }
283
284    /// OpenAI-compatible reasoning-effort control, when the request type
285    /// exposes it as a top-level field.
286    fn reasoning_effort(&self) -> Option<Value> {
287        None
288    }
289
290    fn should_add_generation_prompt(&self) -> bool;
291
292    /// Optional additional args to merge into the chat template context
293    fn chat_template_args(&self) -> Option<&HashMap<String, serde_json::Value>> {
294        None
295    }
296
297    /// Returns the type of input for the prompt. Default is Text.
298    fn prompt_input_type(&self) -> PromptInput {
299        PromptInput::Text(TextInput::Single(String::new()))
300    }
301
302    /// Extract tokens if the input is pre-tokenized
303    fn extract_tokens(&self) -> Option<TokenInput> {
304        None
305    }
306
307    fn extract_text(&self) -> Option<TextInput> {
308        None
309    }
310
311    fn mm_processor_kwargs(&self) -> Option<&serde_json::Value> {
312        None
313    }
314}
315
316/// Native renderers and normalization work on JSON, so avoid a MiniJinja
317/// round trip when the request exposes its typed messages.
318pub(crate) fn messages_to_json(req: &dyn OAIChatLikeRequest) -> Result<serde_json::Value> {
319    // Apply replay metadata before the typed fast path can discard it.
320    if let Some(messages) = req.template_messages_json_override() {
321        return Ok(messages);
322    }
323    if let Some(messages) = req.typed_messages() {
324        // Custom API calls must become function calls even on this fast path.
325        dynamo_protocols::types::chat_messages_for_function_template(messages)
326    } else {
327        crate::python::template_json(req.messages())
328    }
329    .context("Failed to convert messages to JSON")
330}
331
332pub trait OAIPromptFormatter: Send + Sync + 'static {
333    fn supports_add_generation_prompt(&self) -> bool;
334    /// Older/native formatters keep flattened tools unless they opt into this shape.
335    fn supports_native_tools(&self, _request: &dyn OAIChatLikeRequest) -> bool {
336        false
337    }
338    fn render(&self, req: &dyn OAIChatLikeRequest) -> Result<String>;
339
340    /// Source message indices in rendered media order; `None` preserves arrival order.
341    /// An explicit order must be a permutation of all source-message indices.
342    /// Parts within each message retain their original order.
343    fn media_message_order(&self, _request: &dyn OAIChatLikeRequest) -> Option<Vec<usize>> {
344        None
345    }
346
347    fn render_prompt(&self, req: &dyn OAIChatLikeRequest) -> Result<RenderedPrompt> {
348        self.render(req).map(RenderedPrompt::text)
349    }
350}
351
352/// Reject Kimi-style Partial Mode in a formatter that cannot leave the final
353/// assistant turn open for continuation.
354///
355/// `partial: false` and `partial: null` are ordinary message metadata and are
356/// intentionally ignored. Supporting formatters (currently Kimi K3) do not
357/// call this helper and implement the open-turn rendering themselves.
358pub(crate) fn reject_unsupported_partial_assistant(messages: &serde_json::Value) -> Result<()> {
359    let has_partial =
360        messages.as_array().into_iter().flatten().any(|message| {
361            message.get("partial").and_then(serde_json::Value::as_bool) == Some(true)
362        });
363    if has_partial {
364        return Err(PromptRenderError::invalid_request(
365            "assistant `partial: true` is not supported by this model's prompt formatter",
366        )
367        .into());
368    }
369    Ok(())
370}
371
372/// Reject non-empty message-level tool declarations on roles where a formatter
373/// does not support that field. `tools: null` and `tools: []` declare nothing.
374pub(crate) fn reject_unsupported_message_tools(
375    messages: &serde_json::Value,
376    supported_tool_roles: &[&str],
377) -> Result<()> {
378    let offending = messages.as_array().into_iter().flatten().find(|message| {
379        let declares_tools = message
380            .get("tools")
381            .is_some_and(|tools| !tools.is_null() && !tools.as_array().is_some_and(Vec::is_empty));
382        let role_is_supported = message
383            .get("role")
384            .and_then(serde_json::Value::as_str)
385            .is_some_and(|role| supported_tool_roles.contains(&role));
386        declares_tools && !role_is_supported
387    });
388
389    if let Some(message) = offending {
390        let role = message
391            .get("role")
392            .and_then(serde_json::Value::as_str)
393            .unwrap_or("<missing>");
394        return Err(PromptRenderError::invalid_request(format!(
395            "message-level `tools` on role {role:?} are not supported by this model's prompt \
396             formatter"
397        ))
398        .into());
399    }
400    Ok(())
401}
402
403#[derive(Clone)]
404pub enum PromptFormatter {
405    OAI(Arc<dyn OAIPromptFormatter>),
406}
407
408// No-op formatter: used for models without chat_template
409#[derive(Debug, Default)]
410pub struct NoOpFormatter;
411
412impl OAIPromptFormatter for NoOpFormatter {
413    fn supports_add_generation_prompt(&self) -> bool {
414        false
415    }
416
417    fn render(&self, req: &dyn OAIChatLikeRequest) -> Result<String> {
418        let messages = req.messages();
419        let messages_json = serde_json::to_value(&messages)?;
420        reject_unsupported_partial_assistant(&messages_json)?;
421        reject_unsupported_message_tools(&messages_json, &[])?;
422
423        let first_message = messages
424            .get_item_by_index(0)
425            .map_err(|_| anyhow::Error::msg("No message at index 0 or messages array is empty"))?;
426
427        let content = first_message
428            .get_attr("content")
429            .map_err(|_| anyhow::Error::msg("First message has no 'content' field"))?;
430
431        let content_str = content
432            .as_str()
433            .ok_or_else(|| anyhow::Error::msg("Message content is not a string"))?
434            .to_string();
435        Ok(content_str)
436    }
437}
438
439impl PromptFormatter {
440    pub fn no_op() -> Self {
441        Self::OAI(Arc::new(NoOpFormatter))
442    }
443}
444
445#[cfg(test)]
446mod rendered_prompt_tests {
447    use super::{
448        NoOpFormatter, OAIPromptFormatter, PromptRenderError, RenderedPrompt, RenderedSegment,
449    };
450
451    #[test]
452    fn owned_segments_borrow_into_tokenizer_segments() {
453        let prompt = RenderedPrompt::segmented(vec![
454            RenderedSegment::new("<|open|>", true),
455            RenderedSegment::new("user text", false),
456        ]);
457
458        let segments = prompt.encode_segments().expect("segmented prompt");
459        assert_eq!(segments[0].text, "<|open|>");
460        assert!(segments[0].allow_special);
461        assert_eq!(segments[1].text, "user text");
462        assert!(!segments[1].allow_special);
463        assert_eq!(prompt.as_str(), "<|open|>user text");
464    }
465
466    #[test]
467    fn no_op_formatter_rejects_unsupported_partial_assistant() {
468        let request: dynamo_protocols::types::CreateChatCompletionRequest =
469            serde_json::from_value(serde_json::json!({
470                "model": "test",
471                "messages": [
472                    {"role": "user", "content": "Continue"},
473                    {"role": "assistant", "content": "prefix", "partial": true}
474                ]
475            }))
476            .unwrap();
477
478        let error = NoOpFormatter.render(&request).unwrap_err();
479        assert!(matches!(
480            error.downcast_ref::<PromptRenderError>(),
481            Some(PromptRenderError::InvalidRequest(message))
482                if message.contains("`partial: true` is not supported")
483        ));
484    }
485
486    #[test]
487    fn no_op_formatter_rejects_message_level_tools() {
488        let request: dynamo_protocols::types::CreateChatCompletionRequest =
489            serde_json::from_value(serde_json::json!({
490                "model": "test",
491                "messages": [
492                    {"role": "system", "tools": [{"name": "lookup"}]},
493                    {"role": "user", "content": "Continue"}
494                ]
495            }))
496            .unwrap();
497
498        let error = NoOpFormatter.render(&request).unwrap_err();
499        assert!(matches!(
500            error.downcast_ref::<PromptRenderError>(),
501            Some(PromptRenderError::InvalidRequest(message))
502                if message.contains("message-level `tools`")
503        ));
504    }
505
506    #[test]
507    fn rendered_prompt_pending_segments_default_to_zero() {
508        let prompt = RenderedPrompt::segmented(vec![RenderedSegment::new("a", false)]);
509        assert_eq!(prompt.pending_segments(), 0);
510        assert!(prompt.pending_encode_segments().unwrap().is_empty());
511        assert_eq!(RenderedPrompt::text("a".to_string()).pending_segments(), 0);
512        assert!(
513            RenderedPrompt::text("a".to_string())
514                .pending_encode_segments()
515                .is_none()
516        );
517    }
518
519    #[test]
520    fn rendered_prompt_exposes_trailing_pending_segments() {
521        let prompt = RenderedPrompt::segmented_with_pending(
522            vec![
523                RenderedSegment::new("prompt", false),
524                RenderedSegment::new("<|open|>", true),
525                RenderedSegment::new("response", false),
526                RenderedSegment::new("<|sep|>", true),
527            ],
528            3,
529        );
530        assert_eq!(prompt.as_str(), "prompt<|open|>response<|sep|>");
531        assert_eq!(prompt.pending_segments(), 3);
532        let pending: Vec<&str> = prompt
533            .pending_encode_segments()
534            .unwrap()
535            .iter()
536            .map(|segment| segment.text)
537            .collect();
538        assert_eq!(pending, ["<|open|>", "response", "<|sep|>"]);
539    }
540
541    #[test]
542    #[should_panic(expected = "pending_segments")]
543    fn rendered_prompt_rejects_more_pending_than_segments() {
544        RenderedPrompt::segmented_with_pending(vec![RenderedSegment::new("a", false)], 2);
545    }
546}
547
548pub mod response_tools;