Skip to main content

dynamo_renderer/
lib.rs

1// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Prompt Formatting
5//!
6//! Standalone, runtime-free chat-template / prompt formatting for
7//! OpenAI-compatible inference frontends. Renders HuggingFace `chat_template`
8//! jinja2 (via `minijinja` + `minijinja-contrib` pycompat), handles tool
9//! usage formatting and generation-prompt handling.
10//!
11//! Consumers implement [`OAIChatLikeRequest`] for their request type (or use
12//! the ready-made impl for `dynamo-protocols`' OpenAI chat request) and render
13//! with a [`PromptFormatter`] built from a HuggingFace `tokenizer_config.json`
14//! ([`ChatTemplate`]).
15//!
16//! This crate is a *bridge* between OpenAI request types ([`dynamo_protocols`])
17//! and prompt rendering. Most formatters return text; segment-sensitive
18//! formats (native ones, and chat templates unless
19//! [`ChatTemplate::with_request_control_tokens_as_text`] turns it off)
20//! preserve tokenizer policy through [`RenderedPrompt`].
21
22// TODO:
23// 1. Query if `add_generation_prompt` is present in the prompt template
24// 2. Support for models with add_generation_prompt:
25//    - PALS (Prefix-Assisted Language Sampling)
26//    - Continuation - Detected on user turns, where we can return
27//      partial assistant responses without add_generation_prompt
28
29use anyhow::{Context, Result};
30use minijinja::value::Value;
31use std::collections::HashMap;
32use std::sync::Arc;
33
34/// Re-export of `dynamo-tokenizers` as a one-import convenience: consumers that
35/// want both tokenization and chat templating can reach the tokenizer types via
36/// `dynamo_renderer::dynamo_tokenizers::*` without adding a second dependency.
37pub use dynamo_tokenizers;
38
39pub mod beam;
40pub mod deepseek;
41pub mod inkling;
42pub mod kimi_k3;
43pub mod mistral;
44mod python;
45mod template;
46
47pub use template::{
48    ChatTemplate, ChatTemplateContentFormat, ChatTemplateValue, ContextMixins, MediaPlaceholders,
49    deepseek_formatter_for, kimi_k3_formatter_for, may_be_fix_tool_schema, native_formatter_for,
50};
51
52/// Selects which context-mixin behaviors a template renders with.
53///
54/// Carried on the model deployment card (`prompt_context`) and consumed by the
55/// chat-template renderer via [`ContextMixins`].
56#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq, Eq, Hash)]
57#[serde(rename_all = "snake_case")]
58pub enum PromptContextMixin {
59    /// Support OAI Chat Messages and Tools
60    OaiChat,
61
62    /// Enables templates with `{{datetime}}` to be rendered with the current date and time.
63    Llama3DateTime,
64}
65
66/// Shared helper: extract a boolean thinking toggle from `chat_template_args`.
67///
68/// Reads the two equivalent keys (`thinking`, `enable_thinking` — vLLM's
69/// canonical kwarg) in order and returns the first bool value found, or `None`
70/// if neither key is present (or neither carries a bool). Used by the V4
71/// formatter's `resolve_thinking_mode` and by reasoning-parser gating in
72/// consumers so both paths agree on the signal interpretation.
73pub fn thinking_bool_from_args(args: Option<&HashMap<String, serde_json::Value>>) -> Option<bool> {
74    let args = args?;
75    for key in ["thinking", "enable_thinking"] {
76        if let Some(v) = args.get(key).and_then(|x| x.as_bool()) {
77            return Some(v);
78        }
79    }
80    None
81}
82
83#[derive(Debug)]
84pub enum TokenInput {
85    Single(Vec<u32>),
86    Batch(Vec<Vec<u32>>),
87}
88
89#[derive(Debug)]
90pub enum TextInput {
91    Single(String),
92    Batch(Vec<String>),
93}
94
95#[derive(Debug)]
96pub enum PromptInput {
97    Tokens(TokenInput),
98    Text(TextInput),
99}
100
101/// One owned prompt segment with an explicit special-token trust boundary.
102#[derive(Debug, Clone, PartialEq, Eq)]
103pub struct RenderedSegment {
104    pub text: String,
105    pub allow_special: bool,
106}
107
108impl RenderedSegment {
109    pub fn new(text: impl Into<String>, allow_special: bool) -> Self {
110        Self {
111            text: text.into(),
112            allow_special,
113        }
114    }
115
116    pub fn as_encode_segment(&self) -> dynamo_tokenizers::EncodeSegment<'_> {
117        dynamo_tokenizers::EncodeSegment::new(&self.text, self.allow_special)
118    }
119}
120
121/// A rendered prompt plus its optional tokenization boundaries.
122///
123/// The prompt owns its segment text while `dynamo-tokenizers` borrows that text
124/// during encoding. Keeping the types separate preserves the tokenizer crate's
125/// published zero-copy `EncodeSegment<'_>` API.
126#[derive(Debug, Clone, PartialEq, Eq)]
127pub struct RenderedPrompt {
128    text: String,
129    segments: Option<Vec<RenderedSegment>>,
130    /// Number of trailing segments that form the assistant generation stub.
131    ///
132    /// Some reference APIs (Moonshot's Kimi K3) render a channel-opening stub
133    /// such as `<|open|>response<|sep|>` after the conversation so the model
134    /// starts inside the right channel, but exclude those tokens from the
135    /// reported `usage.prompt_tokens`. The renderer records how many trailing
136    /// segments belong to that stub so callers can apply the same convention
137    /// without hardcoding a token count. Zero for every other prompt.
138    pending_segments: usize,
139}
140
141impl RenderedPrompt {
142    pub fn text(text: String) -> Self {
143        Self {
144            text,
145            segments: None,
146            pending_segments: 0,
147        }
148    }
149
150    pub fn segmented(segments: Vec<RenderedSegment>) -> Self {
151        Self::segmented_with_pending(segments, 0)
152    }
153
154    /// Build a segmented prompt whose last `pending_segments` segments are the
155    /// assistant generation stub (see [`RenderedPrompt::pending_segments`]).
156    ///
157    /// # Panics
158    ///
159    /// Panics if `pending_segments` exceeds the number of segments; a renderer
160    /// that claims more pending segments than it produced is a bug.
161    pub fn segmented_with_pending(segments: Vec<RenderedSegment>, pending_segments: usize) -> Self {
162        assert!(
163            pending_segments <= segments.len(),
164            "pending_segments ({pending_segments}) exceeds rendered segments ({})",
165            segments.len()
166        );
167        let text = segments
168            .iter()
169            .map(|segment| segment.text.as_str())
170            .collect();
171        Self {
172            text,
173            segments: Some(segments),
174            pending_segments,
175        }
176    }
177
178    pub fn as_str(&self) -> &str {
179        &self.text
180    }
181
182    pub fn segments(&self) -> Option<&[RenderedSegment]> {
183        self.segments.as_deref()
184    }
185
186    pub fn encode_segments(&self) -> Option<Vec<dynamo_tokenizers::EncodeSegment<'_>>> {
187        Some(
188            self.segments()?
189                .iter()
190                .map(RenderedSegment::as_encode_segment)
191                .collect(),
192        )
193    }
194
195    /// Number of trailing segments that form the assistant generation stub.
196    ///
197    /// Zero when the prompt is unsegmented, when no generation prompt was
198    /// added, or when the renderer has no stub convention to report.
199    pub fn pending_segments(&self) -> usize {
200        self.pending_segments
201    }
202
203    /// The trailing generation-stub segments, or `None` for unsegmented prompts.
204    ///
205    /// Callers that report usage under the reference API's convention encode
206    /// these and subtract their token count from the physical prompt length.
207    pub fn pending_encode_segments(&self) -> Option<Vec<dynamo_tokenizers::EncodeSegment<'_>>> {
208        let segments = self.segments()?;
209        let start = segments.len() - self.pending_segments;
210        Some(
211            segments[start..]
212                .iter()
213                .map(RenderedSegment::as_encode_segment)
214                .collect(),
215        )
216    }
217
218    pub fn into_text(self) -> String {
219        self.text
220    }
221}
222
223/// A prompt-rendering failure caused by the request rather than server state.
224///
225/// Callers can downcast an [`anyhow::Error`] to this type and map it to their
226/// protocol's invalid-request status without treating every template failure as
227/// a client error.
228#[derive(Debug, Clone, PartialEq, Eq)]
229pub enum PromptRenderError {
230    InvalidRequest(String),
231}
232
233impl PromptRenderError {
234    pub fn invalid_request(message: impl Into<String>) -> Self {
235        Self::InvalidRequest(message.into())
236    }
237}
238
239impl std::fmt::Display for PromptRenderError {
240    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
241        match self {
242            Self::InvalidRequest(message) => f.write_str(message),
243        }
244    }
245}
246
247impl std::error::Error for PromptRenderError {}
248
249/// Trait that defines a request that can map to an OpenAI-like request.
250///
251/// Implement this for your request type to render it through a
252/// [`PromptFormatter`]. Media/multimodal IO config is intentionally *not* part
253/// of this trait — it is a preprocessing concern owned by the consumer, kept
254/// off the rendering surface so this crate stays runtime-free.
255pub trait OAIChatLikeRequest {
256    fn model(&self) -> String;
257    fn messages(&self) -> Value;
258    /// Preserve Responses replay metadata that typed Chat messages cannot express.
259    /// Typed messages remain available for media collection.
260    fn template_messages_json_override(&self) -> Option<serde_json::Value> {
261        None
262    }
263
264    /// Optional fast path for rendering without calling [`Self::messages`].
265    ///
266    /// Returning `Some` requires the same serialized messages, including all
267    /// transformations, as `messages()`. Adapters that rewrite messages must
268    /// return `None` unless this slice includes those same changes.
269    fn typed_messages(&self) -> Option<&[dynamo_protocols::types::ChatCompletionRequestMessage]> {
270        None
271    }
272    fn tools(&self) -> Option<Value> {
273        None
274    }
275    /// Tools exactly as the client sent them, before the schema backfill that
276    /// [`Self::tools`] may apply for Jinja templates. Native renderers that
277    /// reproduce a reference encoder use this; `None` when the request type
278    /// does not keep them apart.
279    fn raw_tools(&self) -> Option<Value> {
280        None
281    }
282    /// Namespace/deferred tool definitions retained by a Responses adapter.
283    /// The formatter uses these only when its selected template supports them.
284    fn template_tools(&self) -> Option<Value> {
285        None
286    }
287    fn tool_choice(&self) -> Option<Value> {
288        None
289    }
290    fn response_format(&self) -> Option<Value> {
291        None
292    }
293
294    /// OpenAI-compatible reasoning-effort control, when the request type
295    /// exposes it as a top-level field.
296    fn reasoning_effort(&self) -> Option<Value> {
297        None
298    }
299
300    fn should_add_generation_prompt(&self) -> bool;
301
302    /// Optional additional args to merge into the chat template context
303    fn chat_template_args(&self) -> Option<&HashMap<String, serde_json::Value>> {
304        None
305    }
306
307    /// Returns the type of input for the prompt. Default is Text.
308    fn prompt_input_type(&self) -> PromptInput {
309        PromptInput::Text(TextInput::Single(String::new()))
310    }
311
312    /// Extract tokens if the input is pre-tokenized
313    fn extract_tokens(&self) -> Option<TokenInput> {
314        None
315    }
316
317    fn extract_text(&self) -> Option<TextInput> {
318        None
319    }
320
321    fn mm_processor_kwargs(&self) -> Option<&serde_json::Value> {
322        None
323    }
324}
325
326/// Native renderers and normalization work on JSON, so avoid a MiniJinja
327/// round trip when the request exposes its typed messages.
328pub(crate) fn messages_to_json(req: &dyn OAIChatLikeRequest) -> Result<serde_json::Value> {
329    // Apply replay metadata before the typed fast path can discard it.
330    if let Some(messages) = req.template_messages_json_override() {
331        return Ok(messages);
332    }
333    if let Some(messages) = req.typed_messages() {
334        // Custom API calls must become function calls even on this fast path.
335        dynamo_protocols::types::chat_messages_for_function_template(messages)
336    } else {
337        crate::python::template_json(req.messages())
338    }
339    .context("Failed to convert messages to JSON")
340}
341
342pub trait OAIPromptFormatter: Send + Sync + 'static {
343    fn supports_add_generation_prompt(&self) -> bool;
344    /// Older/native formatters keep flattened tools unless they opt into this shape.
345    fn supports_native_tools(&self, _request: &dyn OAIChatLikeRequest) -> bool {
346        false
347    }
348    fn render(&self, req: &dyn OAIChatLikeRequest) -> Result<String>;
349
350    /// Source message indices in rendered media order; `None` preserves arrival order.
351    /// An explicit order must be a permutation of all source-message indices.
352    /// Parts within each message retain their original order.
353    fn media_message_order(&self, _request: &dyn OAIChatLikeRequest) -> Option<Vec<usize>> {
354        None
355    }
356
357    fn render_prompt(&self, req: &dyn OAIChatLikeRequest) -> Result<RenderedPrompt> {
358        self.render(req).map(RenderedPrompt::text)
359    }
360
361    /// Whether this formatter leaves a final assistant turn open by itself
362    /// when [`OAIChatLikeRequest::should_add_generation_prompt`] is false.
363    /// Callers must then skip their marker-and-truncate emulation of
364    /// `continue_final_message`.
365    fn renders_continuation_natively(&self) -> bool {
366        false
367    }
368}
369
370/// Reject Kimi-style Partial Mode in a formatter that cannot leave the final
371/// assistant turn open for continuation.
372///
373/// `partial: false` and `partial: null` are ordinary message metadata and are
374/// intentionally ignored. Supporting formatters (currently Kimi K3) do not
375/// call this helper and implement the open-turn rendering themselves.
376pub(crate) fn reject_unsupported_partial_assistant(messages: &serde_json::Value) -> Result<()> {
377    let has_partial =
378        messages.as_array().into_iter().flatten().any(|message| {
379            message.get("partial").and_then(serde_json::Value::as_bool) == Some(true)
380        });
381    if has_partial {
382        return Err(PromptRenderError::invalid_request(
383            "assistant `partial: true` is not supported by this model's prompt formatter",
384        )
385        .into());
386    }
387    Ok(())
388}
389
390/// Reject non-empty message-level tool declarations on roles where a formatter
391/// does not support that field. `tools: null` and `tools: []` declare nothing.
392pub(crate) fn reject_unsupported_message_tools(
393    messages: &serde_json::Value,
394    supported_tool_roles: &[&str],
395) -> Result<()> {
396    let offending = messages.as_array().into_iter().flatten().find(|message| {
397        let declares_tools = message
398            .get("tools")
399            .is_some_and(|tools| !tools.is_null() && !tools.as_array().is_some_and(Vec::is_empty));
400        let role_is_supported = message
401            .get("role")
402            .and_then(serde_json::Value::as_str)
403            .is_some_and(|role| supported_tool_roles.contains(&role));
404        declares_tools && !role_is_supported
405    });
406
407    if let Some(message) = offending {
408        let role = message
409            .get("role")
410            .and_then(serde_json::Value::as_str)
411            .unwrap_or("<missing>");
412        return Err(PromptRenderError::invalid_request(format!(
413            "message-level `tools` on role {role:?} are not supported by this model's prompt \
414             formatter"
415        ))
416        .into());
417    }
418    Ok(())
419}
420
421#[derive(Clone)]
422pub enum PromptFormatter {
423    OAI(Arc<dyn OAIPromptFormatter>),
424}
425
426// No-op formatter: used for models without chat_template
427#[derive(Debug, Default)]
428pub struct NoOpFormatter;
429
430impl OAIPromptFormatter for NoOpFormatter {
431    fn supports_add_generation_prompt(&self) -> bool {
432        false
433    }
434
435    fn render(&self, req: &dyn OAIChatLikeRequest) -> Result<String> {
436        let messages = req.messages();
437        let messages_json = serde_json::to_value(&messages)?;
438        reject_unsupported_partial_assistant(&messages_json)?;
439        reject_unsupported_message_tools(&messages_json, &[])?;
440
441        let first_message = messages
442            .get_item_by_index(0)
443            .map_err(|_| anyhow::Error::msg("No message at index 0 or messages array is empty"))?;
444
445        let content = first_message
446            .get_attr("content")
447            .map_err(|_| anyhow::Error::msg("First message has no 'content' field"))?;
448
449        let content_str = content
450            .as_str()
451            .ok_or_else(|| anyhow::Error::msg("Message content is not a string"))?
452            .to_string();
453        Ok(content_str)
454    }
455}
456
457impl PromptFormatter {
458    pub fn no_op() -> Self {
459        Self::OAI(Arc::new(NoOpFormatter))
460    }
461}
462
463#[cfg(test)]
464mod rendered_prompt_tests {
465    use super::{
466        NoOpFormatter, OAIPromptFormatter, PromptRenderError, RenderedPrompt, RenderedSegment,
467    };
468
469    #[test]
470    fn owned_segments_borrow_into_tokenizer_segments() {
471        let prompt = RenderedPrompt::segmented(vec![
472            RenderedSegment::new("<|open|>", true),
473            RenderedSegment::new("user text", false),
474        ]);
475
476        let segments = prompt.encode_segments().expect("segmented prompt");
477        assert_eq!(segments[0].text, "<|open|>");
478        assert!(segments[0].allow_special);
479        assert_eq!(segments[1].text, "user text");
480        assert!(!segments[1].allow_special);
481        assert_eq!(prompt.as_str(), "<|open|>user text");
482    }
483
484    #[test]
485    fn no_op_formatter_rejects_unsupported_partial_assistant() {
486        let request: dynamo_protocols::types::CreateChatCompletionRequest =
487            serde_json::from_value(serde_json::json!({
488                "model": "test",
489                "messages": [
490                    {"role": "user", "content": "Continue"},
491                    {"role": "assistant", "content": "prefix", "partial": true}
492                ]
493            }))
494            .unwrap();
495
496        let error = NoOpFormatter.render(&request).unwrap_err();
497        assert!(matches!(
498            error.downcast_ref::<PromptRenderError>(),
499            Some(PromptRenderError::InvalidRequest(message))
500                if message.contains("`partial: true` is not supported")
501        ));
502    }
503
504    #[test]
505    fn no_op_formatter_rejects_message_level_tools() {
506        let request: dynamo_protocols::types::CreateChatCompletionRequest =
507            serde_json::from_value(serde_json::json!({
508                "model": "test",
509                "messages": [
510                    {"role": "system", "tools": [{"name": "lookup"}]},
511                    {"role": "user", "content": "Continue"}
512                ]
513            }))
514            .unwrap();
515
516        let error = NoOpFormatter.render(&request).unwrap_err();
517        assert!(matches!(
518            error.downcast_ref::<PromptRenderError>(),
519            Some(PromptRenderError::InvalidRequest(message))
520                if message.contains("message-level `tools`")
521        ));
522    }
523
524    #[test]
525    fn rendered_prompt_pending_segments_default_to_zero() {
526        let prompt = RenderedPrompt::segmented(vec![RenderedSegment::new("a", false)]);
527        assert_eq!(prompt.pending_segments(), 0);
528        assert!(prompt.pending_encode_segments().unwrap().is_empty());
529        assert_eq!(RenderedPrompt::text("a".to_string()).pending_segments(), 0);
530        assert!(
531            RenderedPrompt::text("a".to_string())
532                .pending_encode_segments()
533                .is_none()
534        );
535    }
536
537    #[test]
538    fn rendered_prompt_exposes_trailing_pending_segments() {
539        let prompt = RenderedPrompt::segmented_with_pending(
540            vec![
541                RenderedSegment::new("prompt", false),
542                RenderedSegment::new("<|open|>", true),
543                RenderedSegment::new("response", false),
544                RenderedSegment::new("<|sep|>", true),
545            ],
546            3,
547        );
548        assert_eq!(prompt.as_str(), "prompt<|open|>response<|sep|>");
549        assert_eq!(prompt.pending_segments(), 3);
550        let pending: Vec<&str> = prompt
551            .pending_encode_segments()
552            .unwrap()
553            .iter()
554            .map(|segment| segment.text)
555            .collect();
556        assert_eq!(pending, ["<|open|>", "response", "<|sep|>"]);
557    }
558
559    #[test]
560    #[should_panic(expected = "pending_segments")]
561    fn rendered_prompt_rejects_more_pending_than_segments() {
562        RenderedPrompt::segmented_with_pending(vec![RenderedSegment::new("a", false)], 2);
563    }
564}
565
566pub mod response_tools;