Skip to main content

dynamo_renderer/
lib.rs

1// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3
4//! Prompt Formatting
5//!
6//! Standalone, runtime-free chat-template / prompt formatting for
7//! OpenAI-compatible inference frontends. Renders HuggingFace `chat_template`
8//! jinja2 (via `minijinja` + `minijinja-contrib` pycompat), handles tool
9//! usage formatting and generation-prompt handling.
10//!
11//! Consumers implement [`OAIChatLikeRequest`] for their request type (or use
12//! the ready-made impl for `dynamo-protocols`' OpenAI chat request) and render
13//! with a [`PromptFormatter`] built from a HuggingFace `tokenizer_config.json`
14//! ([`ChatTemplate`]).
15//!
16//! This crate is a *bridge* between OpenAI request types ([`dynamo_protocols`])
17//! and prompt rendering. Most formatters return text; segment-sensitive native
18//! formats can preserve tokenizer policy through [`RenderedPrompt`].
19
20// TODO:
21// 1. Query if `add_generation_prompt` is present in the prompt template
22// 2. Support for models with add_generation_prompt:
23//    - PALS (Prefix-Assisted Language Sampling)
24//    - Continuation - Detected on user turns, where we can return
25//      partial assistant responses without add_generation_prompt
26
27use anyhow::{Context, Result};
28use minijinja::value::Value;
29use std::collections::HashMap;
30use std::sync::Arc;
31
32/// Re-export of `dynamo-tokenizers` as a one-import convenience: consumers that
33/// want both tokenization and chat templating can reach the tokenizer types via
34/// `dynamo_renderer::dynamo_tokenizers::*` without adding a second dependency.
35pub use dynamo_tokenizers;
36
37pub mod beam;
38pub mod deepseek;
39pub mod inkling;
40pub mod kimi_k3;
41pub mod mistral;
42mod python;
43mod template;
44
45pub use template::{
46    ChatTemplate, ChatTemplateContentFormat, ChatTemplateValue, ContextMixins, MediaPlaceholders,
47    deepseek_formatter_for, kimi_k3_formatter_for, may_be_fix_tool_schema, native_formatter_for,
48};
49
50/// Selects which context-mixin behaviors a template renders with.
51///
52/// Carried on the model deployment card (`prompt_context`) and consumed by the
53/// chat-template renderer via [`ContextMixins`].
54#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq, Eq, Hash)]
55#[serde(rename_all = "snake_case")]
56pub enum PromptContextMixin {
57    /// Support OAI Chat Messages and Tools
58    OaiChat,
59
60    /// Enables templates with `{{datetime}}` to be rendered with the current date and time.
61    Llama3DateTime,
62}
63
64/// Shared helper: extract a boolean thinking toggle from `chat_template_args`.
65///
66/// Reads the two equivalent keys (`thinking`, `enable_thinking` — vLLM's
67/// canonical kwarg) in order and returns the first bool value found, or `None`
68/// if neither key is present (or neither carries a bool). Used by the V4
69/// formatter's `resolve_thinking_mode` and by reasoning-parser gating in
70/// consumers so both paths agree on the signal interpretation.
71pub fn thinking_bool_from_args(args: Option<&HashMap<String, serde_json::Value>>) -> Option<bool> {
72    let args = args?;
73    for key in ["thinking", "enable_thinking"] {
74        if let Some(v) = args.get(key).and_then(|x| x.as_bool()) {
75            return Some(v);
76        }
77    }
78    None
79}
80
81#[derive(Debug)]
82pub enum TokenInput {
83    Single(Vec<u32>),
84    Batch(Vec<Vec<u32>>),
85}
86
87#[derive(Debug)]
88pub enum TextInput {
89    Single(String),
90    Batch(Vec<String>),
91}
92
93#[derive(Debug)]
94pub enum PromptInput {
95    Tokens(TokenInput),
96    Text(TextInput),
97}
98
99/// One owned prompt segment with an explicit special-token trust boundary.
100#[derive(Debug, Clone, PartialEq, Eq)]
101pub struct RenderedSegment {
102    pub text: String,
103    pub allow_special: bool,
104}
105
106impl RenderedSegment {
107    pub fn new(text: impl Into<String>, allow_special: bool) -> Self {
108        Self {
109            text: text.into(),
110            allow_special,
111        }
112    }
113
114    pub fn as_encode_segment(&self) -> dynamo_tokenizers::EncodeSegment<'_> {
115        dynamo_tokenizers::EncodeSegment::new(&self.text, self.allow_special)
116    }
117}
118
119/// A rendered prompt plus its optional tokenization boundaries.
120///
121/// The prompt owns its segment text while `dynamo-tokenizers` borrows that text
122/// during encoding. Keeping the types separate preserves the tokenizer crate's
123/// published zero-copy `EncodeSegment<'_>` API.
124#[derive(Debug, Clone, PartialEq, Eq)]
125pub struct RenderedPrompt {
126    text: String,
127    segments: Option<Vec<RenderedSegment>>,
128    /// Number of trailing segments that form the assistant generation stub.
129    ///
130    /// Some reference APIs (Moonshot's Kimi K3) render a channel-opening stub
131    /// such as `<|open|>response<|sep|>` after the conversation so the model
132    /// starts inside the right channel, but exclude those tokens from the
133    /// reported `usage.prompt_tokens`. The renderer records how many trailing
134    /// segments belong to that stub so callers can apply the same convention
135    /// without hardcoding a token count. Zero for every other prompt.
136    pending_segments: usize,
137}
138
139impl RenderedPrompt {
140    pub fn text(text: String) -> Self {
141        Self {
142            text,
143            segments: None,
144            pending_segments: 0,
145        }
146    }
147
148    pub fn segmented(segments: Vec<RenderedSegment>) -> Self {
149        Self::segmented_with_pending(segments, 0)
150    }
151
152    /// Build a segmented prompt whose last `pending_segments` segments are the
153    /// assistant generation stub (see [`RenderedPrompt::pending_segments`]).
154    ///
155    /// # Panics
156    ///
157    /// Panics if `pending_segments` exceeds the number of segments; a renderer
158    /// that claims more pending segments than it produced is a bug.
159    pub fn segmented_with_pending(segments: Vec<RenderedSegment>, pending_segments: usize) -> Self {
160        assert!(
161            pending_segments <= segments.len(),
162            "pending_segments ({pending_segments}) exceeds rendered segments ({})",
163            segments.len()
164        );
165        let text = segments
166            .iter()
167            .map(|segment| segment.text.as_str())
168            .collect();
169        Self {
170            text,
171            segments: Some(segments),
172            pending_segments,
173        }
174    }
175
176    pub fn as_str(&self) -> &str {
177        &self.text
178    }
179
180    pub fn segments(&self) -> Option<&[RenderedSegment]> {
181        self.segments.as_deref()
182    }
183
184    pub fn encode_segments(&self) -> Option<Vec<dynamo_tokenizers::EncodeSegment<'_>>> {
185        Some(
186            self.segments()?
187                .iter()
188                .map(RenderedSegment::as_encode_segment)
189                .collect(),
190        )
191    }
192
193    /// Number of trailing segments that form the assistant generation stub.
194    ///
195    /// Zero when the prompt is unsegmented, when no generation prompt was
196    /// added, or when the renderer has no stub convention to report.
197    pub fn pending_segments(&self) -> usize {
198        self.pending_segments
199    }
200
201    /// The trailing generation-stub segments, or `None` for unsegmented prompts.
202    ///
203    /// Callers that report usage under the reference API's convention encode
204    /// these and subtract their token count from the physical prompt length.
205    pub fn pending_encode_segments(&self) -> Option<Vec<dynamo_tokenizers::EncodeSegment<'_>>> {
206        let segments = self.segments()?;
207        let start = segments.len() - self.pending_segments;
208        Some(
209            segments[start..]
210                .iter()
211                .map(RenderedSegment::as_encode_segment)
212                .collect(),
213        )
214    }
215
216    pub fn into_text(self) -> String {
217        self.text
218    }
219}
220
221/// A prompt-rendering failure caused by the request rather than server state.
222///
223/// Callers can downcast an [`anyhow::Error`] to this type and map it to their
224/// protocol's invalid-request status without treating every template failure as
225/// a client error.
226#[derive(Debug, Clone, PartialEq, Eq)]
227pub enum PromptRenderError {
228    InvalidRequest(String),
229}
230
231impl PromptRenderError {
232    pub fn invalid_request(message: impl Into<String>) -> Self {
233        Self::InvalidRequest(message.into())
234    }
235}
236
237impl std::fmt::Display for PromptRenderError {
238    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
239        match self {
240            Self::InvalidRequest(message) => f.write_str(message),
241        }
242    }
243}
244
245impl std::error::Error for PromptRenderError {}
246
247/// Trait that defines a request that can map to an OpenAI-like request.
248///
249/// Implement this for your request type to render it through a
250/// [`PromptFormatter`]. Media/multimodal IO config is intentionally *not* part
251/// of this trait — it is a preprocessing concern owned by the consumer, kept
252/// off the rendering surface so this crate stays runtime-free.
253pub trait OAIChatLikeRequest {
254    fn model(&self) -> String;
255    fn messages(&self) -> Value;
256    /// Preserve Responses replay metadata that typed Chat messages cannot express.
257    /// Typed messages remain available for media collection.
258    fn template_messages_json_override(&self) -> Option<serde_json::Value> {
259        None
260    }
261
262    /// Optional fast path for rendering without calling [`Self::messages`].
263    ///
264    /// Returning `Some` requires the same serialized messages, including all
265    /// transformations, as `messages()`. Adapters that rewrite messages must
266    /// return `None` unless this slice includes those same changes.
267    fn typed_messages(&self) -> Option<&[dynamo_protocols::types::ChatCompletionRequestMessage]> {
268        None
269    }
270    fn tools(&self) -> Option<Value> {
271        None
272    }
273    /// Tools exactly as the client sent them, before the schema backfill that
274    /// [`Self::tools`] may apply for Jinja templates. Native renderers that
275    /// reproduce a reference encoder use this; `None` when the request type
276    /// does not keep them apart.
277    fn raw_tools(&self) -> Option<Value> {
278        None
279    }
280    /// Namespace/deferred tool definitions retained by a Responses adapter.
281    /// The formatter uses these only when its selected template supports them.
282    fn template_tools(&self) -> Option<Value> {
283        None
284    }
285    fn tool_choice(&self) -> Option<Value> {
286        None
287    }
288    fn response_format(&self) -> Option<Value> {
289        None
290    }
291
292    /// OpenAI-compatible reasoning-effort control, when the request type
293    /// exposes it as a top-level field.
294    fn reasoning_effort(&self) -> Option<Value> {
295        None
296    }
297
298    fn should_add_generation_prompt(&self) -> bool;
299
300    /// Optional additional args to merge into the chat template context
301    fn chat_template_args(&self) -> Option<&HashMap<String, serde_json::Value>> {
302        None
303    }
304
305    /// Returns the type of input for the prompt. Default is Text.
306    fn prompt_input_type(&self) -> PromptInput {
307        PromptInput::Text(TextInput::Single(String::new()))
308    }
309
310    /// Extract tokens if the input is pre-tokenized
311    fn extract_tokens(&self) -> Option<TokenInput> {
312        None
313    }
314
315    fn extract_text(&self) -> Option<TextInput> {
316        None
317    }
318
319    fn mm_processor_kwargs(&self) -> Option<&serde_json::Value> {
320        None
321    }
322}
323
324/// Native renderers and normalization work on JSON, so avoid a MiniJinja
325/// round trip when the request exposes its typed messages.
326pub(crate) fn messages_to_json(req: &dyn OAIChatLikeRequest) -> Result<serde_json::Value> {
327    // Apply replay metadata before the typed fast path can discard it.
328    if let Some(messages) = req.template_messages_json_override() {
329        return Ok(messages);
330    }
331    if let Some(messages) = req.typed_messages() {
332        // Custom API calls must become function calls even on this fast path.
333        dynamo_protocols::types::chat_messages_for_function_template(messages)
334    } else {
335        crate::python::template_json(req.messages())
336    }
337    .context("Failed to convert messages to JSON")
338}
339
340pub trait OAIPromptFormatter: Send + Sync + 'static {
341    fn supports_add_generation_prompt(&self) -> bool;
342    /// Older/native formatters keep flattened tools unless they opt into this shape.
343    fn supports_native_tools(&self, _request: &dyn OAIChatLikeRequest) -> bool {
344        false
345    }
346    fn render(&self, req: &dyn OAIChatLikeRequest) -> Result<String>;
347
348    /// Source message indices in rendered media order; `None` preserves arrival order.
349    /// An explicit order must be a permutation of all source-message indices.
350    /// Parts within each message retain their original order.
351    fn media_message_order(&self, _request: &dyn OAIChatLikeRequest) -> Option<Vec<usize>> {
352        None
353    }
354
355    fn render_prompt(&self, req: &dyn OAIChatLikeRequest) -> Result<RenderedPrompt> {
356        self.render(req).map(RenderedPrompt::text)
357    }
358
359    /// Whether this formatter leaves a final assistant turn open by itself
360    /// when [`OAIChatLikeRequest::should_add_generation_prompt`] is false.
361    /// Callers must then skip their marker-and-truncate emulation of
362    /// `continue_final_message`.
363    fn renders_continuation_natively(&self) -> bool {
364        false
365    }
366}
367
368/// Reject Kimi-style Partial Mode in a formatter that cannot leave the final
369/// assistant turn open for continuation.
370///
371/// `partial: false` and `partial: null` are ordinary message metadata and are
372/// intentionally ignored. Supporting formatters (currently Kimi K3) do not
373/// call this helper and implement the open-turn rendering themselves.
374pub(crate) fn reject_unsupported_partial_assistant(messages: &serde_json::Value) -> Result<()> {
375    let has_partial =
376        messages.as_array().into_iter().flatten().any(|message| {
377            message.get("partial").and_then(serde_json::Value::as_bool) == Some(true)
378        });
379    if has_partial {
380        return Err(PromptRenderError::invalid_request(
381            "assistant `partial: true` is not supported by this model's prompt formatter",
382        )
383        .into());
384    }
385    Ok(())
386}
387
388/// Reject non-empty message-level tool declarations on roles where a formatter
389/// does not support that field. `tools: null` and `tools: []` declare nothing.
390pub(crate) fn reject_unsupported_message_tools(
391    messages: &serde_json::Value,
392    supported_tool_roles: &[&str],
393) -> Result<()> {
394    let offending = messages.as_array().into_iter().flatten().find(|message| {
395        let declares_tools = message
396            .get("tools")
397            .is_some_and(|tools| !tools.is_null() && !tools.as_array().is_some_and(Vec::is_empty));
398        let role_is_supported = message
399            .get("role")
400            .and_then(serde_json::Value::as_str)
401            .is_some_and(|role| supported_tool_roles.contains(&role));
402        declares_tools && !role_is_supported
403    });
404
405    if let Some(message) = offending {
406        let role = message
407            .get("role")
408            .and_then(serde_json::Value::as_str)
409            .unwrap_or("<missing>");
410        return Err(PromptRenderError::invalid_request(format!(
411            "message-level `tools` on role {role:?} are not supported by this model's prompt \
412             formatter"
413        ))
414        .into());
415    }
416    Ok(())
417}
418
419#[derive(Clone)]
420pub enum PromptFormatter {
421    OAI(Arc<dyn OAIPromptFormatter>),
422}
423
424// No-op formatter: used for models without chat_template
425#[derive(Debug, Default)]
426pub struct NoOpFormatter;
427
428impl OAIPromptFormatter for NoOpFormatter {
429    fn supports_add_generation_prompt(&self) -> bool {
430        false
431    }
432
433    fn render(&self, req: &dyn OAIChatLikeRequest) -> Result<String> {
434        let messages = req.messages();
435        let messages_json = serde_json::to_value(&messages)?;
436        reject_unsupported_partial_assistant(&messages_json)?;
437        reject_unsupported_message_tools(&messages_json, &[])?;
438
439        let first_message = messages
440            .get_item_by_index(0)
441            .map_err(|_| anyhow::Error::msg("No message at index 0 or messages array is empty"))?;
442
443        let content = first_message
444            .get_attr("content")
445            .map_err(|_| anyhow::Error::msg("First message has no 'content' field"))?;
446
447        let content_str = content
448            .as_str()
449            .ok_or_else(|| anyhow::Error::msg("Message content is not a string"))?
450            .to_string();
451        Ok(content_str)
452    }
453}
454
455impl PromptFormatter {
456    pub fn no_op() -> Self {
457        Self::OAI(Arc::new(NoOpFormatter))
458    }
459}
460
461#[cfg(test)]
462mod rendered_prompt_tests {
463    use super::{
464        NoOpFormatter, OAIPromptFormatter, PromptRenderError, RenderedPrompt, RenderedSegment,
465    };
466
467    #[test]
468    fn owned_segments_borrow_into_tokenizer_segments() {
469        let prompt = RenderedPrompt::segmented(vec![
470            RenderedSegment::new("<|open|>", true),
471            RenderedSegment::new("user text", false),
472        ]);
473
474        let segments = prompt.encode_segments().expect("segmented prompt");
475        assert_eq!(segments[0].text, "<|open|>");
476        assert!(segments[0].allow_special);
477        assert_eq!(segments[1].text, "user text");
478        assert!(!segments[1].allow_special);
479        assert_eq!(prompt.as_str(), "<|open|>user text");
480    }
481
482    #[test]
483    fn no_op_formatter_rejects_unsupported_partial_assistant() {
484        let request: dynamo_protocols::types::CreateChatCompletionRequest =
485            serde_json::from_value(serde_json::json!({
486                "model": "test",
487                "messages": [
488                    {"role": "user", "content": "Continue"},
489                    {"role": "assistant", "content": "prefix", "partial": true}
490                ]
491            }))
492            .unwrap();
493
494        let error = NoOpFormatter.render(&request).unwrap_err();
495        assert!(matches!(
496            error.downcast_ref::<PromptRenderError>(),
497            Some(PromptRenderError::InvalidRequest(message))
498                if message.contains("`partial: true` is not supported")
499        ));
500    }
501
502    #[test]
503    fn no_op_formatter_rejects_message_level_tools() {
504        let request: dynamo_protocols::types::CreateChatCompletionRequest =
505            serde_json::from_value(serde_json::json!({
506                "model": "test",
507                "messages": [
508                    {"role": "system", "tools": [{"name": "lookup"}]},
509                    {"role": "user", "content": "Continue"}
510                ]
511            }))
512            .unwrap();
513
514        let error = NoOpFormatter.render(&request).unwrap_err();
515        assert!(matches!(
516            error.downcast_ref::<PromptRenderError>(),
517            Some(PromptRenderError::InvalidRequest(message))
518                if message.contains("message-level `tools`")
519        ));
520    }
521
522    #[test]
523    fn rendered_prompt_pending_segments_default_to_zero() {
524        let prompt = RenderedPrompt::segmented(vec![RenderedSegment::new("a", false)]);
525        assert_eq!(prompt.pending_segments(), 0);
526        assert!(prompt.pending_encode_segments().unwrap().is_empty());
527        assert_eq!(RenderedPrompt::text("a".to_string()).pending_segments(), 0);
528        assert!(
529            RenderedPrompt::text("a".to_string())
530                .pending_encode_segments()
531                .is_none()
532        );
533    }
534
535    #[test]
536    fn rendered_prompt_exposes_trailing_pending_segments() {
537        let prompt = RenderedPrompt::segmented_with_pending(
538            vec![
539                RenderedSegment::new("prompt", false),
540                RenderedSegment::new("<|open|>", true),
541                RenderedSegment::new("response", false),
542                RenderedSegment::new("<|sep|>", true),
543            ],
544            3,
545        );
546        assert_eq!(prompt.as_str(), "prompt<|open|>response<|sep|>");
547        assert_eq!(prompt.pending_segments(), 3);
548        let pending: Vec<&str> = prompt
549            .pending_encode_segments()
550            .unwrap()
551            .iter()
552            .map(|segment| segment.text)
553            .collect();
554        assert_eq!(pending, ["<|open|>", "response", "<|sep|>"]);
555    }
556
557    #[test]
558    #[should_panic(expected = "pending_segments")]
559    fn rendered_prompt_rejects_more_pending_than_segments() {
560        RenderedPrompt::segmented_with_pending(vec![RenderedSegment::new("a", false)], 2);
561    }
562}
563
564pub mod response_tools;