1use anyhow::{Context, Result};
28use minijinja::value::Value;
29use std::collections::HashMap;
30use std::sync::Arc;
31
32pub use dynamo_tokenizers;
36
37pub mod beam;
38pub mod deepseek;
39pub mod inkling;
40pub mod kimi_k3;
41mod python;
42mod template;
43
44pub use template::{
45 ChatTemplate, ChatTemplateValue, ContextMixins, deepseek_formatter_for, kimi_k3_formatter_for,
46 may_be_fix_tool_schema, native_formatter_for,
47};
48
49#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq, Eq, Hash)]
54#[serde(rename_all = "snake_case")]
55pub enum PromptContextMixin {
56 OaiChat,
58
59 Llama3DateTime,
61}
62
63pub fn thinking_bool_from_args(args: Option<&HashMap<String, serde_json::Value>>) -> Option<bool> {
71 let args = args?;
72 for key in ["thinking", "enable_thinking"] {
73 if let Some(v) = args.get(key).and_then(|x| x.as_bool()) {
74 return Some(v);
75 }
76 }
77 None
78}
79
80#[derive(Debug)]
81pub enum TokenInput {
82 Single(Vec<u32>),
83 Batch(Vec<Vec<u32>>),
84}
85
86#[derive(Debug)]
87pub enum TextInput {
88 Single(String),
89 Batch(Vec<String>),
90}
91
92#[derive(Debug)]
93pub enum PromptInput {
94 Tokens(TokenInput),
95 Text(TextInput),
96}
97
98#[derive(Debug, Clone, PartialEq, Eq)]
100pub struct RenderedSegment {
101 pub text: String,
102 pub allow_special: bool,
103}
104
105impl RenderedSegment {
106 pub fn new(text: impl Into<String>, allow_special: bool) -> Self {
107 Self {
108 text: text.into(),
109 allow_special,
110 }
111 }
112
113 pub fn as_encode_segment(&self) -> dynamo_tokenizers::EncodeSegment<'_> {
114 dynamo_tokenizers::EncodeSegment::new(&self.text, self.allow_special)
115 }
116}
117
118#[derive(Debug, Clone, PartialEq, Eq)]
124pub struct RenderedPrompt {
125 text: String,
126 segments: Option<Vec<RenderedSegment>>,
127 pending_segments: usize,
136}
137
138impl RenderedPrompt {
139 pub fn text(text: String) -> Self {
140 Self {
141 text,
142 segments: None,
143 pending_segments: 0,
144 }
145 }
146
147 pub fn segmented(segments: Vec<RenderedSegment>) -> Self {
148 Self::segmented_with_pending(segments, 0)
149 }
150
151 pub fn segmented_with_pending(segments: Vec<RenderedSegment>, pending_segments: usize) -> Self {
159 assert!(
160 pending_segments <= segments.len(),
161 "pending_segments ({pending_segments}) exceeds rendered segments ({})",
162 segments.len()
163 );
164 let text = segments
165 .iter()
166 .map(|segment| segment.text.as_str())
167 .collect();
168 Self {
169 text,
170 segments: Some(segments),
171 pending_segments,
172 }
173 }
174
175 pub fn as_str(&self) -> &str {
176 &self.text
177 }
178
179 pub fn segments(&self) -> Option<&[RenderedSegment]> {
180 self.segments.as_deref()
181 }
182
183 pub fn encode_segments(&self) -> Option<Vec<dynamo_tokenizers::EncodeSegment<'_>>> {
184 Some(
185 self.segments()?
186 .iter()
187 .map(RenderedSegment::as_encode_segment)
188 .collect(),
189 )
190 }
191
192 pub fn pending_segments(&self) -> usize {
197 self.pending_segments
198 }
199
200 pub fn pending_encode_segments(&self) -> Option<Vec<dynamo_tokenizers::EncodeSegment<'_>>> {
205 let segments = self.segments()?;
206 let start = segments.len() - self.pending_segments;
207 Some(
208 segments[start..]
209 .iter()
210 .map(RenderedSegment::as_encode_segment)
211 .collect(),
212 )
213 }
214
215 pub fn into_text(self) -> String {
216 self.text
217 }
218}
219
220#[derive(Debug, Clone, PartialEq, Eq)]
226pub enum PromptRenderError {
227 InvalidRequest(String),
228}
229
230impl PromptRenderError {
231 pub fn invalid_request(message: impl Into<String>) -> Self {
232 Self::InvalidRequest(message.into())
233 }
234}
235
236impl std::fmt::Display for PromptRenderError {
237 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
238 match self {
239 Self::InvalidRequest(message) => f.write_str(message),
240 }
241 }
242}
243
244impl std::error::Error for PromptRenderError {}
245
246pub trait OAIChatLikeRequest {
253 fn model(&self) -> String;
254 fn messages(&self) -> Value;
255 fn typed_messages(&self) -> Option<&[dynamo_protocols::types::ChatCompletionRequestMessage]> {
261 None
262 }
263 fn tools(&self) -> Option<Value> {
264 None
265 }
266 fn tool_choice(&self) -> Option<Value> {
267 None
268 }
269 fn response_format(&self) -> Option<Value> {
270 None
271 }
272
273 fn reasoning_effort(&self) -> Option<Value> {
276 None
277 }
278
279 fn should_add_generation_prompt(&self) -> bool;
280
281 fn chat_template_args(&self) -> Option<&HashMap<String, serde_json::Value>> {
283 None
284 }
285
286 fn prompt_input_type(&self) -> PromptInput {
288 PromptInput::Text(TextInput::Single(String::new()))
289 }
290
291 fn extract_tokens(&self) -> Option<TokenInput> {
293 None
294 }
295
296 fn extract_text(&self) -> Option<TextInput> {
297 None
298 }
299
300 fn mm_processor_kwargs(&self) -> Option<&serde_json::Value> {
301 None
302 }
303}
304
305pub(crate) fn messages_to_json(req: &dyn OAIChatLikeRequest) -> Result<serde_json::Value> {
308 if let Some(messages) = req.typed_messages() {
309 serde_json::to_value(messages)
310 } else {
311 serde_json::to_value(req.messages())
312 }
313 .context("Failed to convert messages to JSON")
314}
315
316pub trait OAIPromptFormatter: Send + Sync + 'static {
317 fn supports_add_generation_prompt(&self) -> bool;
318 fn render(&self, req: &dyn OAIChatLikeRequest) -> Result<String>;
319
320 fn media_message_order(&self, _request: &dyn OAIChatLikeRequest) -> Option<Vec<usize>> {
324 None
325 }
326
327 fn render_prompt(&self, req: &dyn OAIChatLikeRequest) -> Result<RenderedPrompt> {
328 self.render(req).map(RenderedPrompt::text)
329 }
330}
331
332pub(crate) fn reject_unsupported_partial_assistant(messages: &serde_json::Value) -> Result<()> {
339 let has_partial =
340 messages.as_array().into_iter().flatten().any(|message| {
341 message.get("partial").and_then(serde_json::Value::as_bool) == Some(true)
342 });
343 if has_partial {
344 return Err(PromptRenderError::invalid_request(
345 "assistant `partial: true` is not supported by this model's prompt formatter",
346 )
347 .into());
348 }
349 Ok(())
350}
351
352pub(crate) fn reject_unsupported_message_tools(
355 messages: &serde_json::Value,
356 supported_tool_roles: &[&str],
357) -> Result<()> {
358 let offending = messages.as_array().into_iter().flatten().find(|message| {
359 let declares_tools = message
360 .get("tools")
361 .is_some_and(|tools| !tools.is_null() && !tools.as_array().is_some_and(Vec::is_empty));
362 let role_is_supported = message
363 .get("role")
364 .and_then(serde_json::Value::as_str)
365 .is_some_and(|role| supported_tool_roles.contains(&role));
366 declares_tools && !role_is_supported
367 });
368
369 if let Some(message) = offending {
370 let role = message
371 .get("role")
372 .and_then(serde_json::Value::as_str)
373 .unwrap_or("<missing>");
374 return Err(PromptRenderError::invalid_request(format!(
375 "message-level `tools` on role {role:?} are not supported by this model's prompt \
376 formatter"
377 ))
378 .into());
379 }
380 Ok(())
381}
382
383#[derive(Clone)]
384pub enum PromptFormatter {
385 OAI(Arc<dyn OAIPromptFormatter>),
386}
387
388#[derive(Debug, Default)]
390pub struct NoOpFormatter;
391
392impl OAIPromptFormatter for NoOpFormatter {
393 fn supports_add_generation_prompt(&self) -> bool {
394 false
395 }
396
397 fn render(&self, req: &dyn OAIChatLikeRequest) -> Result<String> {
398 let messages = req.messages();
399 let messages_json = serde_json::to_value(&messages)?;
400 reject_unsupported_partial_assistant(&messages_json)?;
401 reject_unsupported_message_tools(&messages_json, &[])?;
402
403 let first_message = messages
404 .get_item_by_index(0)
405 .map_err(|_| anyhow::Error::msg("No message at index 0 or messages array is empty"))?;
406
407 let content = first_message
408 .get_attr("content")
409 .map_err(|_| anyhow::Error::msg("First message has no 'content' field"))?;
410
411 let content_str = content
412 .as_str()
413 .ok_or_else(|| anyhow::Error::msg("Message content is not a string"))?
414 .to_string();
415 Ok(content_str)
416 }
417}
418
419impl PromptFormatter {
420 pub fn no_op() -> Self {
421 Self::OAI(Arc::new(NoOpFormatter))
422 }
423}
424
425#[cfg(test)]
426mod rendered_prompt_tests {
427 use super::{
428 NoOpFormatter, OAIPromptFormatter, PromptRenderError, RenderedPrompt, RenderedSegment,
429 };
430
431 #[test]
432 fn owned_segments_borrow_into_tokenizer_segments() {
433 let prompt = RenderedPrompt::segmented(vec![
434 RenderedSegment::new("<|open|>", true),
435 RenderedSegment::new("user text", false),
436 ]);
437
438 let segments = prompt.encode_segments().expect("segmented prompt");
439 assert_eq!(segments[0].text, "<|open|>");
440 assert!(segments[0].allow_special);
441 assert_eq!(segments[1].text, "user text");
442 assert!(!segments[1].allow_special);
443 assert_eq!(prompt.as_str(), "<|open|>user text");
444 }
445
446 #[test]
447 fn no_op_formatter_rejects_unsupported_partial_assistant() {
448 let request: dynamo_protocols::types::CreateChatCompletionRequest =
449 serde_json::from_value(serde_json::json!({
450 "model": "test",
451 "messages": [
452 {"role": "user", "content": "Continue"},
453 {"role": "assistant", "content": "prefix", "partial": true}
454 ]
455 }))
456 .unwrap();
457
458 let error = NoOpFormatter.render(&request).unwrap_err();
459 assert!(matches!(
460 error.downcast_ref::<PromptRenderError>(),
461 Some(PromptRenderError::InvalidRequest(message))
462 if message.contains("`partial: true` is not supported")
463 ));
464 }
465
466 #[test]
467 fn no_op_formatter_rejects_message_level_tools() {
468 let request: dynamo_protocols::types::CreateChatCompletionRequest =
469 serde_json::from_value(serde_json::json!({
470 "model": "test",
471 "messages": [
472 {"role": "system", "tools": [{"name": "lookup"}]},
473 {"role": "user", "content": "Continue"}
474 ]
475 }))
476 .unwrap();
477
478 let error = NoOpFormatter.render(&request).unwrap_err();
479 assert!(matches!(
480 error.downcast_ref::<PromptRenderError>(),
481 Some(PromptRenderError::InvalidRequest(message))
482 if message.contains("message-level `tools`")
483 ));
484 }
485
486 #[test]
487 fn rendered_prompt_pending_segments_default_to_zero() {
488 let prompt = RenderedPrompt::segmented(vec![RenderedSegment::new("a", false)]);
489 assert_eq!(prompt.pending_segments(), 0);
490 assert!(prompt.pending_encode_segments().unwrap().is_empty());
491 assert_eq!(RenderedPrompt::text("a".to_string()).pending_segments(), 0);
492 assert!(
493 RenderedPrompt::text("a".to_string())
494 .pending_encode_segments()
495 .is_none()
496 );
497 }
498
499 #[test]
500 fn rendered_prompt_exposes_trailing_pending_segments() {
501 let prompt = RenderedPrompt::segmented_with_pending(
502 vec![
503 RenderedSegment::new("prompt", false),
504 RenderedSegment::new("<|open|>", true),
505 RenderedSegment::new("response", false),
506 RenderedSegment::new("<|sep|>", true),
507 ],
508 3,
509 );
510 assert_eq!(prompt.as_str(), "prompt<|open|>response<|sep|>");
511 assert_eq!(prompt.pending_segments(), 3);
512 let pending: Vec<&str> = prompt
513 .pending_encode_segments()
514 .unwrap()
515 .iter()
516 .map(|segment| segment.text)
517 .collect();
518 assert_eq!(pending, ["<|open|>", "response", "<|sep|>"]);
519 }
520
521 #[test]
522 #[should_panic(expected = "pending_segments")]
523 fn rendered_prompt_rejects_more_pending_than_segments() {
524 RenderedPrompt::segmented_with_pending(vec![RenderedSegment::new("a", false)], 2);
525 }
526}