1use anyhow::{Context, Result};
28use minijinja::value::Value;
29use std::collections::HashMap;
30use std::sync::Arc;
31
32pub use dynamo_tokenizers;
36
37pub mod beam;
38pub mod deepseek;
39pub mod inkling;
40pub mod kimi_k3;
41pub mod mistral;
42mod python;
43mod template;
44
45pub use template::{
46 ChatTemplate, ChatTemplateContentFormat, ChatTemplateValue, ContextMixins, MediaPlaceholders,
47 deepseek_formatter_for, kimi_k3_formatter_for, may_be_fix_tool_schema, native_formatter_for,
48};
49
50#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq, Eq, Hash)]
55#[serde(rename_all = "snake_case")]
56pub enum PromptContextMixin {
57 OaiChat,
59
60 Llama3DateTime,
62}
63
64pub fn thinking_bool_from_args(args: Option<&HashMap<String, serde_json::Value>>) -> Option<bool> {
72 let args = args?;
73 for key in ["thinking", "enable_thinking"] {
74 if let Some(v) = args.get(key).and_then(|x| x.as_bool()) {
75 return Some(v);
76 }
77 }
78 None
79}
80
81#[derive(Debug)]
82pub enum TokenInput {
83 Single(Vec<u32>),
84 Batch(Vec<Vec<u32>>),
85}
86
87#[derive(Debug)]
88pub enum TextInput {
89 Single(String),
90 Batch(Vec<String>),
91}
92
93#[derive(Debug)]
94pub enum PromptInput {
95 Tokens(TokenInput),
96 Text(TextInput),
97}
98
99#[derive(Debug, Clone, PartialEq, Eq)]
101pub struct RenderedSegment {
102 pub text: String,
103 pub allow_special: bool,
104}
105
106impl RenderedSegment {
107 pub fn new(text: impl Into<String>, allow_special: bool) -> Self {
108 Self {
109 text: text.into(),
110 allow_special,
111 }
112 }
113
114 pub fn as_encode_segment(&self) -> dynamo_tokenizers::EncodeSegment<'_> {
115 dynamo_tokenizers::EncodeSegment::new(&self.text, self.allow_special)
116 }
117}
118
119#[derive(Debug, Clone, PartialEq, Eq)]
125pub struct RenderedPrompt {
126 text: String,
127 segments: Option<Vec<RenderedSegment>>,
128 pending_segments: usize,
137}
138
139impl RenderedPrompt {
140 pub fn text(text: String) -> Self {
141 Self {
142 text,
143 segments: None,
144 pending_segments: 0,
145 }
146 }
147
148 pub fn segmented(segments: Vec<RenderedSegment>) -> Self {
149 Self::segmented_with_pending(segments, 0)
150 }
151
152 pub fn segmented_with_pending(segments: Vec<RenderedSegment>, pending_segments: usize) -> Self {
160 assert!(
161 pending_segments <= segments.len(),
162 "pending_segments ({pending_segments}) exceeds rendered segments ({})",
163 segments.len()
164 );
165 let text = segments
166 .iter()
167 .map(|segment| segment.text.as_str())
168 .collect();
169 Self {
170 text,
171 segments: Some(segments),
172 pending_segments,
173 }
174 }
175
176 pub fn as_str(&self) -> &str {
177 &self.text
178 }
179
180 pub fn segments(&self) -> Option<&[RenderedSegment]> {
181 self.segments.as_deref()
182 }
183
184 pub fn encode_segments(&self) -> Option<Vec<dynamo_tokenizers::EncodeSegment<'_>>> {
185 Some(
186 self.segments()?
187 .iter()
188 .map(RenderedSegment::as_encode_segment)
189 .collect(),
190 )
191 }
192
193 pub fn pending_segments(&self) -> usize {
198 self.pending_segments
199 }
200
201 pub fn pending_encode_segments(&self) -> Option<Vec<dynamo_tokenizers::EncodeSegment<'_>>> {
206 let segments = self.segments()?;
207 let start = segments.len() - self.pending_segments;
208 Some(
209 segments[start..]
210 .iter()
211 .map(RenderedSegment::as_encode_segment)
212 .collect(),
213 )
214 }
215
216 pub fn into_text(self) -> String {
217 self.text
218 }
219}
220
221#[derive(Debug, Clone, PartialEq, Eq)]
227pub enum PromptRenderError {
228 InvalidRequest(String),
229}
230
231impl PromptRenderError {
232 pub fn invalid_request(message: impl Into<String>) -> Self {
233 Self::InvalidRequest(message.into())
234 }
235}
236
237impl std::fmt::Display for PromptRenderError {
238 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
239 match self {
240 Self::InvalidRequest(message) => f.write_str(message),
241 }
242 }
243}
244
245impl std::error::Error for PromptRenderError {}
246
247pub trait OAIChatLikeRequest {
254 fn model(&self) -> String;
255 fn messages(&self) -> Value;
256 fn template_messages_json_override(&self) -> Option<serde_json::Value> {
259 None
260 }
261
262 fn typed_messages(&self) -> Option<&[dynamo_protocols::types::ChatCompletionRequestMessage]> {
268 None
269 }
270 fn tools(&self) -> Option<Value> {
271 None
272 }
273 fn raw_tools(&self) -> Option<Value> {
278 None
279 }
280 fn template_tools(&self) -> Option<Value> {
283 None
284 }
285 fn tool_choice(&self) -> Option<Value> {
286 None
287 }
288 fn response_format(&self) -> Option<Value> {
289 None
290 }
291
292 fn reasoning_effort(&self) -> Option<Value> {
295 None
296 }
297
298 fn should_add_generation_prompt(&self) -> bool;
299
300 fn chat_template_args(&self) -> Option<&HashMap<String, serde_json::Value>> {
302 None
303 }
304
305 fn prompt_input_type(&self) -> PromptInput {
307 PromptInput::Text(TextInput::Single(String::new()))
308 }
309
310 fn extract_tokens(&self) -> Option<TokenInput> {
312 None
313 }
314
315 fn extract_text(&self) -> Option<TextInput> {
316 None
317 }
318
319 fn mm_processor_kwargs(&self) -> Option<&serde_json::Value> {
320 None
321 }
322}
323
324pub(crate) fn messages_to_json(req: &dyn OAIChatLikeRequest) -> Result<serde_json::Value> {
327 if let Some(messages) = req.template_messages_json_override() {
329 return Ok(messages);
330 }
331 if let Some(messages) = req.typed_messages() {
332 dynamo_protocols::types::chat_messages_for_function_template(messages)
334 } else {
335 crate::python::template_json(req.messages())
336 }
337 .context("Failed to convert messages to JSON")
338}
339
340pub trait OAIPromptFormatter: Send + Sync + 'static {
341 fn supports_add_generation_prompt(&self) -> bool;
342 fn supports_native_tools(&self, _request: &dyn OAIChatLikeRequest) -> bool {
344 false
345 }
346 fn render(&self, req: &dyn OAIChatLikeRequest) -> Result<String>;
347
348 fn media_message_order(&self, _request: &dyn OAIChatLikeRequest) -> Option<Vec<usize>> {
352 None
353 }
354
355 fn render_prompt(&self, req: &dyn OAIChatLikeRequest) -> Result<RenderedPrompt> {
356 self.render(req).map(RenderedPrompt::text)
357 }
358
359 fn renders_continuation_natively(&self) -> bool {
364 false
365 }
366}
367
368pub(crate) fn reject_unsupported_partial_assistant(messages: &serde_json::Value) -> Result<()> {
375 let has_partial =
376 messages.as_array().into_iter().flatten().any(|message| {
377 message.get("partial").and_then(serde_json::Value::as_bool) == Some(true)
378 });
379 if has_partial {
380 return Err(PromptRenderError::invalid_request(
381 "assistant `partial: true` is not supported by this model's prompt formatter",
382 )
383 .into());
384 }
385 Ok(())
386}
387
388pub(crate) fn reject_unsupported_message_tools(
391 messages: &serde_json::Value,
392 supported_tool_roles: &[&str],
393) -> Result<()> {
394 let offending = messages.as_array().into_iter().flatten().find(|message| {
395 let declares_tools = message
396 .get("tools")
397 .is_some_and(|tools| !tools.is_null() && !tools.as_array().is_some_and(Vec::is_empty));
398 let role_is_supported = message
399 .get("role")
400 .and_then(serde_json::Value::as_str)
401 .is_some_and(|role| supported_tool_roles.contains(&role));
402 declares_tools && !role_is_supported
403 });
404
405 if let Some(message) = offending {
406 let role = message
407 .get("role")
408 .and_then(serde_json::Value::as_str)
409 .unwrap_or("<missing>");
410 return Err(PromptRenderError::invalid_request(format!(
411 "message-level `tools` on role {role:?} are not supported by this model's prompt \
412 formatter"
413 ))
414 .into());
415 }
416 Ok(())
417}
418
419#[derive(Clone)]
420pub enum PromptFormatter {
421 OAI(Arc<dyn OAIPromptFormatter>),
422}
423
424#[derive(Debug, Default)]
426pub struct NoOpFormatter;
427
428impl OAIPromptFormatter for NoOpFormatter {
429 fn supports_add_generation_prompt(&self) -> bool {
430 false
431 }
432
433 fn render(&self, req: &dyn OAIChatLikeRequest) -> Result<String> {
434 let messages = req.messages();
435 let messages_json = serde_json::to_value(&messages)?;
436 reject_unsupported_partial_assistant(&messages_json)?;
437 reject_unsupported_message_tools(&messages_json, &[])?;
438
439 let first_message = messages
440 .get_item_by_index(0)
441 .map_err(|_| anyhow::Error::msg("No message at index 0 or messages array is empty"))?;
442
443 let content = first_message
444 .get_attr("content")
445 .map_err(|_| anyhow::Error::msg("First message has no 'content' field"))?;
446
447 let content_str = content
448 .as_str()
449 .ok_or_else(|| anyhow::Error::msg("Message content is not a string"))?
450 .to_string();
451 Ok(content_str)
452 }
453}
454
455impl PromptFormatter {
456 pub fn no_op() -> Self {
457 Self::OAI(Arc::new(NoOpFormatter))
458 }
459}
460
461#[cfg(test)]
462mod rendered_prompt_tests {
463 use super::{
464 NoOpFormatter, OAIPromptFormatter, PromptRenderError, RenderedPrompt, RenderedSegment,
465 };
466
467 #[test]
468 fn owned_segments_borrow_into_tokenizer_segments() {
469 let prompt = RenderedPrompt::segmented(vec![
470 RenderedSegment::new("<|open|>", true),
471 RenderedSegment::new("user text", false),
472 ]);
473
474 let segments = prompt.encode_segments().expect("segmented prompt");
475 assert_eq!(segments[0].text, "<|open|>");
476 assert!(segments[0].allow_special);
477 assert_eq!(segments[1].text, "user text");
478 assert!(!segments[1].allow_special);
479 assert_eq!(prompt.as_str(), "<|open|>user text");
480 }
481
482 #[test]
483 fn no_op_formatter_rejects_unsupported_partial_assistant() {
484 let request: dynamo_protocols::types::CreateChatCompletionRequest =
485 serde_json::from_value(serde_json::json!({
486 "model": "test",
487 "messages": [
488 {"role": "user", "content": "Continue"},
489 {"role": "assistant", "content": "prefix", "partial": true}
490 ]
491 }))
492 .unwrap();
493
494 let error = NoOpFormatter.render(&request).unwrap_err();
495 assert!(matches!(
496 error.downcast_ref::<PromptRenderError>(),
497 Some(PromptRenderError::InvalidRequest(message))
498 if message.contains("`partial: true` is not supported")
499 ));
500 }
501
502 #[test]
503 fn no_op_formatter_rejects_message_level_tools() {
504 let request: dynamo_protocols::types::CreateChatCompletionRequest =
505 serde_json::from_value(serde_json::json!({
506 "model": "test",
507 "messages": [
508 {"role": "system", "tools": [{"name": "lookup"}]},
509 {"role": "user", "content": "Continue"}
510 ]
511 }))
512 .unwrap();
513
514 let error = NoOpFormatter.render(&request).unwrap_err();
515 assert!(matches!(
516 error.downcast_ref::<PromptRenderError>(),
517 Some(PromptRenderError::InvalidRequest(message))
518 if message.contains("message-level `tools`")
519 ));
520 }
521
522 #[test]
523 fn rendered_prompt_pending_segments_default_to_zero() {
524 let prompt = RenderedPrompt::segmented(vec![RenderedSegment::new("a", false)]);
525 assert_eq!(prompt.pending_segments(), 0);
526 assert!(prompt.pending_encode_segments().unwrap().is_empty());
527 assert_eq!(RenderedPrompt::text("a".to_string()).pending_segments(), 0);
528 assert!(
529 RenderedPrompt::text("a".to_string())
530 .pending_encode_segments()
531 .is_none()
532 );
533 }
534
535 #[test]
536 fn rendered_prompt_exposes_trailing_pending_segments() {
537 let prompt = RenderedPrompt::segmented_with_pending(
538 vec![
539 RenderedSegment::new("prompt", false),
540 RenderedSegment::new("<|open|>", true),
541 RenderedSegment::new("response", false),
542 RenderedSegment::new("<|sep|>", true),
543 ],
544 3,
545 );
546 assert_eq!(prompt.as_str(), "prompt<|open|>response<|sep|>");
547 assert_eq!(prompt.pending_segments(), 3);
548 let pending: Vec<&str> = prompt
549 .pending_encode_segments()
550 .unwrap()
551 .iter()
552 .map(|segment| segment.text)
553 .collect();
554 assert_eq!(pending, ["<|open|>", "response", "<|sep|>"]);
555 }
556
557 #[test]
558 #[should_panic(expected = "pending_segments")]
559 fn rendered_prompt_rejects_more_pending_than_segments() {
560 RenderedPrompt::segmented_with_pending(vec![RenderedSegment::new("a", false)], 2);
561 }
562}
563
564pub mod response_tools;