1use anyhow::{Context, Result};
30use minijinja::value::Value;
31use std::collections::HashMap;
32use std::sync::Arc;
33
34pub use dynamo_tokenizers;
38
39pub mod beam;
40pub mod deepseek;
41pub mod inkling;
42pub mod kimi_k3;
43pub mod mistral;
44mod python;
45mod template;
46
47pub use template::{
48 ChatTemplate, ChatTemplateContentFormat, ChatTemplateValue, ContextMixins, MediaPlaceholders,
49 deepseek_formatter_for, kimi_k3_formatter_for, may_be_fix_tool_schema, native_formatter_for,
50};
51
52#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq, Eq, Hash)]
57#[serde(rename_all = "snake_case")]
58pub enum PromptContextMixin {
59 OaiChat,
61
62 Llama3DateTime,
64}
65
66pub fn thinking_bool_from_args(args: Option<&HashMap<String, serde_json::Value>>) -> Option<bool> {
74 let args = args?;
75 for key in ["thinking", "enable_thinking"] {
76 if let Some(v) = args.get(key).and_then(|x| x.as_bool()) {
77 return Some(v);
78 }
79 }
80 None
81}
82
83#[derive(Debug)]
84pub enum TokenInput {
85 Single(Vec<u32>),
86 Batch(Vec<Vec<u32>>),
87}
88
89#[derive(Debug)]
90pub enum TextInput {
91 Single(String),
92 Batch(Vec<String>),
93}
94
95#[derive(Debug)]
96pub enum PromptInput {
97 Tokens(TokenInput),
98 Text(TextInput),
99}
100
101#[derive(Debug, Clone, PartialEq, Eq)]
103pub struct RenderedSegment {
104 pub text: String,
105 pub allow_special: bool,
106}
107
108impl RenderedSegment {
109 pub fn new(text: impl Into<String>, allow_special: bool) -> Self {
110 Self {
111 text: text.into(),
112 allow_special,
113 }
114 }
115
116 pub fn as_encode_segment(&self) -> dynamo_tokenizers::EncodeSegment<'_> {
117 dynamo_tokenizers::EncodeSegment::new(&self.text, self.allow_special)
118 }
119}
120
121#[derive(Debug, Clone, PartialEq, Eq)]
127pub struct RenderedPrompt {
128 text: String,
129 segments: Option<Vec<RenderedSegment>>,
130 pending_segments: usize,
139}
140
141impl RenderedPrompt {
142 pub fn text(text: String) -> Self {
143 Self {
144 text,
145 segments: None,
146 pending_segments: 0,
147 }
148 }
149
150 pub fn segmented(segments: Vec<RenderedSegment>) -> Self {
151 Self::segmented_with_pending(segments, 0)
152 }
153
154 pub fn segmented_with_pending(segments: Vec<RenderedSegment>, pending_segments: usize) -> Self {
162 assert!(
163 pending_segments <= segments.len(),
164 "pending_segments ({pending_segments}) exceeds rendered segments ({})",
165 segments.len()
166 );
167 let text = segments
168 .iter()
169 .map(|segment| segment.text.as_str())
170 .collect();
171 Self {
172 text,
173 segments: Some(segments),
174 pending_segments,
175 }
176 }
177
178 pub fn as_str(&self) -> &str {
179 &self.text
180 }
181
182 pub fn segments(&self) -> Option<&[RenderedSegment]> {
183 self.segments.as_deref()
184 }
185
186 pub fn encode_segments(&self) -> Option<Vec<dynamo_tokenizers::EncodeSegment<'_>>> {
187 Some(
188 self.segments()?
189 .iter()
190 .map(RenderedSegment::as_encode_segment)
191 .collect(),
192 )
193 }
194
195 pub fn pending_segments(&self) -> usize {
200 self.pending_segments
201 }
202
203 pub fn pending_encode_segments(&self) -> Option<Vec<dynamo_tokenizers::EncodeSegment<'_>>> {
208 let segments = self.segments()?;
209 let start = segments.len() - self.pending_segments;
210 Some(
211 segments[start..]
212 .iter()
213 .map(RenderedSegment::as_encode_segment)
214 .collect(),
215 )
216 }
217
218 pub fn into_text(self) -> String {
219 self.text
220 }
221}
222
223#[derive(Debug, Clone, PartialEq, Eq)]
229pub enum PromptRenderError {
230 InvalidRequest(String),
231}
232
233impl PromptRenderError {
234 pub fn invalid_request(message: impl Into<String>) -> Self {
235 Self::InvalidRequest(message.into())
236 }
237}
238
239impl std::fmt::Display for PromptRenderError {
240 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
241 match self {
242 Self::InvalidRequest(message) => f.write_str(message),
243 }
244 }
245}
246
247impl std::error::Error for PromptRenderError {}
248
249pub trait OAIChatLikeRequest {
256 fn model(&self) -> String;
257 fn messages(&self) -> Value;
258 fn template_messages_json_override(&self) -> Option<serde_json::Value> {
261 None
262 }
263
264 fn typed_messages(&self) -> Option<&[dynamo_protocols::types::ChatCompletionRequestMessage]> {
270 None
271 }
272 fn tools(&self) -> Option<Value> {
273 None
274 }
275 fn raw_tools(&self) -> Option<Value> {
280 None
281 }
282 fn template_tools(&self) -> Option<Value> {
285 None
286 }
287 fn tool_choice(&self) -> Option<Value> {
288 None
289 }
290 fn response_format(&self) -> Option<Value> {
291 None
292 }
293
294 fn reasoning_effort(&self) -> Option<Value> {
297 None
298 }
299
300 fn should_add_generation_prompt(&self) -> bool;
301
302 fn chat_template_args(&self) -> Option<&HashMap<String, serde_json::Value>> {
304 None
305 }
306
307 fn prompt_input_type(&self) -> PromptInput {
309 PromptInput::Text(TextInput::Single(String::new()))
310 }
311
312 fn extract_tokens(&self) -> Option<TokenInput> {
314 None
315 }
316
317 fn extract_text(&self) -> Option<TextInput> {
318 None
319 }
320
321 fn mm_processor_kwargs(&self) -> Option<&serde_json::Value> {
322 None
323 }
324}
325
326pub(crate) fn messages_to_json(req: &dyn OAIChatLikeRequest) -> Result<serde_json::Value> {
329 if let Some(messages) = req.template_messages_json_override() {
331 return Ok(messages);
332 }
333 if let Some(messages) = req.typed_messages() {
334 dynamo_protocols::types::chat_messages_for_function_template(messages)
336 } else {
337 crate::python::template_json(req.messages())
338 }
339 .context("Failed to convert messages to JSON")
340}
341
342pub trait OAIPromptFormatter: Send + Sync + 'static {
343 fn supports_add_generation_prompt(&self) -> bool;
344 fn supports_native_tools(&self, _request: &dyn OAIChatLikeRequest) -> bool {
346 false
347 }
348 fn render(&self, req: &dyn OAIChatLikeRequest) -> Result<String>;
349
350 fn media_message_order(&self, _request: &dyn OAIChatLikeRequest) -> Option<Vec<usize>> {
354 None
355 }
356
357 fn render_prompt(&self, req: &dyn OAIChatLikeRequest) -> Result<RenderedPrompt> {
358 self.render(req).map(RenderedPrompt::text)
359 }
360
361 fn renders_continuation_natively(&self) -> bool {
366 false
367 }
368}
369
370pub(crate) fn reject_unsupported_partial_assistant(messages: &serde_json::Value) -> Result<()> {
377 let has_partial =
378 messages.as_array().into_iter().flatten().any(|message| {
379 message.get("partial").and_then(serde_json::Value::as_bool) == Some(true)
380 });
381 if has_partial {
382 return Err(PromptRenderError::invalid_request(
383 "assistant `partial: true` is not supported by this model's prompt formatter",
384 )
385 .into());
386 }
387 Ok(())
388}
389
390pub(crate) fn reject_unsupported_message_tools(
393 messages: &serde_json::Value,
394 supported_tool_roles: &[&str],
395) -> Result<()> {
396 let offending = messages.as_array().into_iter().flatten().find(|message| {
397 let declares_tools = message
398 .get("tools")
399 .is_some_and(|tools| !tools.is_null() && !tools.as_array().is_some_and(Vec::is_empty));
400 let role_is_supported = message
401 .get("role")
402 .and_then(serde_json::Value::as_str)
403 .is_some_and(|role| supported_tool_roles.contains(&role));
404 declares_tools && !role_is_supported
405 });
406
407 if let Some(message) = offending {
408 let role = message
409 .get("role")
410 .and_then(serde_json::Value::as_str)
411 .unwrap_or("<missing>");
412 return Err(PromptRenderError::invalid_request(format!(
413 "message-level `tools` on role {role:?} are not supported by this model's prompt \
414 formatter"
415 ))
416 .into());
417 }
418 Ok(())
419}
420
421#[derive(Clone)]
422pub enum PromptFormatter {
423 OAI(Arc<dyn OAIPromptFormatter>),
424}
425
426#[derive(Debug, Default)]
428pub struct NoOpFormatter;
429
430impl OAIPromptFormatter for NoOpFormatter {
431 fn supports_add_generation_prompt(&self) -> bool {
432 false
433 }
434
435 fn render(&self, req: &dyn OAIChatLikeRequest) -> Result<String> {
436 let messages = req.messages();
437 let messages_json = serde_json::to_value(&messages)?;
438 reject_unsupported_partial_assistant(&messages_json)?;
439 reject_unsupported_message_tools(&messages_json, &[])?;
440
441 let first_message = messages
442 .get_item_by_index(0)
443 .map_err(|_| anyhow::Error::msg("No message at index 0 or messages array is empty"))?;
444
445 let content = first_message
446 .get_attr("content")
447 .map_err(|_| anyhow::Error::msg("First message has no 'content' field"))?;
448
449 let content_str = content
450 .as_str()
451 .ok_or_else(|| anyhow::Error::msg("Message content is not a string"))?
452 .to_string();
453 Ok(content_str)
454 }
455}
456
457impl PromptFormatter {
458 pub fn no_op() -> Self {
459 Self::OAI(Arc::new(NoOpFormatter))
460 }
461}
462
463#[cfg(test)]
464mod rendered_prompt_tests {
465 use super::{
466 NoOpFormatter, OAIPromptFormatter, PromptRenderError, RenderedPrompt, RenderedSegment,
467 };
468
469 #[test]
470 fn owned_segments_borrow_into_tokenizer_segments() {
471 let prompt = RenderedPrompt::segmented(vec![
472 RenderedSegment::new("<|open|>", true),
473 RenderedSegment::new("user text", false),
474 ]);
475
476 let segments = prompt.encode_segments().expect("segmented prompt");
477 assert_eq!(segments[0].text, "<|open|>");
478 assert!(segments[0].allow_special);
479 assert_eq!(segments[1].text, "user text");
480 assert!(!segments[1].allow_special);
481 assert_eq!(prompt.as_str(), "<|open|>user text");
482 }
483
484 #[test]
485 fn no_op_formatter_rejects_unsupported_partial_assistant() {
486 let request: dynamo_protocols::types::CreateChatCompletionRequest =
487 serde_json::from_value(serde_json::json!({
488 "model": "test",
489 "messages": [
490 {"role": "user", "content": "Continue"},
491 {"role": "assistant", "content": "prefix", "partial": true}
492 ]
493 }))
494 .unwrap();
495
496 let error = NoOpFormatter.render(&request).unwrap_err();
497 assert!(matches!(
498 error.downcast_ref::<PromptRenderError>(),
499 Some(PromptRenderError::InvalidRequest(message))
500 if message.contains("`partial: true` is not supported")
501 ));
502 }
503
504 #[test]
505 fn no_op_formatter_rejects_message_level_tools() {
506 let request: dynamo_protocols::types::CreateChatCompletionRequest =
507 serde_json::from_value(serde_json::json!({
508 "model": "test",
509 "messages": [
510 {"role": "system", "tools": [{"name": "lookup"}]},
511 {"role": "user", "content": "Continue"}
512 ]
513 }))
514 .unwrap();
515
516 let error = NoOpFormatter.render(&request).unwrap_err();
517 assert!(matches!(
518 error.downcast_ref::<PromptRenderError>(),
519 Some(PromptRenderError::InvalidRequest(message))
520 if message.contains("message-level `tools`")
521 ));
522 }
523
524 #[test]
525 fn rendered_prompt_pending_segments_default_to_zero() {
526 let prompt = RenderedPrompt::segmented(vec![RenderedSegment::new("a", false)]);
527 assert_eq!(prompt.pending_segments(), 0);
528 assert!(prompt.pending_encode_segments().unwrap().is_empty());
529 assert_eq!(RenderedPrompt::text("a".to_string()).pending_segments(), 0);
530 assert!(
531 RenderedPrompt::text("a".to_string())
532 .pending_encode_segments()
533 .is_none()
534 );
535 }
536
537 #[test]
538 fn rendered_prompt_exposes_trailing_pending_segments() {
539 let prompt = RenderedPrompt::segmented_with_pending(
540 vec![
541 RenderedSegment::new("prompt", false),
542 RenderedSegment::new("<|open|>", true),
543 RenderedSegment::new("response", false),
544 RenderedSegment::new("<|sep|>", true),
545 ],
546 3,
547 );
548 assert_eq!(prompt.as_str(), "prompt<|open|>response<|sep|>");
549 assert_eq!(prompt.pending_segments(), 3);
550 let pending: Vec<&str> = prompt
551 .pending_encode_segments()
552 .unwrap()
553 .iter()
554 .map(|segment| segment.text)
555 .collect();
556 assert_eq!(pending, ["<|open|>", "response", "<|sep|>"]);
557 }
558
559 #[test]
560 #[should_panic(expected = "pending_segments")]
561 fn rendered_prompt_rejects_more_pending_than_segments() {
562 RenderedPrompt::segmented_with_pending(vec![RenderedSegment::new("a", false)], 2);
563 }
564}
565
566pub mod response_tools;