1use anyhow::{Context, Result};
28use minijinja::value::Value;
29use std::collections::HashMap;
30use std::sync::Arc;
31
32pub use dynamo_tokenizers;
36
37pub mod beam;
38pub mod deepseek;
39pub mod inkling;
40pub mod kimi_k3;
41mod python;
42mod template;
43
44pub use template::{
45 ChatTemplate, ChatTemplateValue, ContextMixins, deepseek_formatter_for, kimi_k3_formatter_for,
46 may_be_fix_tool_schema, native_formatter_for,
47};
48
49#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq, Eq, Hash)]
54#[serde(rename_all = "snake_case")]
55pub enum PromptContextMixin {
56 OaiChat,
58
59 Llama3DateTime,
61}
62
63pub fn thinking_bool_from_args(args: Option<&HashMap<String, serde_json::Value>>) -> Option<bool> {
71 let args = args?;
72 for key in ["thinking", "enable_thinking"] {
73 if let Some(v) = args.get(key).and_then(|x| x.as_bool()) {
74 return Some(v);
75 }
76 }
77 None
78}
79
80#[derive(Debug)]
81pub enum TokenInput {
82 Single(Vec<u32>),
83 Batch(Vec<Vec<u32>>),
84}
85
86#[derive(Debug)]
87pub enum TextInput {
88 Single(String),
89 Batch(Vec<String>),
90}
91
92#[derive(Debug)]
93pub enum PromptInput {
94 Tokens(TokenInput),
95 Text(TextInput),
96}
97
98#[derive(Debug, Clone, PartialEq, Eq)]
100pub struct RenderedSegment {
101 pub text: String,
102 pub allow_special: bool,
103}
104
105impl RenderedSegment {
106 pub fn new(text: impl Into<String>, allow_special: bool) -> Self {
107 Self {
108 text: text.into(),
109 allow_special,
110 }
111 }
112
113 pub fn as_encode_segment(&self) -> dynamo_tokenizers::EncodeSegment<'_> {
114 dynamo_tokenizers::EncodeSegment::new(&self.text, self.allow_special)
115 }
116}
117
118#[derive(Debug, Clone, PartialEq, Eq)]
124pub struct RenderedPrompt {
125 text: String,
126 segments: Option<Vec<RenderedSegment>>,
127 pending_segments: usize,
136}
137
138impl RenderedPrompt {
139 pub fn text(text: String) -> Self {
140 Self {
141 text,
142 segments: None,
143 pending_segments: 0,
144 }
145 }
146
147 pub fn segmented(segments: Vec<RenderedSegment>) -> Self {
148 Self::segmented_with_pending(segments, 0)
149 }
150
151 pub fn segmented_with_pending(segments: Vec<RenderedSegment>, pending_segments: usize) -> Self {
159 assert!(
160 pending_segments <= segments.len(),
161 "pending_segments ({pending_segments}) exceeds rendered segments ({})",
162 segments.len()
163 );
164 let text = segments
165 .iter()
166 .map(|segment| segment.text.as_str())
167 .collect();
168 Self {
169 text,
170 segments: Some(segments),
171 pending_segments,
172 }
173 }
174
175 pub fn as_str(&self) -> &str {
176 &self.text
177 }
178
179 pub fn segments(&self) -> Option<&[RenderedSegment]> {
180 self.segments.as_deref()
181 }
182
183 pub fn encode_segments(&self) -> Option<Vec<dynamo_tokenizers::EncodeSegment<'_>>> {
184 Some(
185 self.segments()?
186 .iter()
187 .map(RenderedSegment::as_encode_segment)
188 .collect(),
189 )
190 }
191
192 pub fn pending_segments(&self) -> usize {
197 self.pending_segments
198 }
199
200 pub fn pending_encode_segments(&self) -> Option<Vec<dynamo_tokenizers::EncodeSegment<'_>>> {
205 let segments = self.segments()?;
206 let start = segments.len() - self.pending_segments;
207 Some(
208 segments[start..]
209 .iter()
210 .map(RenderedSegment::as_encode_segment)
211 .collect(),
212 )
213 }
214
215 pub fn into_text(self) -> String {
216 self.text
217 }
218}
219
220#[derive(Debug, Clone, PartialEq, Eq)]
226pub enum PromptRenderError {
227 InvalidRequest(String),
228}
229
230impl PromptRenderError {
231 pub fn invalid_request(message: impl Into<String>) -> Self {
232 Self::InvalidRequest(message.into())
233 }
234}
235
236impl std::fmt::Display for PromptRenderError {
237 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
238 match self {
239 Self::InvalidRequest(message) => f.write_str(message),
240 }
241 }
242}
243
244impl std::error::Error for PromptRenderError {}
245
246pub trait OAIChatLikeRequest {
253 fn model(&self) -> String;
254 fn messages(&self) -> Value;
255 fn template_messages_json_override(&self) -> Option<serde_json::Value> {
258 None
259 }
260
261 fn typed_messages(&self) -> Option<&[dynamo_protocols::types::ChatCompletionRequestMessage]> {
267 None
268 }
269 fn tools(&self) -> Option<Value> {
270 None
271 }
272 fn template_tools(&self) -> Option<Value> {
275 None
276 }
277 fn tool_choice(&self) -> Option<Value> {
278 None
279 }
280 fn response_format(&self) -> Option<Value> {
281 None
282 }
283
284 fn reasoning_effort(&self) -> Option<Value> {
287 None
288 }
289
290 fn should_add_generation_prompt(&self) -> bool;
291
292 fn chat_template_args(&self) -> Option<&HashMap<String, serde_json::Value>> {
294 None
295 }
296
297 fn prompt_input_type(&self) -> PromptInput {
299 PromptInput::Text(TextInput::Single(String::new()))
300 }
301
302 fn extract_tokens(&self) -> Option<TokenInput> {
304 None
305 }
306
307 fn extract_text(&self) -> Option<TextInput> {
308 None
309 }
310
311 fn mm_processor_kwargs(&self) -> Option<&serde_json::Value> {
312 None
313 }
314}
315
316pub(crate) fn messages_to_json(req: &dyn OAIChatLikeRequest) -> Result<serde_json::Value> {
319 if let Some(messages) = req.template_messages_json_override() {
321 return Ok(messages);
322 }
323 if let Some(messages) = req.typed_messages() {
324 dynamo_protocols::types::chat_messages_for_function_template(messages)
326 } else {
327 crate::python::template_json(req.messages())
328 }
329 .context("Failed to convert messages to JSON")
330}
331
332pub trait OAIPromptFormatter: Send + Sync + 'static {
333 fn supports_add_generation_prompt(&self) -> bool;
334 fn supports_native_tools(&self, _request: &dyn OAIChatLikeRequest) -> bool {
336 false
337 }
338 fn render(&self, req: &dyn OAIChatLikeRequest) -> Result<String>;
339
340 fn media_message_order(&self, _request: &dyn OAIChatLikeRequest) -> Option<Vec<usize>> {
344 None
345 }
346
347 fn render_prompt(&self, req: &dyn OAIChatLikeRequest) -> Result<RenderedPrompt> {
348 self.render(req).map(RenderedPrompt::text)
349 }
350}
351
352pub(crate) fn reject_unsupported_partial_assistant(messages: &serde_json::Value) -> Result<()> {
359 let has_partial =
360 messages.as_array().into_iter().flatten().any(|message| {
361 message.get("partial").and_then(serde_json::Value::as_bool) == Some(true)
362 });
363 if has_partial {
364 return Err(PromptRenderError::invalid_request(
365 "assistant `partial: true` is not supported by this model's prompt formatter",
366 )
367 .into());
368 }
369 Ok(())
370}
371
372pub(crate) fn reject_unsupported_message_tools(
375 messages: &serde_json::Value,
376 supported_tool_roles: &[&str],
377) -> Result<()> {
378 let offending = messages.as_array().into_iter().flatten().find(|message| {
379 let declares_tools = message
380 .get("tools")
381 .is_some_and(|tools| !tools.is_null() && !tools.as_array().is_some_and(Vec::is_empty));
382 let role_is_supported = message
383 .get("role")
384 .and_then(serde_json::Value::as_str)
385 .is_some_and(|role| supported_tool_roles.contains(&role));
386 declares_tools && !role_is_supported
387 });
388
389 if let Some(message) = offending {
390 let role = message
391 .get("role")
392 .and_then(serde_json::Value::as_str)
393 .unwrap_or("<missing>");
394 return Err(PromptRenderError::invalid_request(format!(
395 "message-level `tools` on role {role:?} are not supported by this model's prompt \
396 formatter"
397 ))
398 .into());
399 }
400 Ok(())
401}
402
403#[derive(Clone)]
404pub enum PromptFormatter {
405 OAI(Arc<dyn OAIPromptFormatter>),
406}
407
408#[derive(Debug, Default)]
410pub struct NoOpFormatter;
411
412impl OAIPromptFormatter for NoOpFormatter {
413 fn supports_add_generation_prompt(&self) -> bool {
414 false
415 }
416
417 fn render(&self, req: &dyn OAIChatLikeRequest) -> Result<String> {
418 let messages = req.messages();
419 let messages_json = serde_json::to_value(&messages)?;
420 reject_unsupported_partial_assistant(&messages_json)?;
421 reject_unsupported_message_tools(&messages_json, &[])?;
422
423 let first_message = messages
424 .get_item_by_index(0)
425 .map_err(|_| anyhow::Error::msg("No message at index 0 or messages array is empty"))?;
426
427 let content = first_message
428 .get_attr("content")
429 .map_err(|_| anyhow::Error::msg("First message has no 'content' field"))?;
430
431 let content_str = content
432 .as_str()
433 .ok_or_else(|| anyhow::Error::msg("Message content is not a string"))?
434 .to_string();
435 Ok(content_str)
436 }
437}
438
439impl PromptFormatter {
440 pub fn no_op() -> Self {
441 Self::OAI(Arc::new(NoOpFormatter))
442 }
443}
444
445#[cfg(test)]
446mod rendered_prompt_tests {
447 use super::{
448 NoOpFormatter, OAIPromptFormatter, PromptRenderError, RenderedPrompt, RenderedSegment,
449 };
450
451 #[test]
452 fn owned_segments_borrow_into_tokenizer_segments() {
453 let prompt = RenderedPrompt::segmented(vec![
454 RenderedSegment::new("<|open|>", true),
455 RenderedSegment::new("user text", false),
456 ]);
457
458 let segments = prompt.encode_segments().expect("segmented prompt");
459 assert_eq!(segments[0].text, "<|open|>");
460 assert!(segments[0].allow_special);
461 assert_eq!(segments[1].text, "user text");
462 assert!(!segments[1].allow_special);
463 assert_eq!(prompt.as_str(), "<|open|>user text");
464 }
465
466 #[test]
467 fn no_op_formatter_rejects_unsupported_partial_assistant() {
468 let request: dynamo_protocols::types::CreateChatCompletionRequest =
469 serde_json::from_value(serde_json::json!({
470 "model": "test",
471 "messages": [
472 {"role": "user", "content": "Continue"},
473 {"role": "assistant", "content": "prefix", "partial": true}
474 ]
475 }))
476 .unwrap();
477
478 let error = NoOpFormatter.render(&request).unwrap_err();
479 assert!(matches!(
480 error.downcast_ref::<PromptRenderError>(),
481 Some(PromptRenderError::InvalidRequest(message))
482 if message.contains("`partial: true` is not supported")
483 ));
484 }
485
486 #[test]
487 fn no_op_formatter_rejects_message_level_tools() {
488 let request: dynamo_protocols::types::CreateChatCompletionRequest =
489 serde_json::from_value(serde_json::json!({
490 "model": "test",
491 "messages": [
492 {"role": "system", "tools": [{"name": "lookup"}]},
493 {"role": "user", "content": "Continue"}
494 ]
495 }))
496 .unwrap();
497
498 let error = NoOpFormatter.render(&request).unwrap_err();
499 assert!(matches!(
500 error.downcast_ref::<PromptRenderError>(),
501 Some(PromptRenderError::InvalidRequest(message))
502 if message.contains("message-level `tools`")
503 ));
504 }
505
506 #[test]
507 fn rendered_prompt_pending_segments_default_to_zero() {
508 let prompt = RenderedPrompt::segmented(vec![RenderedSegment::new("a", false)]);
509 assert_eq!(prompt.pending_segments(), 0);
510 assert!(prompt.pending_encode_segments().unwrap().is_empty());
511 assert_eq!(RenderedPrompt::text("a".to_string()).pending_segments(), 0);
512 assert!(
513 RenderedPrompt::text("a".to_string())
514 .pending_encode_segments()
515 .is_none()
516 );
517 }
518
519 #[test]
520 fn rendered_prompt_exposes_trailing_pending_segments() {
521 let prompt = RenderedPrompt::segmented_with_pending(
522 vec![
523 RenderedSegment::new("prompt", false),
524 RenderedSegment::new("<|open|>", true),
525 RenderedSegment::new("response", false),
526 RenderedSegment::new("<|sep|>", true),
527 ],
528 3,
529 );
530 assert_eq!(prompt.as_str(), "prompt<|open|>response<|sep|>");
531 assert_eq!(prompt.pending_segments(), 3);
532 let pending: Vec<&str> = prompt
533 .pending_encode_segments()
534 .unwrap()
535 .iter()
536 .map(|segment| segment.text)
537 .collect();
538 assert_eq!(pending, ["<|open|>", "response", "<|sep|>"]);
539 }
540
541 #[test]
542 #[should_panic(expected = "pending_segments")]
543 fn rendered_prompt_rejects_more_pending_than_segments() {
544 RenderedPrompt::segmented_with_pending(vec![RenderedSegment::new("a", false)], 2);
545 }
546}
547
548pub mod response_tools;