chat_prompts/
lib.rs

1//! `chat-prompts` is part of [LlamaEdge API Server](https://github.com/LlamaEdge/LlamaEdge/tree/main/api-server) project. It provides a collection of prompt templates that are used to generate prompts for the LLMs (See models in [huggingface.co/second-state](https://huggingface.co/second-state)).
2//!
3//! For the details of available prompt templates, see [README.md](https://github.com/LlamaEdge/LlamaEdge/tree/main/api-server/chat-prompts).
4
5pub mod agent;
6pub mod chat;
7pub mod error;
8pub mod utils;
9
10use agent::*;
11use chat::*;
12use clap::ValueEnum;
13use endpoints::chat::{ChatCompletionRequestMessage, Tool};
14use error::Result;
15use serde::{Deserialize, Serialize};
16use std::str::FromStr;
17
18/// Define the chat prompt template types.
19#[derive(Clone, Debug, Copy, PartialEq, Eq, Serialize, Deserialize, ValueEnum)]
20pub enum PromptTemplateType {
21    #[value(name = "llama-2-chat")]
22    Llama2Chat,
23    #[value(name = "llama-3-chat")]
24    Llama3Chat,
25    #[value(name = "llama-3-tool")]
26    Llama3Tool,
27    #[value(name = "llama-4-chat")]
28    Llama4Chat,
29    #[value(name = "mistral-instruct")]
30    MistralInstruct,
31    #[value(name = "mistral-tool")]
32    MistralTool,
33    #[value(name = "mistrallite")]
34    MistralLite,
35    #[value(name = "mistral-small-chat")]
36    MistralSmallChat,
37    #[value(name = "mistral-small-tool")]
38    MistralSmallTool,
39    #[value(name = "openchat")]
40    OpenChat,
41    #[value(name = "codellama-instruct")]
42    CodeLlama,
43    #[value(name = "codellama-super-instruct")]
44    CodeLlamaSuper,
45    #[value(name = "human-assistant")]
46    HumanAssistant,
47    #[value(name = "vicuna-1.0-chat")]
48    VicunaChat,
49    #[value(name = "vicuna-1.1-chat")]
50    Vicuna11Chat,
51    #[value(name = "vicuna-llava")]
52    VicunaLlava,
53    #[value(name = "chatml")]
54    ChatML,
55    #[value(name = "chatml-tool")]
56    ChatMLTool,
57    #[value(name = "chatml-think")]
58    ChatMLThink,
59    #[value(name = "internlm-2-tool")]
60    InternLM2Tool,
61    #[value(name = "baichuan-2")]
62    Baichuan2,
63    #[value(name = "wizard-coder")]
64    WizardCoder,
65    #[value(name = "zephyr")]
66    Zephyr,
67    #[value(name = "stablelm-zephyr")]
68    StableLMZephyr,
69    #[value(name = "intel-neural")]
70    IntelNeural,
71    #[value(name = "deepseek-chat")]
72    DeepseekChat,
73    #[value(name = "deepseek-coder")]
74    DeepseekCoder,
75    #[value(name = "deepseek-chat-2")]
76    DeepseekChat2,
77    #[value(name = "deepseek-chat-25")]
78    DeepseekChat25,
79    #[value(name = "deepseek-chat-3")]
80    DeepseekChat3,
81    #[value(name = "solar-instruct")]
82    SolarInstruct,
83    #[value(name = "phi-2-chat")]
84    Phi2Chat,
85    #[value(name = "phi-2-instruct")]
86    Phi2Instruct,
87    #[value(name = "phi-3-chat")]
88    Phi3Chat,
89    #[value(name = "phi-3-instruct")]
90    Phi3Instruct,
91    #[value(name = "phi-4-chat")]
92    Phi4Chat,
93    #[value(name = "gemma-instruct")]
94    GemmaInstruct,
95    #[value(name = "gemma-3")]
96    Gemma3,
97    #[value(name = "octopus")]
98    Octopus,
99    #[value(name = "glm-4-chat")]
100    Glm4Chat,
101    #[value(name = "groq-llama3-tool")]
102    GroqLlama3Tool,
103    #[value(name = "mediatek-breeze")]
104    BreezeInstruct,
105    #[value(name = "nemotron-chat")]
106    NemotronChat,
107    #[value(name = "nemotron-tool")]
108    NemotronTool,
109    #[value(name = "functionary-32")]
110    FunctionaryV32,
111    #[value(name = "functionary-31")]
112    FunctionaryV31,
113    #[value(name = "minicpmv")]
114    MiniCPMV,
115    #[value(name = "moxin-chat")]
116    MoxinChat,
117    #[value(name = "moxin-instruct")]
118    MoxinInstruct,
119    #[value(name = "falcon3")]
120    Falcon3,
121    #[value(name = "megrez")]
122    Megrez,
123    #[value(name = "qwen2-vision")]
124    Qwen2vl,
125    #[value(name = "qwen3-no-think")]
126    Qwen3NoThink,
127    #[value(name = "qwen3-agent")]
128    Qwen3Agent,
129    #[value(name = "exaone-deep-chat")]
130    ExaoneDeepChat,
131    #[value(name = "exaone-chat")]
132    ExaoneChat,
133    #[value(name = "seed-instruct")]
134    SeedInstruct,
135    #[value(name = "seed-reasoning")]
136    SeedReasoning,
137    #[value(name = "seed-oss-think")]
138    SeedOssThink,
139    #[value(name = "seed-oss-no-think")]
140    SeedOssNoThink,
141    #[value(name = "smol-vision")]
142    Smolvl,
143    #[value(name = "smol3-no-think")]
144    Smol3NoThink,
145    #[value(name = "gpt-oss")]
146    GptOss,
147    #[value(name = "embedding")]
148    Embedding,
149    #[value(name = "tts")]
150    Tts,
151    #[value(name = "none")]
152    Null,
153}
154impl PromptTemplateType {
155    /// Check if the prompt template has a system prompt.
156    pub fn has_system_prompt(&self) -> bool {
157        match self {
158            PromptTemplateType::Llama2Chat
159            | PromptTemplateType::Llama3Chat
160            | PromptTemplateType::Llama3Tool
161            | PromptTemplateType::CodeLlama
162            | PromptTemplateType::CodeLlamaSuper
163            | PromptTemplateType::VicunaChat
164            | PromptTemplateType::VicunaLlava
165            | PromptTemplateType::ChatML
166            | PromptTemplateType::ChatMLTool
167            | PromptTemplateType::InternLM2Tool
168            | PromptTemplateType::Baichuan2
169            | PromptTemplateType::WizardCoder
170            | PromptTemplateType::Zephyr
171            | PromptTemplateType::IntelNeural
172            | PromptTemplateType::DeepseekCoder
173            | PromptTemplateType::DeepseekChat2
174            | PromptTemplateType::DeepseekChat3
175            | PromptTemplateType::Octopus
176            | PromptTemplateType::Phi3Chat
177            | PromptTemplateType::Phi4Chat
178            | PromptTemplateType::Glm4Chat
179            | PromptTemplateType::GroqLlama3Tool
180            | PromptTemplateType::BreezeInstruct
181            | PromptTemplateType::DeepseekChat25
182            | PromptTemplateType::NemotronChat
183            | PromptTemplateType::NemotronTool
184            | PromptTemplateType::MiniCPMV
185            | PromptTemplateType::MoxinChat
186            | PromptTemplateType::Falcon3
187            | PromptTemplateType::Megrez
188            | PromptTemplateType::Qwen2vl
189            | PromptTemplateType::Qwen3NoThink
190            | PromptTemplateType::Qwen3Agent
191            | PromptTemplateType::MistralSmallChat
192            | PromptTemplateType::MistralSmallTool
193            | PromptTemplateType::ExaoneDeepChat
194            | PromptTemplateType::ExaoneChat
195            | PromptTemplateType::ChatMLThink
196            | PromptTemplateType::Llama4Chat
197            | PromptTemplateType::SeedInstruct
198            | PromptTemplateType::SeedOssThink
199            | PromptTemplateType::SeedOssNoThink
200            | PromptTemplateType::Smol3NoThink
201            | PromptTemplateType::GptOss => true,
202            PromptTemplateType::MistralInstruct
203            | PromptTemplateType::MistralTool
204            | PromptTemplateType::MistralLite
205            | PromptTemplateType::HumanAssistant
206            | PromptTemplateType::DeepseekChat
207            | PromptTemplateType::GemmaInstruct
208            | PromptTemplateType::Gemma3
209            | PromptTemplateType::OpenChat
210            | PromptTemplateType::Phi2Chat
211            | PromptTemplateType::Phi2Instruct
212            | PromptTemplateType::Phi3Instruct
213            | PromptTemplateType::SolarInstruct
214            | PromptTemplateType::Vicuna11Chat
215            | PromptTemplateType::StableLMZephyr
216            | PromptTemplateType::FunctionaryV32
217            | PromptTemplateType::FunctionaryV31
218            | PromptTemplateType::SeedReasoning
219            | PromptTemplateType::MoxinInstruct
220            | PromptTemplateType::Smolvl
221            | PromptTemplateType::Embedding
222            | PromptTemplateType::Tts
223            | PromptTemplateType::Null => false,
224        }
225    }
226
227    /// Check if the prompt template supports image input.
228    pub fn is_image_supported(&self) -> bool {
229        matches!(
230            self,
231            PromptTemplateType::MiniCPMV
232                | PromptTemplateType::Qwen2vl
233                | PromptTemplateType::VicunaLlava
234                | PromptTemplateType::Gemma3
235                | PromptTemplateType::Smolvl
236        )
237    }
238}
239impl FromStr for PromptTemplateType {
240    type Err = error::PromptError;
241
242    fn from_str(template: &str) -> std::result::Result<Self, Self::Err> {
243        match template {
244            "llama-2-chat" => Ok(PromptTemplateType::Llama2Chat),
245            "llama-3-chat" => Ok(PromptTemplateType::Llama3Chat),
246            "llama-3-tool" => Ok(PromptTemplateType::Llama3Tool),
247            "llama-4-chat" => Ok(PromptTemplateType::Llama4Chat),
248            "mistral-instruct" => Ok(PromptTemplateType::MistralInstruct),
249            "mistral-tool" => Ok(PromptTemplateType::MistralTool),
250            "mistrallite" => Ok(PromptTemplateType::MistralLite),
251            "mistral-small-chat" => Ok(PromptTemplateType::MistralSmallChat),
252            "mistral-small-tool" => Ok(PromptTemplateType::MistralSmallTool),
253            "codellama-instruct" => Ok(PromptTemplateType::CodeLlama),
254            "codellama-super-instruct" => Ok(PromptTemplateType::CodeLlamaSuper),
255            "belle-llama-2-chat" => Ok(PromptTemplateType::HumanAssistant),
256            "human-assistant" => Ok(PromptTemplateType::HumanAssistant),
257            "vicuna-1.0-chat" => Ok(PromptTemplateType::VicunaChat),
258            "vicuna-1.1-chat" => Ok(PromptTemplateType::Vicuna11Chat),
259            "vicuna-llava" => Ok(PromptTemplateType::VicunaLlava),
260            "chatml" => Ok(PromptTemplateType::ChatML),
261            "chatml-tool" => Ok(PromptTemplateType::ChatMLTool),
262            "chatml-think" => Ok(PromptTemplateType::ChatMLThink),
263            "internlm-2-tool" => Ok(PromptTemplateType::InternLM2Tool),
264            "openchat" => Ok(PromptTemplateType::OpenChat),
265            "baichuan-2" => Ok(PromptTemplateType::Baichuan2),
266            "wizard-coder" => Ok(PromptTemplateType::WizardCoder),
267            "zephyr" => Ok(PromptTemplateType::Zephyr),
268            "stablelm-zephyr" => Ok(PromptTemplateType::StableLMZephyr),
269            "intel-neural" => Ok(PromptTemplateType::IntelNeural),
270            "deepseek-chat" => Ok(PromptTemplateType::DeepseekChat),
271            "deepseek-coder" => Ok(PromptTemplateType::DeepseekCoder),
272            "deepseek-chat-2" => Ok(PromptTemplateType::DeepseekChat2),
273            "deepseek-chat-25" => Ok(PromptTemplateType::DeepseekChat25),
274            "deepseek-chat-3" => Ok(PromptTemplateType::DeepseekChat3),
275            "solar-instruct" => Ok(PromptTemplateType::SolarInstruct),
276            "phi-2-chat" => Ok(PromptTemplateType::Phi2Chat),
277            "phi-2-instruct" => Ok(PromptTemplateType::Phi2Instruct),
278            "phi-3-chat" => Ok(PromptTemplateType::Phi3Chat),
279            "phi-3-instruct" => Ok(PromptTemplateType::Phi3Instruct),
280            "phi-4-chat" => Ok(PromptTemplateType::Phi4Chat),
281            "gemma-instruct" => Ok(PromptTemplateType::GemmaInstruct),
282            "gemma-3" => Ok(PromptTemplateType::Gemma3),
283            "octopus" => Ok(PromptTemplateType::Octopus),
284            "glm-4-chat" => Ok(PromptTemplateType::Glm4Chat),
285            "groq-llama3-tool" => Ok(PromptTemplateType::GroqLlama3Tool),
286            "mediatek-breeze" => Ok(PromptTemplateType::BreezeInstruct),
287            "nemotron-chat" => Ok(PromptTemplateType::NemotronChat),
288            "nemotron-tool" => Ok(PromptTemplateType::NemotronTool),
289            "functionary-32" => Ok(PromptTemplateType::FunctionaryV32),
290            "functionary-31" => Ok(PromptTemplateType::FunctionaryV31),
291            "minicpmv" => Ok(PromptTemplateType::MiniCPMV),
292            "moxin-chat" => Ok(PromptTemplateType::MoxinChat),
293            "moxin-instruct" => Ok(PromptTemplateType::MoxinInstruct),
294            "falcon3" => Ok(PromptTemplateType::Falcon3),
295            "megrez" => Ok(PromptTemplateType::Megrez),
296            "qwen2-vision" => Ok(PromptTemplateType::Qwen2vl),
297            "qwen3-no-think" => Ok(PromptTemplateType::Qwen3NoThink),
298            "qwen3-agent" => Ok(PromptTemplateType::Qwen3Agent),
299            "exaone-deep-chat" => Ok(PromptTemplateType::ExaoneDeepChat),
300            "exaone-chat" => Ok(PromptTemplateType::ExaoneChat),
301            "seed-instruct" => Ok(PromptTemplateType::SeedInstruct),
302            "seed-reasoning" => Ok(PromptTemplateType::SeedReasoning),
303            "seed-oss-think" => Ok(PromptTemplateType::SeedOssThink),
304            "seed-oss-no-think" => Ok(PromptTemplateType::SeedOssNoThink),
305            "smol-vision" => Ok(PromptTemplateType::Smolvl),
306            "smol3-no-think" => Ok(PromptTemplateType::Smol3NoThink),
307            "gpt-oss" => Ok(PromptTemplateType::GptOss),
308            "embedding" => Ok(PromptTemplateType::Embedding),
309            "tts" => Ok(PromptTemplateType::Tts),
310            "none" => Ok(PromptTemplateType::Null),
311            _ => Err(error::PromptError::UnknownPromptTemplateType(
312                template.to_string(),
313            )),
314        }
315    }
316}
317impl std::fmt::Display for PromptTemplateType {
318    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
319        match self {
320            PromptTemplateType::Llama2Chat => write!(f, "llama-2-chat"),
321            PromptTemplateType::Llama3Chat => write!(f, "llama-3-chat"),
322            PromptTemplateType::Llama3Tool => write!(f, "llama-3-tool"),
323            PromptTemplateType::Llama4Chat => write!(f, "llama-4-chat"),
324            PromptTemplateType::MistralInstruct => write!(f, "mistral-instruct"),
325            PromptTemplateType::MistralTool => write!(f, "mistral-tool"),
326            PromptTemplateType::MistralLite => write!(f, "mistrallite"),
327            PromptTemplateType::MistralSmallChat => write!(f, "mistral-small-chat"),
328            PromptTemplateType::MistralSmallTool => write!(f, "mistral-small-tool"),
329            PromptTemplateType::OpenChat => write!(f, "openchat"),
330            PromptTemplateType::CodeLlama => write!(f, "codellama-instruct"),
331            PromptTemplateType::HumanAssistant => write!(f, "human-asistant"),
332            PromptTemplateType::VicunaChat => write!(f, "vicuna-1.0-chat"),
333            PromptTemplateType::Vicuna11Chat => write!(f, "vicuna-1.1-chat"),
334            PromptTemplateType::VicunaLlava => write!(f, "vicuna-llava"),
335            PromptTemplateType::ChatML => write!(f, "chatml"),
336            PromptTemplateType::ChatMLTool => write!(f, "chatml-tool"),
337            PromptTemplateType::ChatMLThink => write!(f, "chatml-think"),
338            PromptTemplateType::InternLM2Tool => write!(f, "internlm-2-tool"),
339            PromptTemplateType::Baichuan2 => write!(f, "baichuan-2"),
340            PromptTemplateType::WizardCoder => write!(f, "wizard-coder"),
341            PromptTemplateType::Zephyr => write!(f, "zephyr"),
342            PromptTemplateType::StableLMZephyr => write!(f, "stablelm-zephyr"),
343            PromptTemplateType::IntelNeural => write!(f, "intel-neural"),
344            PromptTemplateType::DeepseekChat => write!(f, "deepseek-chat"),
345            PromptTemplateType::DeepseekCoder => write!(f, "deepseek-coder"),
346            PromptTemplateType::DeepseekChat2 => write!(f, "deepseek-chat-2"),
347            PromptTemplateType::DeepseekChat25 => write!(f, "deepseek-chat-25"),
348            PromptTemplateType::DeepseekChat3 => write!(f, "deepseek-chat-3"),
349            PromptTemplateType::SolarInstruct => write!(f, "solar-instruct"),
350            PromptTemplateType::Phi2Chat => write!(f, "phi-2-chat"),
351            PromptTemplateType::Phi2Instruct => write!(f, "phi-2-instruct"),
352            PromptTemplateType::Phi3Chat => write!(f, "phi-3-chat"),
353            PromptTemplateType::Phi3Instruct => write!(f, "phi-3-instruct"),
354            PromptTemplateType::Phi4Chat => write!(f, "phi-4-chat"),
355            PromptTemplateType::CodeLlamaSuper => write!(f, "codellama-super-instruct"),
356            PromptTemplateType::GemmaInstruct => write!(f, "gemma-instruct"),
357            PromptTemplateType::Gemma3 => write!(f, "gemma-3"),
358            PromptTemplateType::Octopus => write!(f, "octopus"),
359            PromptTemplateType::Glm4Chat => write!(f, "glm-4-chat"),
360            PromptTemplateType::GroqLlama3Tool => write!(f, "groq-llama3-tool"),
361            PromptTemplateType::BreezeInstruct => write!(f, "mediatek-breeze"),
362            PromptTemplateType::NemotronChat => write!(f, "nemotron-chat"),
363            PromptTemplateType::NemotronTool => write!(f, "nemotron-tool"),
364            PromptTemplateType::FunctionaryV32 => write!(f, "functionary-32"),
365            PromptTemplateType::FunctionaryV31 => write!(f, "functionary-31"),
366            PromptTemplateType::MiniCPMV => write!(f, "minicpmv"),
367            PromptTemplateType::MoxinChat => write!(f, "moxin-chat"),
368            PromptTemplateType::MoxinInstruct => write!(f, "moxin-instruct"),
369            PromptTemplateType::Falcon3 => write!(f, "falcon3"),
370            PromptTemplateType::Megrez => write!(f, "megrez"),
371            PromptTemplateType::Qwen2vl => write!(f, "qwen2-vision"),
372            PromptTemplateType::Qwen3NoThink => write!(f, "qwen3-no-think"),
373            PromptTemplateType::Qwen3Agent => write!(f, "qwen3-agent"),
374            PromptTemplateType::ExaoneDeepChat => write!(f, "exaone-deep-chat"),
375            PromptTemplateType::ExaoneChat => write!(f, "exaone-chat"),
376            PromptTemplateType::SeedInstruct => write!(f, "seed-instruct"),
377            PromptTemplateType::SeedReasoning => write!(f, "seed-reasoning"),
378            PromptTemplateType::SeedOssThink => write!(f, "seed-oss-think"),
379            PromptTemplateType::SeedOssNoThink => write!(f, "seed-oss-no-think"),
380            PromptTemplateType::Smolvl => write!(f, "smol-vision"),
381            PromptTemplateType::Smol3NoThink => write!(f, "smol3-no-think"),
382            PromptTemplateType::GptOss => write!(f, "gpt-oss"),
383            PromptTemplateType::Embedding => write!(f, "embedding"),
384            PromptTemplateType::Tts => write!(f, "tts"),
385            PromptTemplateType::Null => write!(f, "none"),
386        }
387    }
388}
389
390#[enum_dispatch::enum_dispatch(BuildChatPrompt)]
391pub enum ChatPrompt {
392    Llama2ChatPrompt,
393    Llama3ChatPrompt,
394    Llama3ToolPrompt,
395    Llama4ChatPrompt,
396    MistralInstructPrompt,
397    MistralToolPrompt,
398    MistralLitePrompt,
399    MistralSmallChatPrompt,
400    MistralSmallToolPrompt,
401    OpenChatPrompt,
402    CodeLlamaInstructPrompt,
403    CodeLlamaSuperInstructPrompt,
404    HumanAssistantChatPrompt,
405    /// Vicuna 1.0
406    VicunaChatPrompt,
407    /// Vicuna 1.1
408    Vicuna11ChatPrompt,
409    VicunaLlavaPrompt,
410    ChatMLPrompt,
411    ChatMLToolPrompt,
412    ChatMLThinkPrompt,
413    InternLM2ToolPrompt,
414    Baichuan2ChatPrompt,
415    WizardCoderPrompt,
416    ZephyrChatPrompt,
417    StableLMZephyrChatPrompt,
418    NeuralChatPrompt,
419    DeepseekChatPrompt,
420    DeepseekCoderPrompt,
421    DeepseekChat2Prompt,
422    DeepseekChat25Prompt,
423    SolarInstructPrompt,
424    Phi2ChatPrompt,
425    Phi2InstructPrompt,
426    Phi3ChatPrompt,
427    Phi3InstructPrompt,
428    Phi4ChatPrompt,
429    GemmaInstructPrompt,
430    Gemma3Prompt,
431    OctopusPrompt,
432    Glm4ChatPrompt,
433    GroqLlama3ToolPrompt,
434    BreezeInstructPrompt,
435    NemotronChatPrompt,
436    NemotronToolPrompt,
437    FunctionaryV32ToolPrompt,
438    FunctionaryV31ToolPrompt,
439    MiniCPMVPrompt,
440    MoxinChatPrompt,
441    MoxinInstructPrompt,
442    FalconChatPrompt,
443    MegrezPrompt,
444    Qwen2vlPrompt,
445    Qwen3NoThinkPrompt,
446    Qwen3AgentPrompt,
447    ExaoneDeepChatPrompt,
448    ExaoneChatPrompt,
449    SeedInstructPrompt,
450    SeedReasoningPrompt,
451    SeedOssThinkPrompt,
452    SeedOssNoThinkPrompt,
453    SmolvlPrompt,
454    Smol3NoThinkPrompt,
455    GptOssPrompt,
456}
457impl From<PromptTemplateType> for ChatPrompt {
458    fn from(ty: PromptTemplateType) -> Self {
459        match ty {
460            PromptTemplateType::Llama2Chat => ChatPrompt::Llama2ChatPrompt(Llama2ChatPrompt),
461            PromptTemplateType::Llama3Chat => ChatPrompt::Llama3ChatPrompt(Llama3ChatPrompt),
462            PromptTemplateType::Llama3Tool => ChatPrompt::Llama3ToolPrompt(Llama3ToolPrompt),
463            PromptTemplateType::Llama4Chat => ChatPrompt::Llama4ChatPrompt(Llama4ChatPrompt),
464            PromptTemplateType::MistralInstruct => {
465                ChatPrompt::MistralInstructPrompt(MistralInstructPrompt)
466            }
467            PromptTemplateType::MistralTool => ChatPrompt::MistralToolPrompt(MistralToolPrompt),
468            PromptTemplateType::MistralLite => ChatPrompt::MistralLitePrompt(MistralLitePrompt),
469            PromptTemplateType::MistralSmallChat => {
470                ChatPrompt::MistralSmallChatPrompt(MistralSmallChatPrompt)
471            }
472            PromptTemplateType::MistralSmallTool => {
473                ChatPrompt::MistralSmallToolPrompt(MistralSmallToolPrompt)
474            }
475            PromptTemplateType::OpenChat => ChatPrompt::OpenChatPrompt(OpenChatPrompt),
476            PromptTemplateType::CodeLlama => {
477                ChatPrompt::CodeLlamaInstructPrompt(CodeLlamaInstructPrompt)
478            }
479            PromptTemplateType::CodeLlamaSuper => {
480                ChatPrompt::CodeLlamaSuperInstructPrompt(CodeLlamaSuperInstructPrompt)
481            }
482            PromptTemplateType::HumanAssistant => {
483                ChatPrompt::HumanAssistantChatPrompt(HumanAssistantChatPrompt)
484            }
485            PromptTemplateType::VicunaChat => ChatPrompt::VicunaChatPrompt(VicunaChatPrompt),
486            PromptTemplateType::Vicuna11Chat => ChatPrompt::Vicuna11ChatPrompt(Vicuna11ChatPrompt),
487            PromptTemplateType::VicunaLlava => ChatPrompt::VicunaLlavaPrompt(VicunaLlavaPrompt),
488            PromptTemplateType::ChatML => ChatPrompt::ChatMLPrompt(ChatMLPrompt),
489            PromptTemplateType::ChatMLTool => ChatPrompt::ChatMLToolPrompt(ChatMLToolPrompt),
490            PromptTemplateType::ChatMLThink => ChatPrompt::ChatMLThinkPrompt(ChatMLThinkPrompt),
491            PromptTemplateType::InternLM2Tool => {
492                ChatPrompt::InternLM2ToolPrompt(InternLM2ToolPrompt)
493            }
494            PromptTemplateType::Baichuan2 => ChatPrompt::Baichuan2ChatPrompt(Baichuan2ChatPrompt),
495            PromptTemplateType::WizardCoder => ChatPrompt::WizardCoderPrompt(WizardCoderPrompt),
496            PromptTemplateType::Zephyr => ChatPrompt::ZephyrChatPrompt(ZephyrChatPrompt),
497            PromptTemplateType::StableLMZephyr => {
498                ChatPrompt::StableLMZephyrChatPrompt(StableLMZephyrChatPrompt)
499            }
500            PromptTemplateType::IntelNeural => ChatPrompt::NeuralChatPrompt(NeuralChatPrompt),
501            PromptTemplateType::DeepseekChat => ChatPrompt::DeepseekChatPrompt(DeepseekChatPrompt),
502            PromptTemplateType::DeepseekCoder => {
503                ChatPrompt::DeepseekCoderPrompt(DeepseekCoderPrompt)
504            }
505            PromptTemplateType::DeepseekChat2 => {
506                ChatPrompt::DeepseekChat2Prompt(DeepseekChat2Prompt)
507            }
508            PromptTemplateType::DeepseekChat25 => {
509                ChatPrompt::DeepseekChat25Prompt(DeepseekChat25Prompt)
510            }
511            PromptTemplateType::DeepseekChat3 => {
512                ChatPrompt::DeepseekChat25Prompt(DeepseekChat25Prompt)
513            }
514            PromptTemplateType::SolarInstruct => {
515                ChatPrompt::SolarInstructPrompt(SolarInstructPrompt)
516            }
517            PromptTemplateType::Phi2Chat => ChatPrompt::Phi2ChatPrompt(Phi2ChatPrompt),
518            PromptTemplateType::Phi2Instruct => ChatPrompt::Phi2InstructPrompt(Phi2InstructPrompt),
519            PromptTemplateType::Phi3Chat => ChatPrompt::Phi3ChatPrompt(Phi3ChatPrompt),
520            PromptTemplateType::Phi3Instruct => ChatPrompt::Phi3InstructPrompt(Phi3InstructPrompt),
521            PromptTemplateType::Phi4Chat => ChatPrompt::Phi4ChatPrompt(Phi4ChatPrompt),
522            PromptTemplateType::GemmaInstruct => {
523                ChatPrompt::GemmaInstructPrompt(GemmaInstructPrompt)
524            }
525            PromptTemplateType::Gemma3 => ChatPrompt::Gemma3Prompt(Gemma3Prompt),
526            PromptTemplateType::Octopus => ChatPrompt::OctopusPrompt(OctopusPrompt),
527            PromptTemplateType::Glm4Chat => ChatPrompt::Glm4ChatPrompt(Glm4ChatPrompt),
528            PromptTemplateType::GroqLlama3Tool => {
529                ChatPrompt::GroqLlama3ToolPrompt(GroqLlama3ToolPrompt)
530            }
531            PromptTemplateType::BreezeInstruct => {
532                ChatPrompt::BreezeInstructPrompt(BreezeInstructPrompt)
533            }
534            PromptTemplateType::NemotronChat => ChatPrompt::NemotronChatPrompt(NemotronChatPrompt),
535            PromptTemplateType::NemotronTool => ChatPrompt::NemotronToolPrompt(NemotronToolPrompt),
536            PromptTemplateType::FunctionaryV32 => {
537                ChatPrompt::FunctionaryV32ToolPrompt(FunctionaryV32ToolPrompt)
538            }
539            PromptTemplateType::FunctionaryV31 => {
540                ChatPrompt::FunctionaryV31ToolPrompt(FunctionaryV31ToolPrompt)
541            }
542            PromptTemplateType::MiniCPMV => ChatPrompt::MiniCPMVPrompt(MiniCPMVPrompt),
543            PromptTemplateType::MoxinChat => ChatPrompt::MoxinChatPrompt(MoxinChatPrompt),
544            PromptTemplateType::MoxinInstruct => {
545                ChatPrompt::MoxinInstructPrompt(MoxinInstructPrompt)
546            }
547            PromptTemplateType::Falcon3 => ChatPrompt::FalconChatPrompt(FalconChatPrompt),
548            PromptTemplateType::Megrez => ChatPrompt::MegrezPrompt(MegrezPrompt),
549            PromptTemplateType::Qwen2vl => ChatPrompt::Qwen2vlPrompt(Qwen2vlPrompt),
550            PromptTemplateType::Qwen3NoThink => ChatPrompt::Qwen3NoThinkPrompt(Qwen3NoThinkPrompt),
551            PromptTemplateType::Qwen3Agent => ChatPrompt::Qwen3AgentPrompt(Qwen3AgentPrompt),
552            PromptTemplateType::ExaoneDeepChat => {
553                ChatPrompt::ExaoneDeepChatPrompt(ExaoneDeepChatPrompt)
554            }
555            PromptTemplateType::ExaoneChat => ChatPrompt::ExaoneChatPrompt(ExaoneChatPrompt),
556            PromptTemplateType::SeedInstruct => ChatPrompt::SeedInstructPrompt(SeedInstructPrompt),
557            PromptTemplateType::SeedReasoning => {
558                ChatPrompt::SeedReasoningPrompt(SeedReasoningPrompt)
559            }
560            PromptTemplateType::Smolvl => ChatPrompt::SmolvlPrompt(SmolvlPrompt),
561            PromptTemplateType::Smol3NoThink => ChatPrompt::Smol3NoThinkPrompt(Smol3NoThinkPrompt),
562            PromptTemplateType::SeedOssThink => ChatPrompt::SeedOssThinkPrompt(SeedOssThinkPrompt),
563            PromptTemplateType::SeedOssNoThink => {
564                ChatPrompt::SeedOssNoThinkPrompt(SeedOssNoThinkPrompt)
565            }
566            PromptTemplateType::GptOss => ChatPrompt::GptOssPrompt(GptOssPrompt),
567            PromptTemplateType::Embedding => {
568                panic!("Embedding prompt template is not used for building chat prompts")
569            }
570            PromptTemplateType::Tts => {
571                panic!("Tts prompt template is not used for building chat prompts")
572            }
573            PromptTemplateType::Null => {
574                panic!("Null prompt template is not used for building chat prompts")
575            }
576        }
577    }
578}
579
580/// Trait for merging RAG context into chat messages
581pub trait MergeRagContext: Send {
582    /// Merge RAG context into chat messages.
583    ///
584    /// Note that the default implementation simply merges the RAG context into the system message. That is, to use the default implementation, `has_system_prompt` should be set to `true` and `policy` set to `MergeRagContextPolicy::SystemMessage`.
585    ///
586    /// # Arguments
587    ///
588    /// * `messages` - The chat messages to merge the context into.
589    ///
590    /// * `context` - The RAG context to merge into the chat messages.
591    ///
592    /// * `has_system_prompt` - Whether the chat template has a system prompt.
593    ///
594    /// * `policy` - The policy for merging RAG context into chat messages.
595    fn build(
596        messages: &mut Vec<endpoints::chat::ChatCompletionRequestMessage>,
597        context: &[String],
598        has_system_prompt: bool,
599        policy: MergeRagContextPolicy,
600        rag_prompt: Option<String>,
601    ) -> error::Result<()> {
602        if (policy == MergeRagContextPolicy::SystemMessage) && has_system_prompt {
603            if messages.is_empty() {
604                return Err(error::PromptError::NoMessages);
605            }
606
607            if context.is_empty() {
608                return Err(error::PromptError::Operation(
609                    "No context provided.".to_string(),
610                ));
611            }
612
613            let context = context[0].trim_end();
614
615            // update or insert system message
616            match messages[0] {
617                ChatCompletionRequestMessage::System(ref message) => {
618                    // compose new system message content
619                    let content = match rag_prompt {
620                        Some(rag_prompt) if !rag_prompt.is_empty() => {
621                            format!(
622                                "{original_system_message}\n{rag_prompt}\n{context}",
623                                original_system_message = message.content().trim(),
624                                rag_prompt = rag_prompt.trim(),
625                                context = context.trim_end()
626                            )
627                        }
628                        _ => {
629                            format!("{original_system_message}\nUse the following pieces of context to answer the user's question.\nIf you don't know the answer, just say that you don't know, don't try to make up an answer.\n----------------\n{context}", original_system_message=message.content().trim(), context=context.trim_end())
630                        }
631                    };
632
633                    // create system message
634                    let system_message = ChatCompletionRequestMessage::new_system_message(
635                        content,
636                        messages[0].name().cloned(),
637                    );
638
639                    // replace the original system message
640                    messages[0] = system_message;
641                }
642                _ => {
643                    // compose new system message content
644                    let content = match rag_prompt {
645                        Some(rag_prompt) if !rag_prompt.is_empty() => {
646                            format!(
647                                "{rag_prompt}\n{context}",
648                                rag_prompt = rag_prompt.trim(),
649                                context = context.trim_end()
650                            )
651                        }
652                        _ => {
653                            format!("Use the following pieces of context to answer the user's question.\nIf you don't know the answer, just say that you don't know, don't try to make up an answer.\n----------------\n{}", context.trim_end())
654                        }
655                    };
656
657                    // create system message
658                    let system_message = ChatCompletionRequestMessage::new_system_message(
659                        content,
660                        messages[0].name().cloned(),
661                    );
662                    // insert system message
663                    messages.insert(0, system_message);
664                }
665            };
666        }
667
668        Ok(())
669    }
670}
671
672/// Define the strategy for merging RAG context into chat messages.
673#[derive(Clone, Debug, Copy, Default, PartialEq, Eq, Serialize, Deserialize, ValueEnum)]
674pub enum MergeRagContextPolicy {
675    /// Merge RAG context into the system message.
676    ///
677    /// Note that this policy is only applicable when the chat template has a system message.
678    #[default]
679    #[serde(rename = "system-message")]
680    SystemMessage,
681    /// Merge RAG context into the last user message.
682    #[serde(rename = "last-user-message")]
683    LastUserMessage,
684}
685impl std::fmt::Display for MergeRagContextPolicy {
686    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
687        match self {
688            MergeRagContextPolicy::SystemMessage => write!(f, "system-message"),
689            MergeRagContextPolicy::LastUserMessage => write!(f, "last-user-message"),
690        }
691    }
692}
693impl FromStr for MergeRagContextPolicy {
694    type Err = error::PromptError;
695
696    fn from_str(policy: &str) -> std::result::Result<Self, Self::Err> {
697        Ok(match policy {
698            "system-message" => MergeRagContextPolicy::SystemMessage,
699            "last-user-message" => MergeRagContextPolicy::LastUserMessage,
700            _ => {
701                return Err(error::PromptError::UnknownMergeRagContextPolicy(
702                    policy.to_string(),
703                ))
704            }
705        })
706    }
707}
708
709/// Trait for building prompts for chat completions.
710#[enum_dispatch::enum_dispatch]
711pub trait BuildChatPrompt: Send {
712    fn build(&self, messages: &mut Vec<ChatCompletionRequestMessage>) -> Result<String>;
713
714    fn build_with_tools(
715        &self,
716        messages: &mut Vec<ChatCompletionRequestMessage>,
717        _tools: Option<&[Tool]>,
718    ) -> Result<String> {
719        self.build(messages)
720    }
721}
chat_prompts/lib.rs

chat_prompts/
lib.rs
