dynamo_llm/preprocessor/
prompt.rs

1// SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3//
4// Licensed under the Apache License, Version 2.0 (the "License");
5// you may not use this file except in compliance with the License.
6// You may obtain a copy of the License at
7//
8// http://www.apache.org/licenses/LICENSE-2.0
9//
10// Unless required by applicable law or agreed to in writing, software
11// distributed under the License is distributed on an "AS IS" BASIS,
12// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13// See the License for the specific language governing permissions and
14// limitations under the License.
15
16//! Prompt Formatting Module
17//!
18//! Handles formatting of LLM request prompts, including:
19//! - Chat template rendering
20//! - Tool usage formatting
21//! - Generation prompt handling
22//!
23//! The module supports different prompt formatting strategies through the
24//! PromptFormatter
25
26// TODO:
27// 1. Query if `add_generation_prompt` is present in the prompt template
28// 2. Support for models with add_generation_prompt:
29//    - PALS (Prefix-Assisted Language Sampling)
30//    - Continuation - Detected on user turns, where we can return
31//      partial assistant responses without add_generation_prompt
32
33use anyhow::Result;
34use minijinja::value::Value;
35use std::sync::Arc;
36
37mod template;
38
39pub use template::ContextMixins;
40
41/// Trait that defines a request that can map to an OpenAI-like request.
42pub trait OAIChatLikeRequest {
43    fn messages(&self) -> Value;
44    fn tools(&self) -> Option<Value> {
45        None
46    }
47    fn tool_choice(&self) -> Option<Value> {
48        None
49    }
50
51    fn should_add_generation_prompt(&self) -> bool;
52}
53
54pub trait OAIPromptFormatter: Send + Sync + 'static {
55    fn supports_add_generation_prompt(&self) -> bool;
56    fn render(&self, req: &dyn OAIChatLikeRequest) -> Result<String>;
57}
58
59pub enum PromptFormatter {
60    OAI(Arc<dyn OAIPromptFormatter>),
61}
dynamo_llm/preprocessor/prompt.rs

dynamo_llm/preprocessor/
prompt.rs
