1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
use derive_builder::Builder;
use serde::{Deserialize, Serialize};
use utoipa::ToSchema;
use validator::Validate;
/// Common extensions for OpenAI API requests that are not part of the standard OpenAI spec
/// but are commonly needed across different request types.
#[derive(ToSchema, Serialize, Deserialize, Builder, Validate, Debug, Clone, Default)]
pub struct CommonExt {
/// If true, the model will ignore the end of string token and generate to max_tokens.
/// This field can also be specified in nvext, but the root-level value takes precedence.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[builder(default, setter(strip_option))]
pub ignore_eos: Option<bool>,
/// The minimum number of tokens to generate.
/// This is a common parameter needed across different request types.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[builder(default, setter(strip_option))]
pub min_tokens: Option<u32>,
/// Integer that controls the number of top tokens to consider. Set to -1 or 0 to consider all
/// tokens.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[builder(default, setter(strip_option))]
pub top_k: Option<i32>,
/// Relative probability floor
#[serde(default, skip_serializing_if = "Option::is_none")]
#[builder(default, setter(strip_option))]
pub min_p: Option<f32>,
/// How much to penalize tokens based on how frequently they occur in the text.
/// A value of 1 means no penalty, while values larger than 1 discourage and values smaller encourage.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[builder(default, setter(strip_option))]
pub repetition_penalty: Option<f32>,
/// include_stop_str_in_output
#[serde(default, skip_serializing_if = "Option::is_none")]
#[builder(default, setter(strip_option))]
pub include_stop_str_in_output: Option<bool>,
/// Guided Decoding Options
/// If specified, the output will be a JSON object. Can be a string, an object, or null.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[builder(default, setter(strip_option))]
pub guided_json: Option<serde_json::Value>,
/// If specified, the output will follow the regex pattern. Can be a string or null.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[builder(default, setter(strip_option))]
pub guided_regex: Option<String>,
/// If specified, the output will follow the context-free grammar. Can be a string or null.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[builder(default, setter(strip_option))]
pub guided_grammar: Option<String>,
/// If specified, the output will be exactly one of the choices.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[builder(default, setter(strip_option))]
pub guided_choice: Option<Vec<String>>,
/// If specified, the backend to use for guided decoding, can be backends like xgrammar or custom guided decoding backend
#[serde(default, skip_serializing_if = "Option::is_none")]
#[builder(default, setter(strip_option))]
pub guided_decoding_backend: Option<String>,
/// If specified, the output will follow the whitespace pattern. Can be a string or null.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[builder(default, setter(strip_option))]
#[allow(unused)] // Not used
pub guided_whitespace_pattern: Option<String>,
/// Whether to skip special tokens in the decoded output.
/// When true, special tokens (like EOS, BOS, PAD) are removed from the output text.
/// When false, special tokens are included in the output text.
/// Defaults to false if not specified.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[builder(default, setter(strip_option))]
pub skip_special_tokens: Option<bool>,
/// Number of log probabilities to return per prompt token.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[builder(default, setter(strip_option))]
pub prompt_logprobs: Option<u32>,
/// If true, append the assistant generation prompt after the last message.
/// Defaults to true when omitted, matching vLLM 0.27.1 and the Python
/// frontend (HuggingFace Transformers defaults this flag to false).
/// Incompatible with `continue_final_message`. Chat-only: runtime-rejected
/// on `/v1/completions`. Hidden from the shared OpenAPI schema so
/// completions does not advertise these fields.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schema(ignore)]
#[builder(default, setter(strip_option))]
pub add_generation_prompt: Option<bool>,
/// If true, leave the last message open so the model continues that turn
/// instead of starting a new one. Any final message role can be continued.
/// Incompatible with omitted or `true` `add_generation_prompt` (vLLM 0.27.1
/// finalizes the omitted field to true). Chat-only: runtime-rejected on
/// `/v1/completions`. Hidden from the shared OpenAPI schema so completions
/// does not advertise these fields.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schema(ignore)]
#[builder(default, setter(strip_option))]
pub continue_final_message: Option<bool>,
}
impl CommonExt {
pub fn builder() -> CommonExtBuilder {
CommonExtBuilder::default()
}
}
/// Trait for types that provide CommonExt fields
pub trait CommonExtProvider {
/// Get a reference to the CommonExt struct if available
fn common_ext(&self) -> Option<&CommonExt>;
/// Guided Decoding Options
fn get_guided_json(&self) -> Option<serde_json::Value>;
fn get_guided_regex(&self) -> Option<String>;
fn get_guided_grammar(&self) -> Option<String>;
fn get_guided_choice(&self) -> Option<Vec<String>>;
fn get_guided_decoding_backend(&self) -> Option<String>;
#[allow(unused)] // Not used
fn get_guided_whitespace_pattern(&self) -> Option<String>;
/// Other sampling Options
fn get_top_k(&self) -> Option<i32>;
fn get_min_p(&self) -> Option<f32>;
fn get_repetition_penalty(&self) -> Option<f32>;
fn get_include_stop_str_in_output(&self) -> Option<bool>;
/// Output Options
fn get_skip_special_tokens(&self) -> Option<bool>;
/// Number of prompt logprobs to request from the engine.
fn get_prompt_logprobs_count(&self) -> Option<u32> {
None
}
/// Whether to continue the last message instead of starting a new turn.
fn get_continue_final_message(&self) -> Option<bool> {
self.common_ext()
.and_then(|common| common.continue_final_message)
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json;
#[test]
fn test_validation_min_tokens() {
// Test that min_tokens with 0 is valid
let common_ext = CommonExt {
ignore_eos: None,
min_tokens: Some(0), // Should be valid (min = 0)
top_k: None,
min_p: None,
repetition_penalty: None,
include_stop_str_in_output: None,
guided_json: None,
guided_regex: None,
guided_grammar: None,
guided_choice: None,
guided_decoding_backend: None,
guided_whitespace_pattern: None,
skip_special_tokens: None,
prompt_logprobs: None,
add_generation_prompt: None,
continue_final_message: None,
};
assert!(common_ext.validate().is_ok());
}
#[test]
fn test_skip_special_tokens_serialization() {
// Test that skip_special_tokens can be serialized and deserialized
let common_ext = CommonExt::builder()
.skip_special_tokens(true)
.build()
.unwrap();
let json = serde_json::to_string(&common_ext).unwrap();
let deserialized: CommonExt = serde_json::from_str(&json).unwrap();
assert_eq!(deserialized.skip_special_tokens, Some(true));
// Test with false value
let common_ext = CommonExt::builder()
.skip_special_tokens(false)
.build()
.unwrap();
let json = serde_json::to_string(&common_ext).unwrap();
let deserialized: CommonExt = serde_json::from_str(&json).unwrap();
assert_eq!(deserialized.skip_special_tokens, Some(false));
// Test that None is not serialized (skip_serializing_if = "Option::is_none")
let common_ext = CommonExt::builder().build().unwrap();
let json = serde_json::to_string(&common_ext).unwrap();
assert!(!json.contains("skip_special_tokens"));
}
}