openai_protocol/
completion.rs

1use std::collections::HashMap;
2
3use serde::{Deserialize, Serialize};
4use serde_json::{Map, Value};
5use validator::Validate;
6
7use super::{
8    common::*,
9    sampling_params::{validate_top_k_value, validate_top_p_value},
10};
11use crate::validated::Normalizable;
12
13// ============================================================================
14// Completions API (v1/completions) - DEPRECATED but still supported
15// ============================================================================
16
17#[serde_with::skip_serializing_none]
18#[derive(Debug, Clone, Deserialize, Serialize, Validate, schemars::JsonSchema)]
19#[validate(schema(function = "validate_completion_cross_parameters"))]
20pub struct CompletionRequest {
21    /// ID of the model to use (required for OpenAI, optional for some implementations, such as SGLang)
22    pub model: String,
23
24    /// The prompt(s) to generate completions for
25    #[validate(custom(function = "validate_completion_prompt"))]
26    pub prompt: StringOrArray,
27
28    /// Generates `best_of` completions server-side and returns the "best"
29    #[validate(range(min = 0, max = 20))]
30    pub best_of: Option<u32>,
31
32    /// Echo back the prompt in addition to the completion
33    #[serde(default)]
34    pub echo: bool,
35
36    /// Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far
37    #[validate(range(min = -2.0, max = 2.0))]
38    pub frequency_penalty: Option<f32>,
39
40    /// Modify the likelihood of specified tokens appearing in the completion
41    pub logit_bias: Option<HashMap<String, f32>>,
42
43    /// Include the log probabilities on the `logprobs` most likely tokens
44    #[validate(range(min = 0, max = 5))]
45    pub logprobs: Option<u32>,
46
47    /// The maximum number of tokens to generate
48    #[validate(range(min = 0))]
49    pub max_tokens: Option<u32>,
50
51    /// How many completions to generate for each prompt
52    #[validate(range(min = 1, max = 128))]
53    pub n: Option<u32>,
54
55    /// Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far
56    #[validate(range(min = -2.0, max = 2.0))]
57    pub presence_penalty: Option<f32>,
58
59    /// If specified, our system will make a best effort to sample deterministically
60    pub seed: Option<i64>,
61
62    /// Up to 4 sequences where the API will stop generating further tokens
63    #[validate(custom(function = "validate_stop"))]
64    pub stop: Option<StringOrArray>,
65
66    /// Whether to stream back partial progress
67    #[serde(default, deserialize_with = "deserialize_null_as_false")]
68    pub stream: bool,
69
70    /// Options for streaming response
71    pub stream_options: Option<StreamOptions>,
72
73    /// The suffix that comes after a completion of inserted text
74    pub suffix: Option<String>,
75
76    /// What sampling temperature to use, between 0 and 2
77    #[validate(range(min = 0.0, max = 2.0))]
78    pub temperature: Option<f32>,
79
80    /// An alternative to sampling with temperature (nucleus sampling)
81    #[validate(custom(function = "validate_top_p_value"))]
82    pub top_p: Option<f32>,
83
84    /// A unique identifier representing your end-user
85    pub user: Option<String>,
86
87    // =============================================================================
88    // Engine-Specific Sampling Parameters
89    // =============================================================================
90    /// Top-k sampling parameter (-1 to disable)
91    #[validate(custom(function = "validate_top_k_value"))]
92    pub top_k: Option<i32>,
93
94    /// Min-p nucleus sampling parameter
95    #[validate(range(min = 0.0, max = 1.0))]
96    pub min_p: Option<f32>,
97
98    /// Minimum number of tokens to generate
99    #[validate(range(min = 0))]
100    pub min_tokens: Option<u32>,
101
102    /// Repetition penalty for reducing repetitive text
103    #[validate(range(min = 0.0, max = 2.0))]
104    pub repetition_penalty: Option<f32>,
105
106    /// Regex constraint for output generation
107    pub regex: Option<String>,
108
109    /// EBNF grammar constraint for structured output
110    pub ebnf: Option<String>,
111
112    /// JSON schema constraint for structured output
113    pub json_schema: Option<String>,
114
115    /// Specific token IDs to use as stop conditions
116    pub stop_token_ids: Option<Vec<u32>>,
117
118    /// Skip trimming stop tokens from output
119    #[serde(default, skip_serializing_if = "is_false")]
120    pub no_stop_trim: bool,
121
122    /// Ignore end-of-sequence tokens during generation
123    #[serde(default, skip_serializing_if = "is_false")]
124    pub ignore_eos: bool,
125
126    /// Skip special tokens during detokenization
127    #[serde(default = "default_true")]
128    pub skip_special_tokens: bool,
129
130    /// Path to LoRA adapter(s) for model customization
131    pub lora_path: Option<String>,
132
133    /// Session parameters for continual prompting
134    pub session_params: Option<HashMap<String, Value>>,
135
136    /// Return model hidden states
137    #[serde(default, skip_serializing_if = "is_false")]
138    pub return_hidden_states: bool,
139
140    /// Sampling seed for deterministic outputs
141    pub sampling_seed: Option<u64>,
142
143    /// Request ID forwarded to the backend for log correlation (SGLang extension)
144    pub rid: Option<String>,
145
146    /// Additional fields including bootstrap info for PD routing
147    #[serde(flatten)]
148    pub other: Map<String, Value>,
149}
150
151impl Normalizable for CompletionRequest {}
152
153fn validate_completion_prompt(prompt: &StringOrArray) -> Result<(), validator::ValidationError> {
154    match prompt {
155        StringOrArray::String(_) => {}
156        StringOrArray::Array(arr) => {
157            if arr.is_empty() {
158                let mut error = validator::ValidationError::new("prompt_empty");
159                error.message = Some("prompt array cannot be empty".into());
160                return Err(error);
161            }
162        }
163    }
164
165    Ok(())
166}
167
168fn validate_completion_cross_parameters(
169    req: &CompletionRequest,
170) -> Result<(), validator::ValidationError> {
171    if req.stream_options.is_some() && !req.stream {
172        let mut error = validator::ValidationError::new("stream_options_requires_stream");
173        error.message =
174            Some("The 'stream_options' parameter is only allowed when 'stream' is enabled".into());
175        return Err(error);
176    }
177
178    if let (Some(min), Some(max)) = (req.min_tokens, req.max_tokens) {
179        if min > max {
180            let mut error = validator::ValidationError::new("min_tokens_exceeds_max");
181            error.message = Some("min_tokens cannot exceed max_tokens".into());
182            return Err(error);
183        }
184    }
185
186    let constraint_count =
187        req.regex.is_some() as u8 + req.ebnf.is_some() as u8 + req.json_schema.is_some() as u8;
188    if constraint_count > 1 {
189        let mut error = validator::ValidationError::new("multiple_constraints");
190        error.message = Some(
191            "only one structured output constraint (regex, ebnf, or json_schema) can be active at a time"
192                .into(),
193        );
194        return Err(error);
195    }
196
197    if let (Some(best_of), Some(n)) = (req.best_of, req.n) {
198        if best_of <= n {
199            let mut error = validator::ValidationError::new("best_of_less_than_n");
200            error.message = Some("best_of must be greater than n".into());
201            return Err(error);
202        }
203    }
204
205    if req.stream && req.best_of.is_some() {
206        let mut error = validator::ValidationError::new("best_of_not_supported_with_stream");
207        error.message = Some("best_of is not supported when stream is enabled".into());
208        return Err(error);
209    }
210
211    Ok(())
212}
213
214impl GenerationRequest for CompletionRequest {
215    fn rid(&self) -> Option<&str> {
216        self.rid.as_deref()
217    }
218
219    fn is_stream(&self) -> bool {
220        self.stream
221    }
222
223    fn get_model(&self) -> Option<&str> {
224        Some(&self.model)
225    }
226
227    fn extract_text_for_routing(&self) -> String {
228        match &self.prompt {
229            StringOrArray::String(s) => s.clone(),
230            StringOrArray::Array(v) => v.join(" "),
231        }
232    }
233}
234
235// ============================================================================
236// Response Types
237// ============================================================================
238
239#[serde_with::skip_serializing_none]
240#[derive(Debug, Clone, Deserialize, Serialize, schemars::JsonSchema)]
241pub struct CompletionResponse {
242    pub id: String,
243    pub object: String, // "text_completion"
244    pub created: u64,
245    pub model: String,
246    pub choices: Vec<CompletionChoice>,
247    pub usage: Option<Usage>,
248    pub system_fingerprint: Option<String>,
249}
250
251#[derive(Debug, Clone, Deserialize, Serialize, schemars::JsonSchema)]
252pub struct CompletionChoice {
253    pub text: String,
254    pub index: u32,
255    #[serde(skip_serializing_if = "Option::is_none")]
256    pub logprobs: Option<LogProbs>,
257    pub finish_reason: Option<String>, // "stop", "length", "content_filter", etc.
258    /// Information about which stop condition was matched
259    #[serde(skip_serializing_if = "Option::is_none")]
260    pub matched_stop: Option<Value>, // Can be string or integer
261}
262
263#[derive(Debug, Clone, Deserialize, Serialize, schemars::JsonSchema)]
264pub struct CompletionStreamResponse {
265    pub id: String,
266    pub object: String, // "text_completion"
267    pub created: u64,
268    pub choices: Vec<CompletionStreamChoice>,
269    pub model: String,
270    #[serde(skip_serializing_if = "Option::is_none")]
271    pub system_fingerprint: Option<String>,
272    #[serde(skip_serializing_if = "Option::is_none")]
273    pub usage: Option<Usage>,
274}
275
276#[derive(Debug, Clone, Deserialize, Serialize, schemars::JsonSchema)]
277pub struct CompletionStreamChoice {
278    pub text: String,
279    pub index: u32,
280    #[serde(skip_serializing_if = "Option::is_none")]
281    pub logprobs: Option<LogProbs>,
282    pub finish_reason: Option<String>,
283}
284
285#[cfg(test)]
286mod tests {
287    use super::*;
288
289    fn req(mut extra: Value) -> CompletionRequest {
290        let mut value = serde_json::json!({"model": "m", "prompt": "hello"});
291        value
292            .as_object_mut()
293            .expect("object")
294            .append(extra.as_object_mut().expect("object"));
295        serde_json::from_value(value).expect("request must deserialize")
296    }
297
298    #[test]
299    fn default_sglang_flags_are_omitted_and_absent_reads_defaults() {
300        let value = serde_json::to_value(req(serde_json::json!({}))).expect("serialize");
301        for field in ["no_stop_trim", "ignore_eos", "return_hidden_states"] {
302            assert!(value.get(field).is_none(), "{field} serialized at default");
303        }
304
305        let back: CompletionRequest = serde_json::from_value(value).expect("roundtrip");
306        assert!(!back.no_stop_trim);
307        assert!(!back.ignore_eos);
308        assert!(!back.return_hidden_states);
309    }
310
311    #[test]
312    fn non_default_sglang_flags_round_trip() {
313        let value = serde_json::to_value(req(serde_json::json!({
314            "no_stop_trim": true,
315            "ignore_eos": true,
316            "return_hidden_states": true
317        })))
318        .expect("serialize");
319        assert_eq!(value["no_stop_trim"], true);
320        assert_eq!(value["ignore_eos"], true);
321        assert_eq!(value["return_hidden_states"], true);
322
323        let back: CompletionRequest = serde_json::from_value(value).expect("roundtrip");
324        assert!(back.no_stop_trim);
325        assert!(back.ignore_eos);
326        assert!(back.return_hidden_states);
327    }
328}
openai_protocol/completion.rs

openai_protocol/
completion.rs
