openai_interface/
vllm.rs

1//! vLLM's proprietary extensions to the OpenAI-compatible API.
2//!
3//! Everything in this module is gated on the `vllm` cargo feature.
4//!
5//! vLLM serves models behind OpenAI-compatible endpoints, but accepts a
6//! large set of request parameters that OpenAI's API does not define, and
7//! returns extra fields OpenAI never sends. With the official Python client
8//! those parameters have to be smuggled through `extra_body`; here they are
9//! typed.
10//!
11//! The types are split in two because the two text-generation endpoints do
12//! not accept the same set:
13//!
14//! - [`SamplingParams`] — decoding knobs, accepted by both
15//!   `/v1/chat/completions` and the legacy `/v1/completions`.
16//! - [`ChatParams`] — chat-template rendering, structured outputs, KV
17//!   transfer and scheduling, accepted by `/v1/chat/completions` only.
18//!
19//! Both are `#[serde(flatten)]`ed into their request bodies, so the wire
20//! format is identical to what `extra_body` produces: every field lands at
21//! the top level of the JSON body.
22//!
23//! ```rust
24//! # #[cfg(feature = "vllm")] {
25//! use openai_interface::chat::create::request::{Message, RequestBody};
26//! use openai_interface::vllm::SamplingParams;
27//!
28//! let request = RequestBody {
29//!     messages: vec![Message::user("Hello")],
30//!     model: "Qwen/Qwen3-8B".to_string(),
31//!     vllm_sampling: Some(SamplingParams {
32//!         min_p: Some(0.1),
33//!         repetition_penalty: Some(1.05),
34//!         ..Default::default()
35//!     }),
36//!     ..Default::default()
37//! };
38//!
39//! let json = serde_json::to_value(&request).unwrap();
40//! assert_eq!(json["min_p"], serde_json::json!(0.1f32));
41//! assert_eq!(json["repetition_penalty"], serde_json::json!(1.05f32));
42//! # }
43//! ```
44//!
45//! # Note on shared keys
46//!
47//! Two keys vLLM accepts are not defined here because another provider spells
48//! them identically, so they live once on
49//! [`RequestBody`](crate::chat::create::request::RequestBody) instead —
50//! defining them in both places would emit the key twice:
51//!
52//! - `top_k`, gated on `any(feature = "qwen", feature = "vllm")`.
53//! - `request_id` (vLLM's caller-chosen identifier replacing the generated
54//!   UUID), gated on `any(feature = "vllm", feature = "zai")` because Z.ai /
55//!   GLM uses the same key.
56//!
57//! # Differences that need no extra field
58//!
59//! Some vLLM divergences are behavioural, and are documented where the
60//! affected OpenAI-standard field lives:
61//!
62//! - `suffix` is rejected outright on `/v1/completions`.
63//! - `image_url.detail` is rejected outright.
64//! - `response_format.json_schema.strict` is parsed but has no effect:
65//!   guided decoding always enforces the schema.
66//! - When both `max_tokens` and `max_completion_tokens` are set,
67//!   `max_tokens` wins.
68//! - The chain of thought arrives as `reasoning`, not `reasoning_content` —
69//!   see the `reasoning` fields on the response message and streamed delta.
70//!
71//! # Not implemented
72//!
73//! vLLM also exposes endpoints OpenAI has no equivalent of: `/v1/score`,
74//! `/rerank`, `/pooling`, `/classify`, `/tokenize`, `/detokenize`, the
75//! `POST /v1/{chat/completions,completions}/render` prompt-rendering
76//! endpoints, `POST /v1/chat/completions/batch`, and the admin endpoints
77//! (`/load`, `/sleep`, `/wake_up`, `/is_sleeping`, `/reset_prefix_cache`).
78//! None of them are modelled by this crate; this feature covers the fields
79//! vLLM adds to the OpenAI endpoints it does implement.
80//!
81//! See [vLLM's OpenAI-compatible server
82//! docs](https://docs.vllm.ai/en/latest/serving/online_serving/openai_compatible_server/).
83#![cfg(feature = "vllm")]
84
85use serde::{Deserialize, Serialize};
86
87/// Extra sampling parameters accepted by both `/v1/chat/completions` and
88/// `/v1/completions` on a vLLM server.
89///
90/// Flattened into the request body; see the [module docs](crate::vllm).
91/// Every field is optional and omitted from the JSON when unset, so vLLM
92/// applies its own default.
93#[derive(Serialize, Deserialize, Debug, Clone, Default)]
94pub struct SamplingParams {
95    /// Lower bound on the probability mass of the candidate tokens, relative
96    /// to the most likely one. vLLM's default is `0.0` (disabled).
97    #[serde(skip_serializing_if = "Option::is_none")]
98    pub min_p: Option<f32>,
99
100    /// Multiplicative penalty applied to tokens that have already appeared.
101    /// vLLM's default is `1.0` (no penalty). Not the same knob as OpenAI's
102    /// `frequency_penalty` / `presence_penalty`; both may be combined.
103    #[serde(skip_serializing_if = "Option::is_none")]
104    pub repetition_penalty: Option<f32>,
105
106    /// Additional token IDs that end generation, on top of the model's own
107    /// EOS token and any `stop` strings.
108    #[serde(skip_serializing_if = "Option::is_none")]
109    pub stop_token_ids: Option<Vec<u32>>,
110
111    /// Generate exactly `max_tokens` tokens, ignoring EOS. Useful for
112    /// benchmarking; the output usually ends mid-sentence.
113    #[serde(skip_serializing_if = "Option::is_none")]
114    pub ignore_eos: Option<bool>,
115
116    /// Suppress EOS until at least this many tokens have been generated, so
117    /// short outputs cannot be cut off early. vLLM's default is `0`.
118    #[serde(skip_serializing_if = "Option::is_none")]
119    pub min_tokens: Option<u32>,
120
121    /// Strip special tokens from the decoded text. vLLM's default is `true`.
122    #[serde(skip_serializing_if = "Option::is_none")]
123    pub skip_special_tokens: Option<bool>,
124
125    /// Keep a space between adjacent special tokens when decoding. vLLM's
126    /// default is `true`.
127    #[serde(skip_serializing_if = "Option::is_none")]
128    pub spaces_between_special_tokens: Option<bool>,
129
130    /// Include the matched `stop` string in the returned text instead of
131    /// trimming it. vLLM's default is `false`.
132    #[serde(skip_serializing_if = "Option::is_none")]
133    pub include_stop_str_in_output: Option<bool>,
134
135    /// Truncate the prompt to at most this many tokens. `-1` disables
136    /// truncation (vLLM's default); any other value must be positive.
137    #[serde(skip_serializing_if = "Option::is_none")]
138    pub truncate_prompt_tokens: Option<i64>,
139
140    /// Which end of an over-long prompt `truncate_prompt_tokens` removes.
141    #[serde(skip_serializing_if = "Option::is_none")]
142    pub truncation_side: Option<TruncationSide>,
143
144    /// Number of most-likely prompt tokens to report log probabilities for,
145    /// per prompt position. Returned in the response's `prompt_logprobs`
146    /// field. OpenAI has no equivalent.
147    #[serde(skip_serializing_if = "Option::is_none")]
148    pub prompt_logprobs: Option<u32>,
149
150    /// Report the log probability of these specific token IDs at each
151    /// generated position, in addition to the sampled token.
152    #[serde(skip_serializing_if = "Option::is_none")]
153    pub logprob_token_ids: Option<Vec<u32>>,
154
155    /// Restrict decoding to exactly these token IDs.
156    #[serde(skip_serializing_if = "Option::is_none")]
157    pub allowed_token_ids: Option<Vec<u32>>,
158
159    /// Substrings that must not appear in the output. Matching runs over the
160    /// decoded text, so a bad word spanning a token boundary is still caught.
161    #[serde(skip_serializing_if = "Option::is_none")]
162    pub bad_words: Option<Vec<String>>,
163
164    /// Exponential length penalty applied when scoring sequences. Only
165    /// meaningful for beam search; vLLM's default is `1.0`.
166    #[serde(skip_serializing_if = "Option::is_none")]
167    pub length_penalty: Option<f32>,
168
169    /// Deprecated: beam search was removed from the vLLM engine. Accepted
170    /// for wire compatibility only; do not rely on it.
171    #[serde(skip_serializing_if = "Option::is_none")]
172    pub use_beam_search: Option<bool>,
173
174    /// Embed an invisible watermark in the generated text. vLLM's default is
175    /// `true` when the server was started with watermarking support.
176    #[serde(skip_serializing_if = "Option::is_none")]
177    pub watermarking: Option<bool>,
178}
179
180/// Extra `/v1/chat/completions` parameters of a vLLM server: chat-template
181/// rendering, structured output, KV transfer and scheduling.
182///
183/// Flattened into the request body; see the [module docs](crate::vllm).
184/// Rejected by the legacy `/v1/completions` endpoint.
185#[derive(Serialize, Deserialize, Debug, Clone, Default)]
186pub struct ChatParams {
187    /// Echo the rendered prompt back at the start of the completion.
188    #[serde(skip_serializing_if = "Option::is_none")]
189    pub echo: Option<bool>,
190
191    /// Append the template's generation prompt (e.g. `<|assistant|>`) after
192    /// the last message. vLLM's default is `true`.
193    #[serde(skip_serializing_if = "Option::is_none")]
194    pub add_generation_prompt: Option<bool>,
195
196    /// Instead of appending a generation prompt, continue the final message
197    /// as if the assistant had already started it. Mutually exclusive with
198    /// `add_generation_prompt`.
199    #[serde(skip_serializing_if = "Option::is_none")]
200    pub continue_final_message: Option<bool>,
201
202    /// Prepend the tokenizer's BOS token to the rendered prompt. vLLM's
203    /// default is `false`, because the chat template normally emits it.
204    #[serde(skip_serializing_if = "Option::is_none")]
205    pub add_special_tokens: Option<bool>,
206
207    /// Override the model's Jinja chat template with this one, for this
208    /// request only.
209    #[serde(skip_serializing_if = "Option::is_none")]
210    pub chat_template: Option<String>,
211
212    /// Extra variables handed to the chat template. This is how
213    /// template-defined switches are set — e.g. Qwen3's `enable_thinking`
214    /// when the model is served by vLLM rather than by DashScope:
215    /// `{"enable_thinking": false}`.
216    #[serde(skip_serializing_if = "Option::is_none")]
217    pub chat_template_kwargs: Option<serde_json::Map<String, serde_json::Value>>,
218
219    /// Documents made available to templates that take a `documents`
220    /// variable, as a list of string-to-string maps.
221    #[serde(skip_serializing_if = "Option::is_none")]
222    pub documents: Option<Vec<std::collections::HashMap<String, String>>>,
223
224    /// Per-request overrides for the multimodal processor (image resizing,
225    /// video frame sampling, and similar model-specific knobs).
226    #[serde(skip_serializing_if = "Option::is_none")]
227    pub mm_processor_kwargs: Option<serde_json::Map<String, serde_json::Value>>,
228
229    /// Per-request overrides for the multimodal I/O layer, keyed by modality.
230    #[serde(skip_serializing_if = "Option::is_none")]
231    pub media_io_kwargs: Option<serde_json::Map<String, serde_json::Value>>,
232
233    /// Constrain decoding to a grammar. vLLM's successor to the deprecated
234    /// top-level `guided_json` / `guided_regex` / `guided_choice` /
235    /// `guided_grammar` keys.
236    #[serde(skip_serializing_if = "Option::is_none")]
237    pub structured_outputs: Option<StructuredOutputsParams>,
238
239    /// Scheduling priority; higher is served first. Requires the server to
240    /// run with `--scheduling-policy priority`. The `X-Vllm-Priority`
241    /// request header overrides this field.
242    #[serde(skip_serializing_if = "Option::is_none")]
243    pub priority: Option<i64>,
244
245    /// Session identifier, used to keep related requests on the same engine
246    /// for KV-cache reuse.
247    #[serde(skip_serializing_if = "Option::is_none")]
248    pub session_id: Option<String>,
249
250    /// Prefix-cache isolation: requests with a different salt never reuse
251    /// each other's cached blocks.
252    #[serde(skip_serializing_if = "Option::is_none")]
253    pub cache_salt: Option<String>,
254
255    /// Minimum number of generated tokens between streamed chunks, used to
256    /// batch output and cut per-chunk overhead. `1` streams every token.
257    #[serde(skip_serializing_if = "Option::is_none")]
258    pub stream_interval: Option<u32>,
259
260    /// Parameters for disaggregated prefill (KV-cache transfer between
261    /// instances). Echoed back on the response.
262    #[serde(skip_serializing_if = "Option::is_none")]
263    pub kv_transfer_params: Option<serde_json::Map<String, serde_json::Value>>,
264
265    /// Parameters for encoder-cache transfer between instances. Echoed back
266    /// on the response.
267    #[serde(skip_serializing_if = "Option::is_none")]
268    pub ec_transfer_params: Option<serde_json::Map<String, serde_json::Value>>,
269
270    /// Report generated text as the literal token-ID strings
271    /// (`token_id:1234`) instead of decoded text.
272    #[serde(skip_serializing_if = "Option::is_none")]
273    pub return_tokens_as_token_ids: Option<bool>,
274
275    /// Add a `token_ids` list to each choice, alongside the decoded text.
276    #[serde(skip_serializing_if = "Option::is_none")]
277    pub return_token_ids: Option<bool>,
278
279    /// Add per-token character offsets to the rendered prompt. Requires a
280    /// fast tokenizer and text-only input.
281    #[serde(skip_serializing_if = "Option::is_none")]
282    pub return_token_offsets: Option<bool>,
283
284    /// Return the fully rendered prompt text in `prompt_text`.
285    #[serde(skip_serializing_if = "Option::is_none")]
286    pub return_prompt_text: Option<bool>,
287
288    /// Abort generation when the output starts repeating an n-gram pattern.
289    #[serde(skip_serializing_if = "Option::is_none")]
290    pub repetition_detection: Option<RepetitionDetectionParams>,
291
292    /// Escape hatch for parameters vLLM added after this crate was released:
293    /// string-keyed scalars or lists passed straight to the engine.
294    #[serde(skip_serializing_if = "Option::is_none")]
295    pub vllm_xargs: Option<serde_json::Map<String, serde_json::Value>>,
296
297    /// First prompt position for which to report per-token expert routing.
298    /// Requires the server to run with `--enable-return-routed-experts`.
299    #[serde(skip_serializing_if = "Option::is_none")]
300    pub routed_experts_prompt_start: Option<u32>,
301}
302
303/// vLLM's `structured_outputs` parameter: constrains decoding to a grammar
304/// so the output is guaranteed to parse.
305///
306/// Exactly one of `json`, `regex`, `choice`, `grammar`, `json_object` and
307/// `structural_tag` must be set; the server rejects zero or more than one.
308///
309/// Replaces the deprecated top-level `guided_json`, `guided_regex`,
310/// `guided_choice`, `guided_grammar`, `guided_decoding_backend`,
311/// `guided_whitespace_pattern` and `structural_tag` keys, which current
312/// vLLM no longer accepts at all — sending one is a `400` reporting the
313/// unexpected keyword.
314#[derive(Serialize, Deserialize, Debug, Clone, Default)]
315pub struct StructuredOutputsParams {
316    /// JSON Schema the output must validate against. vLLM accepts either an
317    /// inline schema object or a schema serialized as a string.
318    #[serde(skip_serializing_if = "Option::is_none")]
319    pub json: Option<serde_json::Value>,
320
321    /// Regular expression the whole output must match.
322    #[serde(skip_serializing_if = "Option::is_none")]
323    pub regex: Option<String>,
324
325    /// Closed set of allowed outputs; the model must produce exactly one.
326    #[serde(skip_serializing_if = "Option::is_none")]
327    pub choice: Option<Vec<String>>,
328
329    /// Context-free grammar in EBNF, e.g. for a query language.
330    #[serde(skip_serializing_if = "Option::is_none")]
331    pub grammar: Option<String>,
332
333    /// Require syntactically valid JSON without constraining its schema.
334    /// Equivalent to OpenAI's `response_format: {"type": "json_object"}`.
335    #[serde(skip_serializing_if = "Option::is_none")]
336    pub json_object: Option<bool>,
337
338    /// A structural tag interleaving schema fragments with free text.
339    #[serde(skip_serializing_if = "Option::is_none")]
340    pub structural_tag: Option<String>,
341
342    /// Forbid whitespace between JSON tokens, producing compact output.
343    #[serde(skip_serializing_if = "Option::is_none")]
344    pub disable_any_whitespace: Option<bool>,
345
346    /// Reject input schemas that do not set
347    /// `additionalProperties: false`, which guided decoding needs.
348    #[serde(skip_serializing_if = "Option::is_none")]
349    pub disable_additional_properties: Option<bool>,
350
351    /// Regex describing the whitespace allowed between JSON tokens.
352    #[serde(skip_serializing_if = "Option::is_none")]
353    pub whitespace_pattern: Option<String>,
354}
355
356/// vLLM's `repetition_detection` parameter: aborts generation once the
357/// output settles into a repeated n-gram.
358///
359/// Detection is off unless `max_pattern_size` is at least 1 and `min_count`
360/// is at least 2; `min_pattern_size` must not exceed `max_pattern_size`.
361#[derive(Serialize, Deserialize, Debug, Clone, Default)]
362pub struct RepetitionDetectionParams {
363    /// Largest n-gram size to look for. `0` disables detection.
364    #[serde(skip_serializing_if = "Option::is_none")]
365    pub max_pattern_size: Option<u32>,
366
367    /// Smallest n-gram size to look for. Defaults to `1` when `0`.
368    #[serde(skip_serializing_if = "Option::is_none")]
369    pub min_pattern_size: Option<u32>,
370
371    /// How often an n-gram must repeat before generation is cut off.
372    #[serde(skip_serializing_if = "Option::is_none")]
373    pub min_count: Option<u32>,
374}
375
376crate::wire_string_enum! {
377    /// Which end of an over-long prompt `truncate_prompt_tokens` removes.
378    pub enum TruncationSide {
379        /// Drop tokens from the beginning, keeping the most recent context.
380        Left => "left",
381        /// Drop tokens from the end, keeping the earliest context.
382        Right => "right",
383    }
384}
385
386/// vLLM's log probability of one token at one position, as reported in the
387/// `prompt_logprobs` response field.
388///
389/// OpenAI's logprob objects carry `token` / `logprob` / `bytes` /
390/// `top_logprobs`; vLLM's prompt logprobs instead key each position's map by
391/// token ID and report the vocabulary rank plus the decoded text.
392#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
393pub struct Logprob {
394    /// The log probability of this token.
395    pub logprob: f64,
396    /// This token's rank in the vocabulary at this position, starting at 1.
397    /// Omitted when the server did not compute it.
398    pub rank: Option<u32>,
399    /// The decoded text of this token.
400    pub decoded_token: Option<String>,
401}
402
403/// Why generation stopped, as reported in vLLM's `stop_reason` response
404/// field.
405///
406/// Not part of the OpenAI schema. It complements `finish_reason`, which only
407/// says `stop`: `stop_reason` names the *specific* terminator that fired.
408#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
409#[serde(untagged)]
410pub enum StopReason {
411    /// The stop string that ended generation, when a `stop` entry matched.
412    Text(String),
413    /// The token ID that ended generation, when `stop_token_ids` or the
414    /// model's own EOS token matched.
415    TokenId(u32),
416}
417
418#[cfg(test)]
419mod tests {
420    use super::*;
421    use crate::chat::create::request::{Message, RequestBody};
422
423    /// The vLLM sampling parameters must land at the top level of the body,
424    /// exactly where the official client's `extra_body` puts them.
425    #[test]
426    fn sampling_params_flatten_to_top_level() {
427        let request = RequestBody {
428            messages: vec![Message::user("Hello")],
429            model: "Qwen/Qwen3-8B".to_string(),
430            vllm_sampling: Some(SamplingParams {
431                min_p: Some(0.1),
432                repetition_penalty: Some(1.05),
433                stop_token_ids: Some(vec![151645, 151643]),
434                ignore_eos: Some(false),
435                min_tokens: Some(16),
436                truncate_prompt_tokens: Some(-1),
437                truncation_side: Some(TruncationSide::Left),
438                prompt_logprobs: Some(3),
439                allowed_token_ids: Some(vec![1, 2, 3]),
440                bad_words: Some(vec!["<|im_start|>".to_string()]),
441                ..Default::default()
442            }),
443            ..Default::default()
444        };
445
446        let json = serde_json::to_value(&request).unwrap();
447        assert_eq!(json["min_p"], serde_json::json!(0.1f32));
448        assert_eq!(json["repetition_penalty"], serde_json::json!(1.05f32));
449        assert_eq!(json["stop_token_ids"], serde_json::json!([151645, 151643]));
450        assert_eq!(json["ignore_eos"], false);
451        assert_eq!(json["min_tokens"], 16);
452        assert_eq!(json["truncate_prompt_tokens"], -1);
453        assert_eq!(json["truncation_side"], "left");
454        assert_eq!(json["prompt_logprobs"], 3);
455        assert_eq!(json["bad_words"], serde_json::json!(["<|im_start|>"]));
456        // Unset fields stay off the wire so vLLM applies its own defaults.
457        assert!(json.get("skip_special_tokens").is_none());
458        assert!(json.get("watermarking").is_none());
459    }
460
461    /// The chat-only parameters, including the nested `structured_outputs`
462    /// object and the free-form template kwargs.
463    #[test]
464    fn chat_params_flatten_to_top_level() {
465        let request = RequestBody {
466            messages: vec![Message::user("Hello")],
467            model: "Qwen/Qwen3-8B".to_string(),
468            vllm_chat: Some(ChatParams {
469                chat_template_kwargs: Some(
470                    serde_json::from_value(serde_json::json!({"enable_thinking": false})).unwrap(),
471                ),
472                structured_outputs: Some(StructuredOutputsParams {
473                    choice: Some(vec!["positive".to_string(), "negative".to_string()]),
474                    disable_any_whitespace: Some(true),
475                    ..Default::default()
476                }),
477                priority: Some(10),
478                kv_transfer_params: Some(
479                    serde_json::from_value(serde_json::json!({"do_remote_decode": true})).unwrap(),
480                ),
481                ..Default::default()
482            }),
483            ..Default::default()
484        };
485
486        let json = serde_json::to_value(&request).unwrap();
487        assert_eq!(json["chat_template_kwargs"]["enable_thinking"], false);
488        assert_eq!(
489            json["structured_outputs"],
490            serde_json::json!({"choice": ["positive", "negative"], "disable_any_whitespace": true})
491        );
492        assert_eq!(json["priority"], 10);
493        assert_eq!(json["kv_transfer_params"]["do_remote_decode"], true);
494        assert!(json.get("echo").is_none());
495    }
496
497    /// Structured outputs accept a schema either inline or as a string.
498    #[test]
499    fn structured_outputs_json_accepts_object_and_string() {
500        let inline = StructuredOutputsParams {
501            json: Some(serde_json::json!({"type": "object"})),
502            ..Default::default()
503        };
504        assert_eq!(
505            serde_json::to_value(&inline).unwrap()["json"],
506            serde_json::json!({"type": "object"})
507        );
508
509        let as_string = StructuredOutputsParams {
510            json: Some(serde_json::json!(r#"{"type":"object"}"#)),
511            ..Default::default()
512        };
513        assert_eq!(
514            serde_json::to_value(&as_string).unwrap()["json"],
515            serde_json::json!(r#"{"type":"object"}"#)
516        );
517    }
518
519    /// A vLLM request body must survive a parse-then-serialize round trip
520    /// without losing or duplicating any key — the crate is used to build
521    /// proxies, and both flattened vLLM structs share the body with the
522    /// `extra_body_map` catch-all.
523    #[test]
524    fn vllm_body_round_trips_without_duplicating_keys() {
525        let json = r#"{
526            "model": "Qwen/Qwen3-8B",
527            "messages": [{"role": "user", "content": "Hello"}],
528            "min_p": 0.1,
529            "prompt_logprobs": 2,
530            "chat_template_kwargs": {"enable_thinking": true},
531            "structured_outputs": {"regex": "[a-z]+"},
532            "some_future_vllm_field": 42
533        }"#;
534
535        let parsed: RequestBody = serde_json::from_str(json).unwrap();
536
537        let sampling = parsed.vllm_sampling.as_ref().expect("vllm_sampling");
538        assert_eq!(sampling.min_p, Some(0.1));
539        assert_eq!(sampling.prompt_logprobs, Some(2));
540
541        let chat = parsed.vllm_chat.as_ref().expect("vllm_chat");
542        assert_eq!(
543            chat.chat_template_kwargs
544                .as_ref()
545                .expect("chat_template_kwargs")["enable_thinking"],
546            serde_json::json!(true)
547        );
548        assert_eq!(
549            chat.structured_outputs
550                .as_ref()
551                .expect("structured_outputs")
552                .regex
553                .as_deref(),
554            Some("[a-z]+")
555        );
556
557        // Only the genuinely unknown key may fall through to the catch-all;
558        // if the flattened structs failed to claim their own keys, they
559        // would appear here too and be serialized twice.
560        let extra = parsed.extra_body_map.as_ref().expect("extra_body_map");
561        assert_eq!(extra.len(), 1, "extra_body_map: {extra:?}");
562        assert_eq!(extra["some_future_vllm_field"], 42);
563
564        let reserialized = serde_json::to_string(&parsed).unwrap();
565        let reparsed: serde_json::Value = serde_json::from_str(&reserialized).unwrap();
566        assert_eq!(reparsed["min_p"], 0.1);
567        assert_eq!(reparsed["prompt_logprobs"], 2);
568        assert_eq!(reparsed["structured_outputs"]["regex"], "[a-z]+");
569        assert_eq!(reparsed["some_future_vllm_field"], 42);
570        // serde_json collapses duplicate keys silently, so count them in the
571        // raw text instead.
572        assert_eq!(
573            reserialized.matches(r#""min_p""#).count(),
574            1,
575            "duplicated key in {reserialized}"
576        );
577        assert_eq!(
578            reserialized.matches(r#""structured_outputs""#).count(),
579            1,
580            "duplicated key in {reserialized}"
581        );
582    }
583
584    /// With no vLLM parameters set, the body must be byte-identical to a
585    /// plain OpenAI request: nothing leaks onto the wire.
586    #[test]
587    fn unset_vllm_params_add_no_keys() {
588        let request = RequestBody {
589            messages: vec![Message::user("Hello")],
590            model: "gpt-4.1".to_string(),
591            vllm_sampling: Some(SamplingParams::default()),
592            vllm_chat: Some(ChatParams::default()),
593            ..Default::default()
594        };
595
596        let json = serde_json::to_string(&request).unwrap();
597        assert_eq!(
598            json,
599            r#"{"messages":[{"role":"user","content":"Hello"}],"model":"gpt-4.1"}"#
600        );
601    }
602
603    /// The legacy `/v1/completions` body takes the sampling parameters but
604    /// not the chat-only ones.
605    #[test]
606    fn completions_body_takes_sampling_params_only() {
607        let request = crate::completions::request::CompletionRequest {
608            model: "Qwen/Qwen3-8B".to_string(),
609            prompt: crate::completions::request::Prompt::PromptString("Hello".to_string()),
610            vllm_sampling: Some(SamplingParams {
611                min_p: Some(0.1),
612                skip_special_tokens: Some(false),
613                ..Default::default()
614            }),
615            ..Default::default()
616        };
617
618        let json = serde_json::to_value(&request).unwrap();
619        assert_eq!(json["min_p"], serde_json::json!(0.1f32));
620        assert_eq!(json["skip_special_tokens"], false);
621    }
622
623    /// A non-streaming vLLM response carrying every extra field: prompt
624    /// logprobs keyed by token ID, the rendered prompt, `stop_reason` and
625    /// the echoed KV-transfer parameters.
626    #[test]
627    fn non_streaming_response_extras_parse() {
628        use crate::chat::ChatCompletion;
629
630        let json = r#"{
631            "id": "chatcmpl-abc123",
632            "object": "chat.completion",
633            "created": 1735113344,
634            "model": "Qwen/Qwen3-8B",
635            "choices": [{
636                "index": 0,
637                "message": {"role": "assistant", "content": "Hi"},
638                "logprobs": null,
639                "finish_reason": "stop",
640                "stop_reason": "<|im_end|>",
641                "token_ids": [9707, 151645]
642            }],
643            "usage": {"prompt_tokens": 4, "completion_tokens": 2, "total_tokens": 6},
644            "prompt_logprobs": [
645                null,
646                {"9707": {"logprob": -0.001, "rank": 1, "decoded_token": "Hello"}}
647            ],
648            "prompt_token_ids": [151644, 8948, 198, 9707],
649            "prompt_text": "<|im_start|>user\nHello<|im_end|>\n",
650            "kv_transfer_params": {"do_remote_decode": true}
651        }"#;
652
653        let parsed: ChatCompletion = json.parse().expect("vLLM response must parse");
654
655        assert_eq!(
656            parsed.prompt_token_ids.unwrap(),
657            vec![151644, 8948, 198, 9707]
658        );
659        assert!(parsed.prompt_text.unwrap().starts_with("<|im_start|>"));
660        assert_eq!(
661            parsed.kv_transfer_params.unwrap()["do_remote_decode"],
662            serde_json::json!(true)
663        );
664
665        // Position 0 is unreported; position 1 maps token ID 9707 to its logprob.
666        let prompt_logprobs = parsed.prompt_logprobs.expect("prompt_logprobs");
667        assert_eq!(prompt_logprobs.len(), 2);
668        assert!(prompt_logprobs[0].is_none());
669        let entry = &prompt_logprobs[1].as_ref().expect("position 1")[&9707];
670        assert_eq!(entry.rank, Some(1));
671        assert_eq!(entry.decoded_token.as_deref(), Some("Hello"));
672        assert!((entry.logprob - -0.001).abs() < 1e-9);
673
674        let choice = &parsed.choices[0];
675        assert_eq!(choice.token_ids.clone().unwrap(), vec![9707, 151645]);
676        assert_eq!(
677            choice.stop_reason,
678            Some(StopReason::Text("<|im_end|>".to_string()))
679        );
680        assert_eq!(choice.finish_reason.as_str(), "stop");
681    }
682
683    /// `stop_reason` is `int | str` on the wire; both arms must parse, and
684    /// so must a streamed chunk that carries the choice-level extras.
685    #[test]
686    fn stop_reason_accepts_token_id_and_string() {
687        use crate::chat::create::response::streaming::ChatCompletionChunk;
688
689        let by_id: ChatCompletionChunk = r#"{
690            "id": "chatcmpl-abc123", "object": "chat.completion.chunk",
691            "created": 1735113344, "model": "Qwen/Qwen3-8B",
692            "prompt_token_ids": [151644, 9707],
693            "choices": [{
694                "index": 0, "delta": {"role": "assistant", "content": "Hi"},
695                "logprobs": null, "finish_reason": "stop",
696                "stop_reason": 151645, "token_ids": [9707]
697            }]
698        }"#
699        .parse()
700        .expect("chunk with an integer stop_reason must parse");
701
702        assert_eq!(
703            by_id.choices[0].stop_reason,
704            Some(StopReason::TokenId(151645))
705        );
706        assert_eq!(by_id.choices[0].token_ids.clone().unwrap(), vec![9707]);
707        assert_eq!(by_id.prompt_token_ids.clone().unwrap(), vec![151644, 9707]);
708        assert!(by_id.prompt_text.is_none());
709
710        let by_text: ChatCompletionChunk = r#"{
711            "id": "chatcmpl-abc123", "object": "chat.completion.chunk",
712            "created": 1735113344, "model": "Qwen/Qwen3-8B",
713            "choices": [{
714                "index": 0, "delta": {"content": "Hi"},
715                "finish_reason": "stop", "stop_reason": "<|im_end|>"
716            }]
717        }"#
718        .parse()
719        .expect("chunk with a string stop_reason must parse");
720
721        assert_eq!(
722            by_text.choices[0].stop_reason,
723            Some(StopReason::Text("<|im_end|>".to_string()))
724        );
725        assert!(by_text.prompt_token_ids.is_none());
726    }
727
728    /// An OpenAI response with none of the extras must still parse, leaving
729    /// every vLLM field `None` — the feature must not break other providers.
730    #[test]
731    fn plain_openai_response_leaves_vllm_fields_none() {
732        use crate::chat::ChatCompletion;
733
734        let parsed: ChatCompletion = r#"{
735            "id": "chatcmpl-1", "object": "chat.completion", "created": 1,
736            "model": "gpt-4.1",
737            "choices": [{
738                "index": 0,
739                "message": {"role": "assistant", "content": "Hi"},
740                "finish_reason": "stop"
741            }],
742            "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}
743        }"#
744        .parse()
745        .expect("plain OpenAI response must parse with the vllm feature on");
746
747        assert!(parsed.prompt_logprobs.is_none());
748        assert!(parsed.prompt_token_ids.is_none());
749        assert!(parsed.prompt_text.is_none());
750        assert!(parsed.kv_transfer_params.is_none());
751        assert!(parsed.ec_transfer_params.is_none());
752        assert!(parsed.choices[0].stop_reason.is_none());
753        assert!(parsed.choices[0].token_ids.is_none());
754        assert!(parsed.choices[0].routed_experts.is_none());
755    }
756
757    /// vLLM streams the chain of thought as `reasoning`, not the
758    /// cross-vendor `reasoning_content`; the accumulator must pick it up
759    /// and keep it separate from the other key.
760    #[test]
761    fn reasoning_streams_under_vllm_key_and_accumulates() {
762        use crate::chat::create::accumulator::ChatCompletionAccumulator;
763        use crate::chat::create::response::streaming::ChatCompletionChunk;
764
765        let chunks = [
766            r#"{"id":"c","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{"role":"assistant","reasoning":"Let me "},"finish_reason":null}]}"#,
767            r#"{"id":"c","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{"reasoning":"think."},"finish_reason":null}]}"#,
768            r#"{"id":"c","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{"content":"Hi"},"finish_reason":"stop"}]}"#,
769        ];
770
771        let mut accumulator = ChatCompletionAccumulator::new();
772        for chunk in chunks {
773            let parsed: ChatCompletionChunk = chunk.parse().expect("chunk must parse");
774            accumulator.push(&parsed);
775        }
776
777        assert_eq!(accumulator.reasoning(), "Let me think.");
778        // The two keys are distinct fields; vLLM never fills the other one.
779        assert_eq!(accumulator.reasoning_content(), "");
780        assert_eq!(accumulator.content(), "Hi");
781
782        let message = accumulator.into_message();
783        assert_eq!(message.reasoning.as_deref(), Some("Let me think."));
784        assert_eq!(message.reasoning_content, None);
785        assert_eq!(message.content.as_deref(), Some("Hi"));
786    }
787
788    /// vLLM reports speculative-decoding hits in the usage breakdown, and
789    /// prompt caching through the standard `prompt_tokens_details` rather
790    /// than a field of its own.
791    #[test]
792    fn usage_extras_parse() {
793        use crate::chat::ChatCompletion;
794
795        let parsed: ChatCompletion = r#"{
796            "id": "chatcmpl-1", "object": "chat.completion", "created": 1,
797            "model": "Qwen/Qwen3-8B",
798            "choices": [{
799                "index": 0,
800                "message": {"role": "assistant", "content": "Hi"},
801                "finish_reason": "stop"
802            }],
803            "usage": {
804                "prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120,
805                "prompt_tokens_details": {"cached_tokens": 64},
806                "completion_tokens_details": {
807                    "reasoning_tokens": null,
808                    "num_speculative_tokens": 8
809                }
810            }
811        }"#
812        .parse()
813        .expect("vLLM usage must parse");
814
815        let usage = parsed.usage.expect("usage");
816        assert_eq!(
817            usage
818                .completion_tokens_details
819                .as_ref()
820                .expect("completion_tokens_details")
821                .num_speculative_tokens,
822            Some(8)
823        );
824        assert_eq!(
825            usage
826                .prompt_tokens_details
827                .as_ref()
828                .expect("prompt_tokens_details")
829                .cached_tokens,
830            Some(64)
831        );
832    }
833
834    /// vLLM's `/v1/models` entries carry `root`, `parent` and
835    /// `max_model_len` on top of the OpenAI model object.
836    #[test]
837    fn model_card_extras_parse() {
838        let models: crate::models::list::response::ListModelsResponse = r#"{
839            "object": "list",
840            "data": [{
841                "id": "Qwen/Qwen3-8B",
842                "object": "model",
843                "created": 1735113344,
844                "owned_by": "vllm",
845                "root": "Qwen/Qwen3-8B",
846                "parent": null,
847                "max_model_len": 131072
848            }]
849        }"#
850        .parse()
851        .expect("vLLM model list must parse");
852
853        let model = &models.data[0];
854        assert_eq!(model.root.as_deref(), Some("Qwen/Qwen3-8B"));
855        assert_eq!(model.parent, None);
856        assert_eq!(model.max_model_len, Some(131072));
857        assert_eq!(model.owned_by, "vllm");
858    }
859}
openai_interface/vllm.rs

openai_interface/
vllm.rs
