dynamo_protocols/types/
chat.rs

1// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2// SPDX-License-Identifier: Apache-2.0
3//
4// Re-exports upstream async-openai chat types and defines inference-serving
5// extensions on top. Types prefixed with `Dynamo` or entirely absent from the
6// upstream spec are documented with the rationale for the extension.
7
8use std::pin::Pin;
9
10use derive_builder::Builder;
11use futures::Stream;
12use serde::{Deserialize, Serialize};
13use url::Url;
14use uuid::Uuid;
15
16use crate::error::OpenAIError;
17
18// ---------------------------------------------------------------------------
19// Re-exports from upstream async-openai (unchanged types)
20// ---------------------------------------------------------------------------
21// These types are structurally identical to the upstream definitions.
22// Consumers should use them via `dynamo_protocols::types::*` as before.
23
24pub use async_openai::types::chat::{
25    ChatChoiceLogprobs,
26    ChatCompletionAudio,
27    ChatCompletionAudioFormat,
28    ChatCompletionAudioVoice,
29    ChatCompletionFunctionCall,
30    ChatCompletionFunctions,
31    ChatCompletionFunctionsArgs,
32    ChatCompletionRequestAssistantMessageAudio,
33    ChatCompletionRequestAssistantMessageContent,
34    ChatCompletionRequestAssistantMessageContentPart,
35    ChatCompletionRequestDeveloperMessage,
36    ChatCompletionRequestDeveloperMessageArgs,
37    ChatCompletionRequestDeveloperMessageContent,
38    ChatCompletionRequestFunctionMessage,
39    ChatCompletionRequestFunctionMessageArgs,
40    ChatCompletionRequestMessageContentPartAudio,
41    ChatCompletionRequestMessageContentPartRefusal,
42    ChatCompletionRequestMessageContentPartText,
43    ChatCompletionRequestSystemMessage,
44    // Builder types (generated by derive_builder)
45    ChatCompletionRequestSystemMessageArgs,
46    ChatCompletionRequestSystemMessageContent,
47    ChatCompletionRequestSystemMessageContentPart,
48    ChatCompletionRequestToolMessage,
49    ChatCompletionRequestToolMessageArgs,
50    ChatCompletionRequestToolMessageContent,
51    ChatCompletionRequestToolMessageContentPart,
52    ChatCompletionResponseMessageAudio,
53    ChatCompletionTokenLogprob,
54    Choice,
55    CompletionFinishReason,
56    CompletionTokensDetails,
57    CompletionUsage,
58    FunctionObject,
59    FunctionObjectArgs,
60    InputAudio,
61    InputAudioFormat,
62    Logprobs,
63    PredictionContent,
64    PredictionContentContent,
65    Prompt,
66    PromptTokensDetails,
67    ReasoningEffort,
68    ResponseFormat,
69    ResponseFormatJsonSchema,
70    Role,
71    ServiceTier,
72    TopLogprobs,
73    WebSearchContextSize,
74    WebSearchLocation,
75    WebSearchOptions,
76    WebSearchUserLocation,
77    WebSearchUserLocationType,
78};
79
80// Upstream renamed Stop -> StopConfiguration; re-export under old name for compat
81pub use async_openai::types::chat::StopConfiguration as Stop;
82
83// Upstream renamed FinishReason (streaming) -- re-export
84pub use async_openai::types::chat::FinishReason;
85
86// Upstream uses FunctionType where we used ChatCompletionToolType.
87// Re-export both names for compatibility.
88pub use async_openai::types::chat::FunctionType;
89
90// ---------------------------------------------------------------------------
91// Flexible `arguments` deserialisation helpers
92// ---------------------------------------------------------------------------
93// Some agent frameworks (e.g. LangChain, custom harnesses) send tool-call
94// arguments as a pre-parsed JSON object instead of the canonical JSON
95// string.  The helpers below normalise both representations to a `String` so
96// downstream code never needs to branch on the wire format.
97
98fn deserialize_arguments<'de, D>(deserializer: D) -> Result<String, D::Error>
99where
100    D: serde::Deserializer<'de>,
101{
102    use serde::de::Error;
103    let value = serde_json::Value::deserialize(deserializer)?;
104    match value {
105        serde_json::Value::String(s) => Ok(s),
106        v @ serde_json::Value::Object(_) => {
107            // serde_json::to_string on a Value is infallible
108            Ok(serde_json::to_string(&v).unwrap())
109        }
110        other => Err(D::Error::custom(format!(
111            "expected string or object for `arguments`, got {other}"
112        ))),
113    }
114}
115
116fn deserialize_arguments_opt<'de, D>(deserializer: D) -> Result<Option<String>, D::Error>
117where
118    D: serde::Deserializer<'de>,
119{
120    use serde::de::Error;
121    let value = Option::<serde_json::Value>::deserialize(deserializer)?;
122    match value {
123        None => Ok(None),
124        Some(serde_json::Value::String(s)) => Ok(Some(s)),
125        Some(v @ serde_json::Value::Object(_)) => serde_json::to_string(&v)
126            .map(Some)
127            .map_err(|e| D::Error::custom(e.to_string())),
128        Some(other) => Err(D::Error::custom(format!(
129            "expected string or object for `arguments`, got {other}"
130        ))),
131    }
132}
133
134// ---------------------------------------------------------------------------
135// FunctionCall / FunctionCallStream — local definitions with flexible deser
136// ---------------------------------------------------------------------------
137// Upstream `async-openai` only accepts a JSON string for `arguments`.
138// We define these locally so we can attach `#[serde(deserialize_with)]` and
139// accept both string and object representations on the wire.
140
141/// The name and arguments of a function that should be called.
142///
143/// Accepts `arguments` as either a JSON string (`"{\"key\":\"value\"}"`) or a
144/// JSON object (`{"key": "value"}`); both are normalised to a JSON string
145/// on deserialisation so callers always see the canonical form.
146#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
147pub struct FunctionCall {
148    pub name: String,
149    #[serde(deserialize_with = "deserialize_arguments")]
150    pub arguments: String,
151}
152
153/// Streaming variant of [`FunctionCall`] where both fields are optional.
154#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
155pub struct FunctionCallStream {
156    pub name: Option<String>,
157    #[serde(default, deserialize_with = "deserialize_arguments_opt")]
158    pub arguments: Option<String>,
159}
160
161/// Streaming tool-call chunk.
162///
163/// Defined locally (instead of re-exporting from upstream) because its
164/// `function` field references our local [`FunctionCallStream`] with the
165/// flexible `arguments` deserialiser.
166#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
167pub struct ChatCompletionMessageToolCallChunk {
168    pub index: u32,
169    pub id: Option<String>,
170    pub r#type: Option<FunctionType>,
171    pub function: Option<FunctionCallStream>,
172}
173
174// ---------------------------------------------------------------------------
175// Types with structural differences from upstream (kept locally)
176// ---------------------------------------------------------------------------
177
178/// Image detail level. Kept locally because upstream uses different field types in ImageUrl.
179#[derive(Debug, Serialize, Deserialize, Default, Clone, PartialEq)]
180#[serde(rename_all = "lowercase")]
181pub enum ImageDetail {
182    #[default]
183    Auto,
184    Low,
185    High,
186}
187
188/// Image content part -- uses our extended `ImageUrl` with `url::Url` and `uuid`.
189#[derive(Debug, Serialize, Deserialize, Clone, Builder, PartialEq)]
190#[builder(name = "ChatCompletionRequestMessageContentPartImageArgs")]
191#[builder(pattern = "mutable")]
192#[builder(setter(into, strip_option))]
193#[builder(derive(Debug))]
194#[builder(build_fn(error = "OpenAIError"))]
195pub struct ChatCompletionRequestMessageContentPartImage {
196    pub image_url: ImageUrl,
197}
198
199/// Image URL with `url::Url` type and optional UUID.
200///
201/// Differs from upstream: uses `url::Url` instead of `String`, adds `uuid` field
202/// for tracking multimodal assets through the pipeline.
203#[derive(Debug, Serialize, Deserialize, Clone, Builder, PartialEq)]
204#[builder(name = "ImageUrlArgs")]
205#[builder(pattern = "mutable")]
206#[builder(setter(into, strip_option))]
207#[builder(derive(Debug))]
208#[builder(build_fn(error = "OpenAIError"))]
209pub struct ImageUrl {
210    pub url: Url,
211    pub detail: Option<ImageDetail>,
212    #[serde(skip_serializing_if = "Option::is_none")]
213    pub uuid: Option<Uuid>,
214}
215
216#[derive(Clone, Serialize, Default, Debug, Deserialize, PartialEq)]
217#[serde(rename_all = "lowercase")]
218pub enum ChatCompletionToolType {
219    #[default]
220    Function,
221}
222
223#[derive(Clone, Serialize, Default, Debug, Deserialize, PartialEq)]
224pub struct FunctionName {
225    pub name: String,
226}
227
228#[derive(Clone, Serialize, Default, Debug, Deserialize, PartialEq)]
229pub struct ChatCompletionNamedToolChoice {
230    pub r#type: ChatCompletionToolType,
231    pub function: FunctionName,
232}
233
234fn default_function_type() -> FunctionType {
235    FunctionType::Function
236}
237
238/// Tool call kept locally to preserve `type: "function"` in unary request/response payloads.
239///
240/// Differs from upstream: `type` is serialized by default and also defaults to
241/// `function` when omitted during deserialization, preserving compatibility with
242/// both Dynamo's historical wire format and upstream spec-compliant inputs.
243#[derive(Clone, Serialize, Debug, Deserialize, PartialEq)]
244pub struct ChatCompletionMessageToolCall {
245    pub id: String,
246    #[serde(default = "default_function_type")]
247    pub r#type: FunctionType,
248    pub function: FunctionCall,
249}
250
251/// Tool choice enum kept locally because upstream changed variant names.
252#[derive(Clone, Serialize, Default, Debug, Deserialize, PartialEq)]
253#[serde(rename_all = "lowercase")]
254pub enum ChatCompletionToolChoiceOption {
255    #[default]
256    None,
257    Auto,
258    Required,
259    #[serde(untagged)]
260    Named(ChatCompletionNamedToolChoice),
261}
262
263#[derive(Clone, Serialize, Default, Debug, Builder, Deserialize, PartialEq)]
264#[builder(name = "ChatCompletionToolArgs")]
265#[builder(pattern = "mutable")]
266#[builder(setter(into, strip_option), default)]
267#[builder(derive(Debug))]
268#[builder(build_fn(error = "OpenAIError"))]
269pub struct ChatCompletionTool {
270    #[builder(default = "ChatCompletionToolType::Function")]
271    pub r#type: ChatCompletionToolType,
272    pub function: FunctionObject,
273}
274
275// ---------------------------------------------------------------------------
276// Inference-serving extensions (not in upstream)
277// ---------------------------------------------------------------------------
278
279/// Matched stop condition from the backend.
280///
281/// Inference backends (vLLM, SGLang) report which stop condition triggered:
282/// - `String`: a matched user-provided stop sequence
283/// - `Int`: a matched stop token ID
284#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
285#[serde(untagged)]
286pub enum StopReason {
287    String(String),
288    Int(i64),
289}
290
291/// Reasoning content from a previous assistant turn.
292///
293/// Deserializes from either:
294/// - A plain string: `"reasoning_content": "thinking..."` -> `Text("thinking...")`
295/// - An array of strings: `"reasoning_content": ["seg1", "seg2"]` -> `Segments(["seg1", "seg2"])`
296///
297/// The `Segments` variant preserves interleaved reasoning order needed for KV cache-correct
298/// context reconstruction. `segments[i]` is the reasoning that preceded `tool_calls[i]`;
299/// `segments[tool_calls.len()]` is any trailing reasoning after the last tool call.
300#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
301#[serde(untagged)]
302pub enum ReasoningContent {
303    /// Flat string -- single reasoning block or legacy backward-compat form.
304    Text(String),
305    /// Interleaved segments. segments[i] precedes tool_calls[i];
306    /// segments[N] is trailing reasoning after the last tool call.
307    Segments(Vec<String>),
308}
309
310impl ReasoningContent {
311    /// Join all segments (or return text as-is) into a single flat string.
312    pub fn to_flat_string(&self) -> String {
313        match self {
314            ReasoningContent::Text(s) => s.clone(),
315            ReasoningContent::Segments(segs) => segs
316                .iter()
317                .filter(|s| !s.is_empty())
318                .cloned()
319                .collect::<Vec<_>>()
320                .join("\n"),
321        }
322    }
323
324    /// Returns the segments if this is the `Segments` variant, `None` for `Text`.
325    pub fn segments(&self) -> Option<&[String]> {
326        match self {
327            ReasoningContent::Segments(segs) => Some(segs),
328            ReasoningContent::Text(_) => None,
329        }
330    }
331}
332
333// -- Multimodal content types for responses (not in upstream) --
334
335/// Response content part for text in assistant messages
336#[derive(Clone, Serialize, Debug, Deserialize, PartialEq)]
337pub struct ChatCompletionResponseContentPartText {
338    pub text: String,
339}
340
341/// Response content part for image URLs in assistant messages
342#[derive(Clone, Serialize, Debug, Deserialize, PartialEq)]
343pub struct ChatCompletionResponseContentPartImageUrl {
344    pub image_url: ImageUrlResponse,
345}
346
347/// Response content part for video URLs in assistant messages
348#[derive(Clone, Serialize, Debug, Deserialize, PartialEq)]
349pub struct ChatCompletionResponseContentPartVideoUrl {
350    pub video_url: VideoUrlResponse,
351}
352
353/// Response content part for audio URLs in assistant messages
354#[derive(Clone, Serialize, Debug, Deserialize, PartialEq)]
355pub struct ChatCompletionResponseContentPartAudioUrl {
356    pub audio_url: AudioUrlResponse,
357}
358
359#[derive(Clone, Serialize, Debug, Deserialize, PartialEq)]
360pub struct ImageUrlResponse {
361    pub url: String,
362    #[serde(skip_serializing_if = "Option::is_none")]
363    pub detail: Option<String>,
364}
365
366#[derive(Clone, Serialize, Debug, Deserialize, PartialEq)]
367pub struct VideoUrlResponse {
368    pub url: String,
369}
370
371#[derive(Clone, Serialize, Debug, Deserialize, PartialEq)]
372pub struct AudioUrlResponse {
373    pub url: String,
374}
375
376/// Content parts for assistant responses supporting multiple modalities
377#[derive(Clone, Serialize, Debug, Deserialize, PartialEq)]
378#[serde(tag = "type", rename_all = "snake_case")]
379pub enum ChatCompletionResponseContentPart {
380    Text(ChatCompletionResponseContentPartText),
381    ImageUrl(ChatCompletionResponseContentPartImageUrl),
382    VideoUrl(ChatCompletionResponseContentPartVideoUrl),
383    AudioUrl(ChatCompletionResponseContentPartAudioUrl),
384}
385
386/// Assistant message content -- can be a simple string or multimodal content parts.
387///
388/// Upstream uses `Option<String>` for the content field. We extend this to
389/// support multimodal responses (text + images + video + audio) from backends
390/// like vLLM that can return non-text content.
391#[derive(Clone, Serialize, Debug, Deserialize, PartialEq)]
392#[serde(untagged)]
393pub enum ChatCompletionMessageContent {
394    /// Simple text content (backward compatible)
395    Text(String),
396    /// Array of content parts (for multimodal responses)
397    Parts(Vec<ChatCompletionResponseContentPart>),
398}
399
400// -- Multimodal input types (video/audio URL support, not in upstream) --
401
402#[derive(Debug, Serialize, Deserialize, Clone, Builder, PartialEq)]
403#[builder(name = "VideoUrlArgs")]
404#[builder(pattern = "mutable")]
405#[builder(setter(into, strip_option))]
406#[builder(derive(Debug))]
407#[builder(build_fn(error = "OpenAIError"))]
408pub struct VideoUrl {
409    pub url: Url,
410    pub detail: Option<ImageDetail>,
411    #[serde(skip_serializing_if = "Option::is_none")]
412    pub uuid: Option<Uuid>,
413}
414
415#[derive(Debug, Serialize, Deserialize, Clone, Builder, PartialEq)]
416#[builder(name = "ChatCompletionRequestMessageContentPartVideoArgs")]
417#[builder(pattern = "mutable")]
418#[builder(setter(into, strip_option))]
419#[builder(derive(Debug))]
420#[builder(build_fn(error = "OpenAIError"))]
421pub struct ChatCompletionRequestMessageContentPartVideo {
422    pub video_url: VideoUrl,
423}
424
425#[derive(Debug, Serialize, Deserialize, Clone, Builder, PartialEq)]
426#[builder(name = "AudioUrlArgs")]
427#[builder(pattern = "mutable")]
428#[builder(setter(into, strip_option))]
429#[builder(derive(Debug))]
430#[builder(build_fn(error = "OpenAIError"))]
431pub struct AudioUrl {
432    pub url: Url,
433    #[serde(skip_serializing_if = "Option::is_none")]
434    pub uuid: Option<Uuid>,
435}
436
437#[derive(Debug, Serialize, Deserialize, Clone, Builder, PartialEq)]
438#[builder(name = "ChatCompletionRequestMessageContentPartAudioUrlArgs")]
439#[builder(pattern = "mutable")]
440#[builder(setter(into, strip_option))]
441#[builder(derive(Debug))]
442#[builder(build_fn(error = "OpenAIError"))]
443pub struct ChatCompletionRequestMessageContentPartAudioUrl {
444    pub audio_url: AudioUrl,
445}
446
447// -- Extended request/response types --
448
449/// User message content -- references our extended content part enum.
450#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
451#[serde(untagged)]
452pub enum ChatCompletionRequestUserMessageContent {
453    Text(String),
454    Array(Vec<ChatCompletionRequestUserMessageContentPart>),
455}
456
457#[derive(Debug, Serialize, Deserialize, Default, Clone, Builder, PartialEq)]
458#[builder(name = "ChatCompletionRequestUserMessageArgs")]
459#[builder(pattern = "mutable")]
460#[builder(setter(into, strip_option), default)]
461#[builder(derive(Debug))]
462#[builder(build_fn(error = "OpenAIError"))]
463pub struct ChatCompletionRequestUserMessage {
464    pub content: ChatCompletionRequestUserMessageContent,
465    #[serde(skip_serializing_if = "Option::is_none")]
466    pub name: Option<String>,
467}
468
469impl Default for ChatCompletionRequestUserMessageContent {
470    fn default() -> Self {
471        Self::Text(String::new())
472    }
473}
474
475impl From<&str> for ChatCompletionRequestUserMessageContent {
476    fn from(value: &str) -> Self {
477        Self::Text(value.into())
478    }
479}
480
481impl From<String> for ChatCompletionRequestUserMessageContent {
482    fn from(value: String) -> Self {
483        Self::Text(value)
484    }
485}
486
487impl From<Vec<ChatCompletionRequestUserMessageContentPart>>
488    for ChatCompletionRequestUserMessageContent
489{
490    fn from(value: Vec<ChatCompletionRequestUserMessageContentPart>) -> Self {
491        Self::Array(value)
492    }
493}
494
495/// User message content part with video and audio URL support.
496///
497/// Extends upstream `ChatCompletionRequestUserMessageContentPart` with:
498/// - `VideoUrl`: video input for multimodal models
499/// - `AudioUrl`: audio URL input (distinct from base64 InputAudio)
500#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
501#[serde(tag = "type")]
502#[serde(rename_all = "snake_case")]
503pub enum ChatCompletionRequestUserMessageContentPart {
504    Text(ChatCompletionRequestMessageContentPartText),
505    ImageUrl(ChatCompletionRequestMessageContentPartImage),
506    VideoUrl(ChatCompletionRequestMessageContentPartVideo),
507    AudioUrl(ChatCompletionRequestMessageContentPartAudioUrl),
508    InputAudio(ChatCompletionRequestMessageContentPartAudio),
509}
510
511/// Assistant message with reasoning content support.
512///
513/// Extends upstream `ChatCompletionRequestAssistantMessage` with:
514/// - `reasoning_content`: interleaved reasoning segments for KV cache correctness
515///   (DeepSeek-R1, QwQ models)
516#[derive(Debug, Serialize, Deserialize, Default, Clone, Builder, PartialEq)]
517#[builder(name = "ChatCompletionRequestAssistantMessageArgs")]
518#[builder(pattern = "mutable")]
519#[builder(setter(into, strip_option), default)]
520#[builder(derive(Debug))]
521#[builder(build_fn(error = "OpenAIError"))]
522pub struct ChatCompletionRequestAssistantMessage {
523    #[serde(skip_serializing_if = "Option::is_none")]
524    pub content: Option<ChatCompletionRequestAssistantMessageContent>,
525    /// Reasoning content from a previous assistant turn.
526    #[serde(skip_serializing_if = "Option::is_none")]
527    pub reasoning_content: Option<ReasoningContent>,
528    #[serde(skip_serializing_if = "Option::is_none")]
529    pub refusal: Option<String>,
530    #[serde(skip_serializing_if = "Option::is_none")]
531    pub name: Option<String>,
532    #[serde(skip_serializing_if = "Option::is_none")]
533    pub audio: Option<ChatCompletionRequestAssistantMessageAudio>,
534    #[serde(skip_serializing_if = "Option::is_none")]
535    pub tool_calls: Option<Vec<ChatCompletionMessageToolCall>>,
536    #[deprecated]
537    #[serde(skip_serializing_if = "Option::is_none")]
538    pub function_call: Option<FunctionCall>,
539}
540
541/// Chat completion request message enum.
542///
543/// Redefined to use our extended `ChatCompletionRequestAssistantMessage`
544/// (with reasoning_content) and `ChatCompletionRequestUserMessage`
545/// (which references our extended content parts with video/audio).
546#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
547#[serde(tag = "role")]
548#[serde(rename_all = "lowercase")]
549pub enum ChatCompletionRequestMessage {
550    Developer(ChatCompletionRequestDeveloperMessage),
551    System(ChatCompletionRequestSystemMessage),
552    User(ChatCompletionRequestUserMessage),
553    Assistant(ChatCompletionRequestAssistantMessage),
554    Tool(ChatCompletionRequestToolMessage),
555    Function(ChatCompletionRequestFunctionMessage),
556}
557
558/// Response tier enum for responses (distinct from request `ServiceTier`).
559///
560/// Not in upstream -- backends report which tier actually served the request.
561#[derive(Clone, Serialize, Debug, Deserialize, PartialEq)]
562#[serde(rename_all = "lowercase")]
563pub enum ServiceTierResponse {
564    Scale,
565    Default,
566    Flex,
567    Priority,
568}
569
570/// Chat completion response message with multimodal content and reasoning.
571///
572/// Extends upstream `ChatCompletionResponseMessage` with:
573/// - `content`: `Option<ChatCompletionMessageContent>` (multimodal) instead of `Option<String>`
574/// - `reasoning_content`: model reasoning output (DeepSeek-R1, QwQ)
575#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
576pub struct ChatCompletionResponseMessage {
577    #[serde(skip_serializing_if = "Option::is_none")]
578    pub content: Option<ChatCompletionMessageContent>,
579    #[serde(skip_serializing_if = "Option::is_none")]
580    pub refusal: Option<String>,
581    #[serde(skip_serializing_if = "Option::is_none")]
582    pub tool_calls: Option<Vec<ChatCompletionMessageToolCall>>,
583    pub role: Role,
584    #[serde(skip_serializing_if = "Option::is_none")]
585    #[deprecated]
586    pub function_call: Option<FunctionCall>,
587    #[serde(skip_serializing_if = "Option::is_none")]
588    pub audio: Option<ChatCompletionResponseMessageAudio>,
589    /// Reasoning content produced by the model (DeepSeek-R1, QwQ).
590    pub reasoning_content: Option<String>,
591}
592
593/// Stream options with per-chunk usage reporting.
594///
595/// Extends upstream `ChatCompletionStreamOptions` with:
596/// - `continuous_usage_stats`: emit usage in every chunk, not just the final one
597#[derive(Debug, Serialize, Deserialize, Clone, Copy, PartialEq)]
598pub struct ChatCompletionStreamOptions {
599    pub include_usage: bool,
600    /// When true, usage statistics are included in every streaming chunk.
601    /// Backends like vLLM/SGLang support this for real-time token counting.
602    #[serde(default)]
603    pub continuous_usage_stats: bool,
604}
605
606/// Chat completion request with multimodal processor support.
607///
608/// Extends upstream `CreateChatCompletionRequest` with:
609/// - `mm_processor_kwargs`: multimodal processor configuration (vLLM-specific)
610/// - Uses our extended `ChatCompletionRequestMessage` (with reasoning, video/audio)
611/// - Uses our extended `ChatCompletionStreamOptions` (with continuous_usage_stats)
612#[derive(Clone, Serialize, Default, Debug, Builder, Deserialize, PartialEq)]
613#[builder(name = "CreateChatCompletionRequestArgs")]
614#[builder(pattern = "mutable")]
615#[builder(setter(into, strip_option), default)]
616#[builder(derive(Debug))]
617#[builder(build_fn(error = "OpenAIError"))]
618pub struct CreateChatCompletionRequest {
619    pub messages: Vec<ChatCompletionRequestMessage>,
620    pub model: String,
621    /// Multimodal processor configuration (vLLM-specific)
622    #[serde(skip_serializing_if = "Option::is_none")]
623    pub mm_processor_kwargs: Option<serde_json::Value>,
624    #[serde(skip_serializing_if = "Option::is_none")]
625    pub store: Option<bool>,
626    #[serde(skip_serializing_if = "Option::is_none")]
627    pub reasoning_effort: Option<ReasoningEffort>,
628    #[serde(skip_serializing_if = "Option::is_none")]
629    pub metadata: Option<serde_json::Value>,
630    #[serde(skip_serializing_if = "Option::is_none")]
631    pub frequency_penalty: Option<f32>,
632    #[serde(skip_serializing_if = "Option::is_none")]
633    pub logit_bias: Option<std::collections::HashMap<String, serde_json::Value>>,
634    #[serde(skip_serializing_if = "Option::is_none")]
635    pub logprobs: Option<bool>,
636    #[serde(skip_serializing_if = "Option::is_none")]
637    pub top_logprobs: Option<u8>,
638    #[deprecated]
639    #[serde(skip_serializing_if = "Option::is_none")]
640    pub max_tokens: Option<u32>,
641    #[serde(skip_serializing_if = "Option::is_none")]
642    pub max_completion_tokens: Option<u32>,
643    #[serde(skip_serializing_if = "Option::is_none")]
644    pub n: Option<u8>,
645    #[serde(skip_serializing_if = "Option::is_none")]
646    pub modalities: Option<Vec<async_openai::types::chat::ResponseModalities>>,
647    #[serde(skip_serializing_if = "Option::is_none")]
648    pub prediction: Option<PredictionContent>,
649    #[serde(skip_serializing_if = "Option::is_none")]
650    pub audio: Option<ChatCompletionAudio>,
651    #[serde(skip_serializing_if = "Option::is_none")]
652    pub presence_penalty: Option<f32>,
653    #[serde(skip_serializing_if = "Option::is_none")]
654    pub response_format: Option<ResponseFormat>,
655    #[serde(skip_serializing_if = "Option::is_none")]
656    pub seed: Option<i64>,
657    #[serde(skip_serializing_if = "Option::is_none")]
658    pub service_tier: Option<ServiceTier>,
659    #[serde(skip_serializing_if = "Option::is_none")]
660    pub stop: Option<Stop>,
661    #[serde(default, skip_serializing_if = "Option::is_none")]
662    pub stream: Option<bool>,
663    #[serde(skip_serializing_if = "Option::is_none")]
664    pub stream_options: Option<ChatCompletionStreamOptions>,
665    #[serde(skip_serializing_if = "Option::is_none")]
666    pub temperature: Option<f32>,
667    #[serde(skip_serializing_if = "Option::is_none")]
668    pub top_p: Option<f32>,
669    #[serde(skip_serializing_if = "Option::is_none")]
670    pub tools: Option<Vec<ChatCompletionTool>>,
671    #[serde(skip_serializing_if = "Option::is_none")]
672    pub tool_choice: Option<ChatCompletionToolChoiceOption>,
673    #[serde(skip_serializing_if = "Option::is_none")]
674    pub parallel_tool_calls: Option<bool>,
675    #[serde(skip_serializing_if = "Option::is_none")]
676    pub user: Option<String>,
677    #[deprecated]
678    #[serde(skip_serializing_if = "Option::is_none")]
679    pub function_call: Option<ChatCompletionFunctionCall>,
680    #[deprecated]
681    #[serde(skip_serializing_if = "Option::is_none")]
682    pub functions: Option<Vec<ChatCompletionFunctions>>,
683    #[serde(skip_serializing_if = "Option::is_none")]
684    pub web_search_options: Option<WebSearchOptions>,
685}
686
687/// Chat choice with extended response message.
688///
689/// Uses our `ChatCompletionResponseMessage` (multimodal content + reasoning).
690#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
691pub struct ChatChoice {
692    pub index: u32,
693    pub message: ChatCompletionResponseMessage,
694    pub finish_reason: Option<FinishReason>,
695    /// Matched stop condition from the backend.
696    #[serde(skip_serializing_if = "Option::is_none")]
697    pub stop_reason: Option<StopReason>,
698    pub logprobs: Option<ChatChoiceLogprobs>,
699}
700
701/// Non-streaming chat completion response.
702#[derive(Debug, Deserialize, Clone, PartialEq, Serialize)]
703pub struct CreateChatCompletionResponse {
704    pub id: String,
705    pub choices: Vec<ChatChoice>,
706    pub created: u32,
707    pub model: String,
708    pub service_tier: Option<ServiceTierResponse>,
709    pub system_fingerprint: Option<String>,
710    pub object: String,
711    pub usage: Option<CompletionUsage>,
712}
713
714pub type ChatCompletionResponseStream =
715    Pin<Box<dyn Stream<Item = Result<CreateChatCompletionStreamResponse, OpenAIError>> + Send>>;
716
717/// Streaming delta with reasoning content.
718///
719/// Extends upstream `ChatCompletionStreamResponseDelta` with:
720/// - `content`: `Option<ChatCompletionMessageContent>` (multimodal) instead of `Option<String>`
721/// - `reasoning_content`: streaming reasoning tokens (DeepSeek-R1, QwQ)
722#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
723pub struct ChatCompletionStreamResponseDelta {
724    #[serde(skip_serializing_if = "Option::is_none")]
725    pub content: Option<ChatCompletionMessageContent>,
726    #[serde(skip_serializing_if = "Option::is_none")]
727    pub function_call: Option<ChatCompletionStreamResponseDeltaFunctionCall>,
728    #[serde(skip_serializing_if = "Option::is_none")]
729    pub tool_calls: Option<Vec<ChatCompletionMessageToolCallChunk>>,
730    #[serde(skip_serializing_if = "Option::is_none")]
731    pub role: Option<Role>,
732    #[serde(skip_serializing_if = "Option::is_none")]
733    pub refusal: Option<String>,
734    /// Streaming reasoning content (DeepSeek-R1, QwQ models).
735    #[serde(skip_serializing_if = "Option::is_none")]
736    pub reasoning_content: Option<String>,
737}
738
739#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
740pub struct ChatCompletionStreamResponseDeltaFunctionCall {
741    pub name: Option<String>,
742    #[serde(default, deserialize_with = "deserialize_arguments_opt")]
743    pub arguments: Option<String>,
744}
745
746/// Streaming chat choice with stop reason support.
747///
748/// Extends upstream `ChatChoiceStream` with:
749/// - `stop_reason`: the matched stop sequence (string) or stop token ID (integer)
750///   reported by inference backends
751#[derive(Debug, Deserialize, Clone, PartialEq, Serialize)]
752pub struct ChatChoiceStream {
753    pub index: u32,
754    pub delta: ChatCompletionStreamResponseDelta,
755    pub finish_reason: Option<FinishReason>,
756    /// Matched stop condition from the backend.
757    #[serde(skip_serializing_if = "Option::is_none")]
758    pub stop_reason: Option<StopReason>,
759    pub logprobs: Option<ChatChoiceLogprobs>,
760}
761
762/// Streaming chat completion response with extended choices.
763#[derive(Debug, Deserialize, Clone, PartialEq, Serialize)]
764pub struct CreateChatCompletionStreamResponse {
765    pub id: String,
766    pub choices: Vec<ChatChoiceStream>,
767    pub created: u32,
768    pub model: String,
769    pub service_tier: Option<ServiceTierResponse>,
770    pub system_fingerprint: Option<String>,
771    pub object: String,
772    pub usage: Option<CompletionUsage>,
773}
774
775#[cfg(test)]
776mod tests {
777    use super::*;
778
779    #[test]
780    fn tool_call_defaults_type_on_deserialize() {
781        let tool_call: ChatCompletionMessageToolCall = serde_json::from_value(serde_json::json!({
782            "id": "call_123",
783            "function": {
784                "name": "get_weather",
785                "arguments": "{\"location\":\"SF\"}"
786            }
787        }))
788        .unwrap();
789
790        assert_eq!(tool_call.r#type, FunctionType::Function);
791    }
792
793    #[test]
794    fn tool_call_serializes_type_for_wire_compat() {
795        let tool_call = ChatCompletionMessageToolCall {
796            id: "call_123".into(),
797            r#type: FunctionType::Function,
798            function: FunctionCall {
799                name: "get_weather".into(),
800                arguments: "{\"location\":\"SF\"}".into(),
801            },
802        };
803
804        let json = serde_json::to_value(tool_call).unwrap();
805        assert_eq!(json["type"], "function");
806    }
807
808    // -- dict-format arguments tests --
809
810    #[test]
811    fn function_call_accepts_string_arguments() {
812        let fc: FunctionCall = serde_json::from_value(serde_json::json!({
813            "name": "get_weather",
814            "arguments": "{\"location\":\"SF\"}"
815        }))
816        .unwrap();
817        assert_eq!(fc.arguments, "{\"location\":\"SF\"}");
818    }
819
820    #[test]
821    fn function_call_accepts_dict_arguments() {
822        let fc: FunctionCall = serde_json::from_value(serde_json::json!({
823            "name": "get_weather",
824            "arguments": {"location": "SF"}
825        }))
826        .unwrap();
827        assert_eq!(fc.arguments, "{\"location\":\"SF\"}");
828    }
829
830    #[test]
831    fn function_call_rejects_integer_arguments() {
832        let result = serde_json::from_value::<FunctionCall>(serde_json::json!({
833            "name": "f",
834            "arguments": 42
835        }));
836        assert!(result.is_err());
837    }
838
839    #[test]
840    fn function_call_rejects_boolean_arguments() {
841        let result = serde_json::from_value::<FunctionCall>(serde_json::json!({
842            "name": "f",
843            "arguments": true
844        }));
845        assert!(result.is_err());
846    }
847
848    #[test]
849    fn function_call_rejects_null_arguments() {
850        let result = serde_json::from_value::<FunctionCall>(serde_json::json!({
851            "name": "f",
852            "arguments": null
853        }));
854        assert!(result.is_err());
855    }
856
857    #[test]
858    fn function_call_rejects_array_arguments() {
859        let result = serde_json::from_value::<FunctionCall>(serde_json::json!({
860            "name": "f",
861            "arguments": [1, 2, 3]
862        }));
863        assert!(result.is_err());
864    }
865
866    #[test]
867    fn function_call_stream_null_arguments_produces_none() {
868        let fcs: FunctionCallStream = serde_json::from_value(serde_json::json!({
869            "name": "f",
870            "arguments": null
871        }))
872        .unwrap();
873        assert_eq!(fcs.arguments, None);
874    }
875
876    #[test]
877    fn function_call_stream_rejects_integer_arguments() {
878        let result = serde_json::from_value::<FunctionCallStream>(serde_json::json!({
879            "name": "f",
880            "arguments": 42
881        }));
882        assert!(result.is_err());
883    }
884
885    #[test]
886    fn function_call_stream_rejects_boolean_arguments() {
887        let result = serde_json::from_value::<FunctionCallStream>(serde_json::json!({
888            "name": "f",
889            "arguments": true
890        }));
891        assert!(result.is_err());
892    }
893
894    #[test]
895    fn function_call_stream_accepts_dict_arguments() {
896        let fcs: FunctionCallStream = serde_json::from_value(serde_json::json!({
897            "name": "get_weather",
898            "arguments": {"location": "SF"}
899        }))
900        .unwrap();
901        assert_eq!(fcs.arguments.as_deref(), Some("{\"location\":\"SF\"}"));
902    }
903
904    #[test]
905    fn function_call_stream_accepts_null_arguments() {
906        let fcs: FunctionCallStream = serde_json::from_value(serde_json::json!({
907            "name": "get_weather"
908        }))
909        .unwrap();
910        assert_eq!(fcs.arguments, None);
911    }
912
913    #[test]
914    fn tool_call_with_dict_arguments_roundtrip() {
915        let tc: ChatCompletionMessageToolCall = serde_json::from_value(serde_json::json!({
916            "id": "call_abc",
917            "type": "function",
918            "function": {
919                "name": "search",
920                "arguments": {"query": "hello", "limit": 10}
921            }
922        }))
923        .unwrap();
924        // Compare as parsed JSON values since key order is non-deterministic
925        let parsed: serde_json::Value = serde_json::from_str(&tc.function.arguments).unwrap();
926        assert_eq!(parsed, serde_json::json!({"query": "hello", "limit": 10}));
927        // Re-serialisation produces a string, not an object
928        let json = serde_json::to_value(&tc).unwrap();
929        assert!(json["function"]["arguments"].is_string());
930    }
931
932    #[test]
933    fn stream_delta_function_call_accepts_dict_arguments() {
934        let delta: ChatCompletionStreamResponseDeltaFunctionCall =
935            serde_json::from_value(serde_json::json!({
936                "name": "get_weather",
937                "arguments": {"location": "SF"}
938            }))
939            .unwrap();
940        assert_eq!(delta.arguments.as_deref(), Some("{\"location\":\"SF\"}"));
941    }
942}
dynamo_protocols/types/chat.rs

dynamo_protocols/types/
chat.rs
