openai_interface/chat/create/

response.rs

//! This module provides structures for streaming and non-streaming
//! chat completion responses.

pub mod streaming {
    //! Streaming chat completion response.

    use serde::Deserialize;

    use crate::chat::ServiceTier;

    #[derive(Debug, Deserialize, Clone)]
    pub struct ChatCompletionChunk {
        /// A unique identifier for the chat completion.
        pub id: String,
        /// A list of chat completion choices. Can be more than one
        /// if `n` is greater than 1. Can also be empty for the last chunk if you set
        /// `stream_options: {"include_usage": true}`.
        pub choices: Vec<CompletionChunkChoice>,
        /// The Unix timestamp (in seconds) of when the chat completion was created.
        /// Each chunk has the same timestamp.
        pub created: u64,
        /// The model used for the chat completion.
        pub model: String,
        /// The object type, which is always `chat.completion.chunk`
        pub object: ChatCompletionChunkObject,
        /// Specifies the processing type used for serving the request.
        ///
        /// When the `service_tier` parameter is set, the response body will include the
        /// `service_tier` value based on the processing mode actually used to serve the
        /// request. This response value may be different from the value set in the
        /// request parameter.
        pub service_tier: Option<ServiceTier>,
        /// This fingerprint represents the backend configuration that the model runs with.
        /// Can be used in conjunction with the `seed` request parameter to understand when
        /// backend changes have been made that might impact determinism.
        pub system_fingerprint: Option<String>,
        /// An optional field that will only be present when you set
        /// `stream_options: {"include_usage": true}` in your request. When present, it
        /// contains a null value **except for the last chunk** which contains the token
        /// usage statistics for the entire request.
        ///
        /// **NOTE:** If the stream is interrupted or cancelled, you may not receive the
        /// final usage chunk which contains the total token usage for the request.
        pub usage: Option<CompletionUsage>,
        /// Moderation results for the request input and generated output.
        ///
        /// Present on the moderation chunk when moderated completions are
        /// requested via the `moderation` request parameter.
        pub moderation: Option<crate::chat::ChatModeration>,
    }

    #[derive(Debug, Deserialize, Clone)]
    pub enum ChatCompletionChunkObject {
        #[serde(rename = "chat.completion.chunk")]
        ChatCompletionChunk,
    }

    #[derive(Debug, Deserialize, Clone)]
    pub struct CompletionChunkChoice {
        /// A chat completion delta generated by streamed model responses.
        pub delta: ChoiceDelta,
        /// The index of the choice in the list of choices.
        pub index: u32,
        /// Log probability information for the choice.
        pub logprobs: Option<ChoiceLogprobs>,
        /// The reason the model stopped generating tokens.
        ///
        /// This will be `stop` if the model hit a natural stop point or a provided stop
        /// sequence, `length` if the maximum number of tokens specified in the request was
        /// reached, `content_filter` if content was omitted due to a flag from our content
        /// filters, `tool_calls` if the model called a tool, or `function_call`
        /// (deprecated) if the model called a function.
        pub finish_reason: Option<FinishReason>,
    }

    #[derive(Debug, Deserialize, Clone)]
    #[serde(rename_all = "snake_case")]
    pub enum FinishReason {
        /// The maximum number of tokens specified in the request was reached.
        Length,
        /// The model hit a natural stop point or a provided stop sequence.
        Stop,
        /// Content was omitted due to a flag from our content filters.
        ContentFilter,
        /// The model called a function (deprecated).
        FunctionCall,
        /// The model called a tool.
        ToolCalls,
        /// DeepSeek: the request is interrupted due to insufficient resource
        /// of the inference system.
        #[cfg(feature = "deepseek")]
        InsufficientSystemResource,
    }

    #[derive(Debug, Deserialize, Clone)]
    pub struct ChoiceDelta {
        /// The contents of the chunk message.
        pub content: Option<String>,
        /// DeepSeek: the reasoning contents of the chunk message. Only
        /// present for thinking models.
        #[cfg(feature = "deepseek")]
        pub reasoning_content: Option<String>,
        /// Deprecated and replaced by `tool_calls`.
        ///
        /// The name and arguments of a function that should be called, as generated by the
        /// model.
        pub function_call: Option<ChoiceDeltaFunctionCall>,
        /// The refusal message generated by the model.
        pub refusal: Option<String>,
        /// The role of the author of this message.
        pub role: Option<CompletionRole>,
        /// A list of tool calls generated by the model, such as function calls.
        pub tool_calls: Option<Vec<ChoiceDeltaToolCall>>,
        /// Annotations for the chunk message, such as URL citations emitted
        /// by search deployments.
        ///
        /// Not part of the official OpenAI chunk schema, but Azure OpenAI
        /// ("on your data" / search deployments) does stream `annotations`
        /// inside `delta`, and dropping them silently loses the citations.
        /// Gated behind the `azure` cargo feature, following the crate's
        /// convention for provider-specific fields.
        #[cfg(feature = "azure")]
        pub annotations: Option<Vec<crate::chat::Annotation>>,
        /// Data about a streamed audio response from the model.
        ///
        /// Not part of the official OpenAI chunk schema, but audio-capable
        /// Azure OpenAI deployments stream `audio` inside `delta`. Gated
        /// behind the `azure` cargo feature, following the crate's
        /// convention for provider-specific fields.
        #[cfg(feature = "azure")]
        pub audio: Option<crate::chat::ChatCompletionAudio>,
    }

    #[derive(Debug, Deserialize, Clone)]
    pub struct ChoiceDeltaToolCallFunction {
        /// The arguments to call the function with, as generated by the model in JSON
        /// format. Note that the model does not always generate valid JSON, and may
        /// hallucinate parameters not defined by your function schema. Validate the
        /// arguments in your code before calling your function.
        pub arguments: Option<String>,
        /// The name of the function to call.
        pub name: Option<String>,
    }

    #[derive(Debug, Deserialize, Clone)]
    pub struct ChoiceDeltaFunctionCall {
        /// The arguments to call the function with, as generated by the model in JSON
        /// format. Note that the model does not always generate valid JSON, and may
        /// hallucinate parameters not defined by your function schema. Validate the
        /// arguments in your code before calling your function.
        pub arguments: Option<String>,
        /// The name of the function to call.
        pub name: Option<String>,
    }

    #[derive(Debug, Deserialize, Clone)]
    pub struct ChoiceDeltaToolCall {
        /// The index of the tool call in the list of tool calls.
        pub index: usize,
        /// The ID of the tool call.
        pub id: Option<String>,
        /// The function that the model called.
        pub function: Option<ChoiceDeltaToolCallFunction>,
        /// The type of the tool. Currently, only `function` is supported.
        #[serde(rename = "type")]
        pub type_: Option<ChoiceDeltaToolCallType>,
    }

    #[derive(Debug, Deserialize, Clone)]
    #[serde(rename_all = "snake_case")]
    pub enum ChoiceDeltaToolCallType {
        Function,
    }

    #[derive(Debug, Deserialize, Clone)]
    #[serde(rename_all = "snake_case")]
    pub enum CompletionRole {
        Assistant,
        Developer,
        System,
        Tool,
        User,
    }

    /// Log probability information for a choice.
    #[derive(Debug, Deserialize, Clone)]
    pub struct ChoiceLogprobs {
        /// A list of message content tokens with log probability information.
        pub content: Option<Vec<LogprobeContent>>,
        /// DeepSeek: a list of reasoning content tokens with log probability
        /// information. Only present for thinking models.
        #[cfg(feature = "deepseek")]
        pub reasoning_content: Option<Vec<LogprobeContent>>,
        /// A list of message refusal tokens with log probability information.
        pub refusal: Option<Vec<LogprobeContent>>,
    }

    /// A list of message content tokens with log probability information.
    #[derive(Debug, Deserialize, Clone)]
    pub struct LogprobeContent {
        pub token: String,
        pub logprob: f32,
        pub bytes: Option<Vec<u8>>,
        pub top_logprobs: Vec<TopLogprob>,
    }

    /// List of the most likely tokens and their log probability, at this
    /// token position. In rare cases, there may be fewer than the number of requested top_logprobs returned.
    #[derive(Debug, Deserialize, Clone)]
    pub struct TopLogprob {
        pub token: String,
        pub logprob: f32,
        pub bytes: Option<Vec<u8>>,
    }

    #[derive(Debug, Deserialize, Clone)]
    pub struct CompletionUsage {
        /// Number of tokens in the generated completion.
        pub completion_tokens: usize,
        /// Number of tokens in the prompt.
        pub prompt_tokens: usize,

        /// DeepSeek: number of tokens in the prompt that hits the context cache.
        #[cfg(feature = "deepseek")]
        pub prompt_cache_hit_tokens: Option<usize>,
        /// DeepSeek: number of tokens in the prompt that misses the context cache.
        #[cfg(feature = "deepseek")]
        pub prompt_cache_miss_tokens: Option<usize>,

        /// Total number of tokens used in the request (prompt + completion).
        pub total_tokens: usize,
        /// Breakdown of tokens used in a completion.
        pub completion_tokens_details: Option<CompletionTokensDetails>,
        /// Breakdown of tokens used in the prompt.
        pub prompt_tokens_details: Option<PromptTokensDetails>,
    }

    #[derive(Debug, Deserialize, Clone)]
    pub struct CompletionTokensDetails {
        /// When using Predicted Outputs, the number of tokens in the prediction that
        /// appeared in the completion.
        pub accepted_prediction_tokens: Option<usize>,
        /// Audio input tokens generated by the model.
        pub audio_tokens: Option<usize>,
        /// Tokens generated by the model for reasoning.
        pub reasoning_tokens: Option<usize>,
        /// When using Predicted Outputs, the number of tokens in the prediction that did
        /// not appear in the completion. However, like reasoning tokens, these tokens are
        /// still counted in the total completion tokens for purposes of billing, output,
        /// and context window limits.
        pub rejected_prediction_tokens: Option<usize>,
    }

    #[derive(Debug, Deserialize, Clone)]
    pub struct PromptTokensDetails {
        /// Audio input tokens present in the prompt.
        pub audio_tokens: Option<usize>,
        /// Cached tokens present in the prompt.
        pub cached_tokens: Option<usize>,
    }

    crate::impl_from_str!(ChatCompletionChunk);

    #[cfg(test)]
    mod test {
        use std::str::FromStr;

        use super::*;

        #[test]
        fn streaming_example_deepseek() {
            let streams = vec![
                r#"{"id": "1f633d8bfc032625086f14113c411638", "choices": [{"index": 0, "delta": {"content": "", "role": "assistant"}, "finish_reason": null, "logprobs": null}], "created": 1718345013, "model": "deepseek-chat", "system_fingerprint": "fp_a49d71b8a1", "object": "chat.completion.chunk", "usage": null}"#,
                r#"{"choices": [{"delta": {"content": "Hello", "role": "assistant"}, "finish_reason": null, "index": 0, "logprobs": null}], "created": 1718345013, "id": "1f633d8bfc032625086f14113c411638", "model": "deepseek-chat", "object": "chat.completion.chunk", "system_fingerprint": "fp_a49d71b8a1"}"#,
                r#"{"choices": [{"delta": {"content": "!", "role": "assistant"}, "finish_reason": null, "index": 0, "logprobs": null}], "created": 1718345013, "id": "1f633d8bfc032625086f14113c411638", "model": "deepseek-chat", "object": "chat.completion.chunk", "system_fingerprint": "fp_a49d71b8a1"}"#,
                r#"{"choices": [{"delta": {"content": " How", "role": "assistant"}, "finish_reason": null, "index": 0, "logprobs": null}], "created": 1718345013, "id": "1f633d8bfc032625086f14113c411638", "model": "deepseek-chat", "object": "chat.completion.chunk", "system_fingerprint": "fp_a49d71b8a1"}"#,
                r#"{"choices": [{"delta": {"content": " can", "role": "assistant"}, "finish_reason": null, "index": 0, "logprobs": null}], "created": 1718345013, "id": "1f633d8bfc032625086f14113c411638", "model": "deepseek-chat", "object": "chat.completion.chunk", "system_fingerprint": "fp_a49d71b8a1"}"#,
                r#"{"choices": [{"delta": {"content": " I", "role": "assistant"}, "finish_reason": null, "index": 0, "logprobs": null}], "created": 1718345013, "id": "1f633d8bfc032625086f14113c411638", "model": "deepseek-chat", "object": "chat.completion.chunk", "system_fingerprint": "fp_a49d71b8a1"}"#,
                r#"{"choices": [{"delta": {"content": " assist", "role": "assistant"}, "finish_reason": null, "index": 0, "logprobs": null}], "created": 1718345013, "id": "1f633d8bfc032625086f14113c411638", "model": "deepseek-chat", "object": "chat.completion.chunk", "system_fingerprint": "fp_a49d71b8a1"}"#,
                r#"{"choices": [{"delta": {"content": " you", "role": "assistant"}, "finish_reason": null, "index": 0, "logprobs": null}], "created": 1718345013, "id": "1f633d8bfc032625086f14113c411638", "model": "deepseek-chat", "object": "chat.completion.chunk", "system_fingerprint": "fp_a49d71b8a1"}"#,
                r#"{"choices": [{"delta": {"content": " today", "role": "assistant"}, "finish_reason": null, "index": 0, "logprobs": null}], "created": 1718345013, "id": "1f633d8bfc032625086f14113c411638", "model": "deepseek-chat", "object": "chat.completion.chunk", "system_fingerprint": "fp_a49d71b8a1"}"#,
                r#"{"choices": [{"delta": {"content": "?", "role": "assistant"}, "finish_reason": null, "index": 0, "logprobs": null}], "created": 1718345013, "id": "1f633d8bfc032625086f14113c411638", "model": "deepseek-chat", "object": "chat.completion.chunk", "system_fingerprint": "fp_a49d71b8a1"}"#,
                r#"{"choices": [{"delta": {"content": "", "role": null}, "finish_reason": "stop", "index": 0, "logprobs": null}], "created": 1718345013, "id": "1f633d8bfc032625086f14113c411638", "model": "deepseek-chat", "object": "chat.completion.chunk", "system_fingerprint": "fp_a49d71b8a1", "usage": {"completion_tokens": 9, "prompt_tokens": 17, "total_tokens": 26}}"#,
            ];

            for stream in streams {
                let parsed = ChatCompletionChunk::from_str(stream);
                match parsed {
                    Ok(completion) => {
                        println!("Deserialized: {:#?}", completion);
                    }
                    Err(e) => {
                        panic!("Failed to deserialize {}: {}", stream, e);
                    }
                }
            }
        }

        #[test]
        fn streaming_example_qwen() {
            let streams = vec![
                r#"{"id":"chatcmpl-e30f5ae7-3063-93c4-90fe-beb5f900bd57","choices":[{"delta":{"content":"","function_call":null,"refusal":null,"role":"assistant","tool_calls":null},"finish_reason":null,"index":0,"logprobs":null}],"created":1735113344,"model":"qwen-plus","object":"chat.completion.chunk","service_tier":null,"system_fingerprint":null,"usage":null}"#,
                r#"{"id":"chatcmpl-e30f5ae7-3063-93c4-90fe-beb5f900bd57","choices":[{"delta":{"content":"我是","function_call":null,"refusal":null,"role":null,"tool_calls":null},"finish_reason":null,"index":0,"logprobs":null}],"created":1735113344,"model":"qwen-plus","object":"chat.completion.chunk","service_tier":null,"system_fingerprint":null,"usage":null}"#,
                r#"{"id":"chatcmpl-e30f5ae7-3063-93c4-90fe-beb5f900bd57","choices":[{"delta":{"content":"来自","function_call":null,"refusal":null,"role":null,"tool_calls":null},"finish_reason":null,"index":0,"logprobs":null}],"created":1735113344,"model":"qwen-plus","object":"chat.completion.chunk","service_tier":null,"system_fingerprint":null,"usage":null}"#,
                r#"{"id":"chatcmpl-e30f5ae7-3063-93c4-90fe-beb5f900bd57","choices":[{"delta":{"content":"阿里","function_call":null,"refusal":null,"role":null,"tool_calls":null},"finish_reason":null,"index":0,"logprobs":null}],"created":1735113344,"model":"qwen-plus","object":"chat.completion.chunk","service_tier":null,"system_fingerprint":null,"usage":null}"#,
                r#"{"id":"chatcmpl-e30f5ae7-3063-93c4-90fe-beb5f900bd57","choices":[{"delta":{"content":"云的超大规模","function_call":null,"refusal":null,"role":null,"tool_calls":null},"finish_reason":null,"index":0,"logprobs":null}],"created":1735113344,"model":"qwen-plus","object":"chat.completion.chunk","service_tier":null,"system_fingerprint":null,"usage":null}"#,
                r#"{"id":"chatcmpl-e30f5ae7-3063-93c4-90fe-beb5f900bd57","choices":[{"delta":{"content":"语言","function_call":null,"refusal":null,"role":null,"tool_calls":null},"finish_reason":null,"index":0,"logprobs":null}],"created":1735113344,"model":"qwen-plus","object":"chat.completion.chunk","service_tier":null,"system_fingerprint":null,"usage":null}"#,
                r#"{"id":"chatcmpl-e30f5ae7-3063-93c4-90fe-beb5f900bd57","choices":[{"delta":{"content":"模型","function_call":null,"refusal":null,"role":null,"tool_calls":null},"finish_reason":null,"index":0,"logprobs":null}],"created":1735113344,"model":"qwen-plus","object":"chat.completion.chunk","service_tier":null,"system_fingerprint":null,"usage":null}"#,
                r#"{"id":"chatcmpl-e30f5ae7-3063-93c4-90fe-beb5f900bd57","choices":[{"delta":{"content":"。","function_call":null,"refusal":null,"role":null,"tool_calls":null},"finish_reason":null,"index":0,"logprobs":null}],"created":1735113344,"model":"qwen-plus","object":"chat.completion.chunk","service_tier":null,"system_fingerprint":null,"usage":null}"#,
                r#"{"id":"chatcmpl-e30f5ae7-3063-93c4-90fe-beb5f900bd57","choices":[{"delta":{"content":"问。","function_call":null,"refusal":null,"role":null,"tool_calls":null},"finish_reason":null,"index":0,"logprobs":null}],"created":1735113344,"model":"qwen-plus","object":"chat.completion.chunk","service_tier":null,"system_fingerprint":null,"usage":null}"#,
                r#"{"id":"chatcmpl-e30f5ae7-3063-93c4-90fe-beb5f900bd57","choices":[{"delta":{"content":"","function_call":null,"refusal":null,"role":null,"tool_calls":null},"finish_reason":"stop","index":0,"logprobs":null}],"created":1735113344,"model":"qwen-plus","object":"chat.completion.chunk","service_tier":null,"system_fingerprint":null,"usage":null}"#,
                r#"{"id":"chatcmpl-e30f5ae7-3063-93c4-90fe-beb5f900bd57","choices":[],"created":1735113344,"model":"qwen-plus","object":"chat.completion.chunk","service_tier":null,"system_fingerprint":null,"usage":{"completion_tokens":17,"prompt_tokens":22,"total_tokens":39,"completion_tokens_details":null,"prompt_tokens_details":{"audio_tokens":null,"cached_tokens":0}}}"#,
            ];

            for stream in streams {
                let parsed = ChatCompletionChunk::from_str(stream);
                match parsed {
                    Ok(completion) => {
                        println!("Deserialized: {:#?}", completion);
                    }
                    Err(e) => {
                        panic!("Failed to deserialize {}: {}", stream, e);
                    }
                }
            }
        }

        /// Azure OpenAI streams `annotations` (URL citations from "on your
        /// data" deployments) and `audio` inside `delta`, outside the
        /// official chunk schema; both are gated behind the `azure` feature
        /// and must deserialize instead of being dropped.
        #[cfg(feature = "azure")]
        #[test]
        fn streaming_example_azure_annotations_and_audio() {
            let chunk = ChatCompletionChunk::from_str(
                r#"{"id":"chatcmpl-abc","choices":[{"delta":{"content":"According to the doc","annotations":[{"type":"url_citation","url_citation":{"start_index":0,"end_index":20,"title":"Azure Docs","url":"https://learn.microsoft.com/azure"}}],"audio":{"id":"audio_abc","data":"SGVsbG8=","expires_at":1735113344,"transcript":"Hello"}},"finish_reason":null,"index":0,"logprobs":null}],"created":1735113344,"model":"gpt-4o","object":"chat.completion.chunk","system_fingerprint":"fp_abc"}"#,
            )
            .expect("chunk with annotations and audio must deserialize");

            let delta = &chunk.choices[0].delta;
            let annotations = delta.annotations.as_ref().expect("annotations");
            assert_eq!(annotations.len(), 1);
            assert_eq!(annotations[0].url_citation.title, "Azure Docs");
            assert_eq!(
                annotations[0].url_citation.url,
                "https://learn.microsoft.com/azure"
            );

            let audio = delta.audio.as_ref().expect("audio");
            assert_eq!(audio.id, "audio_abc");
            assert_eq!(audio.transcript, "Hello");
        }
    }
}

pub mod no_streaming {
    //! Non-streaming chat completion response.

    /// Alias for `crate::chat::ChatCompletion`, which is shared
    /// by many other modules. This alias is for compatibility.
    pub type ChatCompletion = crate::chat::ChatCompletion;
}