openai_interface/chat/

mod.rs

//! # Chat Completions API Module
//!
//! This module provides components shared by many submodules.

use std::str::FromStr;

use serde::{Deserialize, Serialize};

use crate::errors::OapiError;

pub mod create;
pub mod delete;
pub mod retrieve;
pub mod update;

crate::wire_string_enum! {
    /// The service tier used for processing the request.
    ///
    /// This enum represents the different service tiers that can be specified when
    /// making a request to the API. Each tier corresponds to different performance
    /// characteristics and pricing models.
    pub enum ServiceTier {
        /// Automatically select the service tier based on project settings.
        Auto => "auto",
        /// Use the default service tier with standard pricing and performance.
        Default => "default",
        /// Use the flex service tier for flexible processing requirements.
        Flex => "flex",
        /// Use the scale service tier for scalable processing needs.
        Scale => "scale",
        /// Use the priority service tier for high-priority requests.
        Priority => "priority",
        /// Fast mode. Request-level opt-in for
        /// [Fast mode](https://platform.openai.com/docs/guides/fast-mode); the
        /// response reports the actual tier as `priority`.
        Fast => "fast",
    }
}

/// Moderation results for the request input and generated output.
///
/// Present on the response when moderated completions are requested via the
/// `moderation` request parameter.
#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct ChatModeration {
    /// Moderation for the request input.
    pub input: ModerationSide,
    /// Moderation for the generated output.
    pub output: ModerationSide,
}

/// The moderation outcome for one side (input or output) of the completion:
/// either successful results or an error produced while attempting
/// moderation.
#[derive(Debug, Deserialize, Serialize, Clone)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ModerationSide {
    /// Successful moderation results. Always `moderation_results`.
    ModerationResults {
        /// The moderation model used to generate the results.
        model: String,
        /// A list of moderation results.
        results: Vec<ModerationSideResult>,
    },
    /// An error produced while attempting moderation. Always `error`.
    Error {
        /// The error code.
        code: String,
        /// The error message.
        message: String,
    },
}

/// A moderation result produced for the response input or generated output.
#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct ModerationSideResult {
    /// A dictionary of moderation categories to booleans, `true` if the
    /// input is flagged under this category.
    pub categories: std::collections::HashMap<String, bool>,
    /// Which modalities of input are reflected by the score for each
    /// category, e.g. `["text"]` or `["text", "image"]`.
    pub category_applied_input_types: std::collections::HashMap<String, Vec<String>>,
    /// A dictionary of moderation categories to scores.
    pub category_scores: std::collections::HashMap<String, f64>,
    /// A boolean indicating whether the content was flagged by any category.
    pub flagged: bool,
    /// The moderation model that produced this result.
    pub model: String,
    /// The object type, which is always `moderation_result`.
    #[serde(rename = "type")]
    pub type_: ModerationResultType,
}

crate::wire_string_enum! {
    /// The object type of a moderation result. Always `moderation_result`.
    pub enum ModerationResultType {
        ModerationResult => "moderation_result",
    }
}

#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct ChatCompletion {
    /// A unique identifier for the chat completion.
    pub id: String,
    /// A list of chat completion choices. Can be more than one
    /// if `n` is greater than 1.
    pub choices: Vec<Choice>,
    /// The Unix timestamp (in seconds) of when the chat completion was created.
    pub created: u64,
    /// The model used for the chat completion.
    pub model: String,
    /// Specifies the processing type used for serving the request.
    ///
    /// - If set to 'auto', then the request will be processed with the service tier
    ///   configured in the Project settings. Unless otherwise configured, the Project
    ///   will use 'default'.
    /// - If set to 'default', then the request will be processed with the standard
    ///   pricing and performance for the selected model.
    /// - If set to '[flex](https://platform.openai.com/docs/guides/flex-processing)' or
    ///   '[priority](https://openai.com/api-priority-processing/)', then the request
    ///   will be processed with the corresponding service tier.
    /// - When not set, the default behavior is 'auto'.
    ///
    /// When the `service_tier` parameter is set, the response body will include the
    /// `service_tier` value based on the processing mode actually used to serve the
    /// request. This response value may be different from the value set in the
    /// parameter.
    pub service_tier: Option<ServiceTier>,
    /// The system fingerprint used for the chat completion.
    /// Can be used in conjunction with the `seed` request parameter to understand when
    /// backend changes have been made that might impact determinism.
    pub system_fingerprint: Option<String>,
    /// The object type, which is always `chat.completion`.
    ///
    /// `Some` only when the backend sends a recognized value; some
    /// non-OpenAI gateways omit or repurpose the field.
    pub object: Option<ChatCompletionObject>,
    /// Usage statistics for the completion request.
    pub usage: Option<CompletionUsage>,
    /// Moderation results for the request input and generated output.
    ///
    /// Present when moderated completions are requested via the `moderation`
    /// request parameter.
    pub moderation: Option<ChatModeration>,

    /// vLLM: log probabilities of the prompt tokens, one entry per prompt
    /// position, or `null` for positions the server did not report. Each
    /// entry maps a token ID to its [`crate::vllm::Logprob`].
    ///
    /// Requested with `vllm_sampling.prompt_logprobs`; OpenAI has no
    /// equivalent.
    #[cfg(feature = "vllm")]
    pub prompt_logprobs: Option<Vec<Option<std::collections::HashMap<u32, crate::vllm::Logprob>>>>,
    /// vLLM: the prompt's token IDs after chat-template rendering.
    #[cfg(feature = "vllm")]
    pub prompt_token_ids: Option<Vec<u32>>,
    /// vLLM: the fully rendered prompt text.
    ///
    /// Only set when the request set `vllm_chat.return_prompt_text`.
    #[cfg(feature = "vllm")]
    pub prompt_text: Option<String>,
    /// vLLM: KV-transfer parameters for disaggregated prefill, echoing and
    /// extending the request's `kv_transfer_params`.
    #[cfg(feature = "vllm")]
    pub kv_transfer_params: Option<serde_json::Map<String, serde_json::Value>>,
    /// vLLM: encoder-cache transfer parameters, echoing and extending the
    /// request's `ec_transfer_params`.
    #[cfg(feature = "vllm")]
    pub ec_transfer_params: Option<serde_json::Map<String, serde_json::Value>>,
}

crate::wire_string_enum! {
    /// The object type, which is always `chat.completion`.
    pub enum ChatCompletionObject {
        /// The object type is always `chat.completion`.
        ChatCompletion => "chat.completion",
    }
}

#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct Choice {
    /// The reason the model stopped generating tokens.
    ///
    /// This will be `stop` if the model hit a natural stop point or a provided stop
    /// sequence, `length` if the maximum number of tokens specified in the request was
    /// reached, `content_filter` if content was omitted due to a flag from our content
    /// filters, `tool_calls` if the model called a tool, or `function_call`
    /// (deprecated) if the model called a function.
    pub finish_reason: FinishReason,
    /// The index of the choice in the list of choices.
    pub index: u32,
    /// Log probability information for the choice.
    pub logprobs: Option<ChoiceLogprobs>,
    /// A chat completion message generated by the model.
    pub message: ChatCompletionMessage,

    /// vLLM: which terminator ended generation — the matched stop string, or
    /// the matched token ID. Not part of the OpenAI schema; `finish_reason`
    /// alone only reports `stop` for both cases.
    #[cfg(feature = "vllm")]
    pub stop_reason: Option<crate::vllm::StopReason>,
    /// vLLM: the generated token IDs, for tracing tokens in agent scenarios.
    /// Only set when the request set `vllm_chat.return_token_ids`.
    #[cfg(feature = "vllm")]
    pub token_ids: Option<Vec<u32>>,
    /// vLLM: per-token expert routing decisions for mixture-of-experts
    /// models, as base64-encoded NumPy `.npy` bytes of shape
    /// `(num_tokens - 1, num_layers, num_experts_per_tok)`.
    ///
    /// Only set when the server runs with `--enable-return-routed-experts`.
    #[cfg(feature = "vllm")]
    pub routed_experts: Option<String>,
}

crate::wire_string_enum! {
    /// The reason the model stopped generating tokens.
    ///
    /// Values that are not part of the official API (some gateways emit
    /// their own, e.g. `eos`) are preserved as
    /// [`FinishReason::Unknown`] instead of failing deserialization.
    pub enum FinishReason {
        /// The maximum number of tokens specified in the request was reached.
        Length => "length",
        /// The model hit a natural stop point or a provided stop sequence.
        Stop => "stop",
        /// The model called a tool.
        ToolCalls => "tool_calls",
        /// The model called a function (deprecated).
        FunctionCall => "function_call",
        /// Content was omitted due to a flag from our content filters.
        ContentFilter => "content_filter",
        /// DeepSeek: the request is interrupted due to insufficient resource
        /// of the inference system.
        InsufficientSystemResource => "insufficient_system_resource",
    }
}

#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct ChatCompletionMessage {
    /// The role of the author of this message. This shall always
    /// be [`Role::Assistant`]
    pub role: Role,
    /// If the audio output modality is requested, this object contains data
    /// about the audio response from the model.
    /// [Learn more from OpenAI](https://platform.openai.com/docs/guides/audio).
    pub audio: Option<ChatCompletionAudio>,
    /// The contents of the message.
    pub content: Option<String>,
    /// For thinking models: the reasoning contents of the assistant
    /// message, before the final answer.
    #[cfg(feature = "reasoning")]
    pub reasoning_content: Option<String>,
    /// vLLM: the reasoning contents of the assistant message, under the key
    /// vLLM actually emits.
    ///
    /// vLLM accepts `reasoning_content` on the way in but serializes the
    /// chain of thought as `reasoning` on the way out — in its `ChatMessage`
    /// the two names are aliases of one field, and `reasoning` is the
    /// serialization name. Current vLLM does **not** emit
    /// `reasoning_content` in responses, so for a vLLM backend the field
    /// above stays `None` and this one carries the chain of thought. Map it
    /// back onto `reasoning_content` when feeding it into a follow-up
    /// request.
    #[cfg(feature = "vllm")]
    pub reasoning: Option<String>,
    /// The tool calls generated by the model, such as function calls.
    /// Tool calls deserialization is not supported yet.
    pub tool_calls: Option<Vec<ChatCompletionMessageToolCall>>,
    /// The refusal message generated by the model.
    pub refusal: Option<String>,
    /// Annotations for the message, when applicable, such as URL citations
    /// when the model uses a web search tool.
    pub annotations: Option<Vec<Annotation>>,
}

/// If the audio output modality is requested, this object contains data about
/// the audio response from the model.
/// [Learn more from OpenAI](https://platform.openai.com/docs/guides/audio).
#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct ChatCompletionAudio {
    /// Unique identifier for this audio response.
    pub id: String,
    /// Base64 encoded audio bytes generated by the model, in the format
    /// specified in the request.
    pub data: String,
    /// The Unix timestamp (in seconds) for when this audio response will no
    /// longer be accessible on the server for use in multi-turn conversations.
    pub expires_at: u64,
    /// Transcript of the audio generated by the model.
    pub transcript: String,
}

/// An annotation for a chat completion message.
#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct Annotation {
    /// The type of the annotation. Always `url_citation`.
    #[serde(rename = "type")]
    pub type_: AnnotationType,
    /// The URL citation.
    pub url_citation: UrlCitation,
}

crate::wire_string_enum! {
    /// The type of an annotation.
    pub enum AnnotationType {
        /// A URL citation when using web search.
        UrlCitation => "url_citation",
    }
}

/// A URL citation when the model uses a web search tool.
#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct UrlCitation {
    /// The index of the first character of the URL citation in the message.
    pub start_index: usize,
    /// The index of the last character of the URL citation in the message.
    pub end_index: usize,
    /// The title of the web resource.
    pub title: String,
    /// The URL of the web resource.
    pub url: String,
}

#[derive(Debug, Deserialize, Serialize, Clone)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ChatCompletionMessageToolCall {
    /// The type of the tool. Currently, only `function` is supported.
    /// The field { type = "function" } is added automatically.
    Function {
        /// The ID of the tool call.
        id: String,
        /// The function that the model called.
        function: MessageToolCallFunction,
    },
    /// The type of the tool. Always `custom`.
    /// The field { type = "custom" } is added automatically.
    Custom {
        /// The id of the tool call.
        id: String,
        /// The custom tool that the model called.
        custom: MessageToolCallCustom,
    },
}

#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct MessageToolCallCustom {
    /// The input for the custom tool call generated by the model.
    pub input: String,
    /// The name of the custom tool to call.
    pub name: String,
}

#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct MessageToolCallFunction {
    /// The arguments to call the function with, as generated by the model in JSON
    /// format. Note that the model does not always generate valid JSON, and may
    /// hallucinate parameters not defined by your function schema. Validate the
    /// arguments in your code before calling your function.
    pub arguments: String,
    /// The name of the function to call.
    pub name: String,
}

crate::wire_string_enum! {
    /// The role of the author of a message, as reported in responses and
    /// streamed deltas.
    ///
    /// Values that are not part of the official API are preserved as
    /// [`Role::Unknown`] instead of failing deserialization.
    pub enum Role {
        /// The author is the model.
        Assistant => "assistant",
        /// The author is a developer-defined persona.
        Developer => "developer",
        /// The author is the system prompt.
        System => "system",
        /// The author is a tool response.
        Tool => "tool",
        /// The author is the end user.
        User => "user",
    }
}

/// Legacy alias of [`Role`].
pub type ResponseRole = Role;

#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct ChoiceLogprobs {
    /// A list of message content tokens with log probability information.
    pub content: Option<Vec<TokenLogProb>>,
    /// A list of reasoning content tokens with log probability information.
    /// Only present for thinking models.
    #[cfg(feature = "reasoning")]
    pub reasoning_content: Option<Vec<TokenLogProb>>,
    /// A list of message refusal tokens with log probability information.
    pub refusal: Option<Vec<TokenLogProb>>,
}

#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct TokenLogProb {
    /// The token.
    pub token: String,
    /// The log probability of this token, if it is within the top 20 most likely
    /// tokens. Otherwise, the value `-9999.0` is used to signify that the token is very
    /// unlikely.
    pub logprob: f32,
    /// A list of integers representing the UTF-8 bytes representation of the token.
    ///
    /// Useful in instances where characters are represented by multiple tokens and
    /// their byte representations must be combined to generate the correct text
    /// representation. Can be `null` if there is no bytes representation for the token.
    pub bytes: Option<Vec<u8>>,
    /// List of the most likely tokens and their log probability, at this token
    /// position. In rare cases, there may be fewer than the number of requested
    /// `top_logprobs` returned.
    pub top_logprobs: Vec<TopLogprob>,
}

#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct TopLogprob {
    /// The token.
    pub token: String,
    /// A list of integers representing the UTF-8 bytes representation of the token.
    ///
    /// Useful in instances where characters are represented by multiple tokens and
    /// their byte representations must be combined to generate the correct text
    /// representation. Can be `null` if there is no bytes representation for the token.
    pub logprob: f32,
    /// List of the most likely tokens and their log probability, at this token
    /// position. In rare cases, there may be fewer than the number of requested
    /// `top_logprobs` returned.
    pub bytes: Option<Vec<u8>>,
}

#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
pub struct CompletionUsage {
    /// Number of tokens in the generated completion.
    pub completion_tokens: u64,
    /// Number of tokens in the prompt.
    pub prompt_tokens: u64,

    /// DeepSeek: number of tokens in the prompt that hits the context cache.
    #[cfg(feature = "deepseek")]
    pub prompt_cache_hit_tokens: Option<u64>,
    /// DeepSeek: number of tokens in the prompt that misses the context cache.
    #[cfg(feature = "deepseek")]
    pub prompt_cache_miss_tokens: Option<u64>,

    /// Total number of tokens used in the request (prompt + completion).
    pub total_tokens: u64,
    /// Breakdown of tokens used in a completion.
    pub completion_tokens_details: Option<CompletionTokensDetails>,
    /// Breakdown of tokens used in the prompt.
    pub prompt_tokens_details: Option<PromptTokensDetails>,
}

#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
pub struct CompletionTokensDetails {
    /// When using Predicted Outputs, the number of tokens in the prediction that
    /// appeared in the completion.
    pub accepted_prediction_tokens: Option<u64>,
    /// Audio input tokens generated by the model.
    pub audio_tokens: Option<u64>,
    /// Tokens generated by the model for reasoning.
    pub reasoning_tokens: Option<u64>,
    /// When using Predicted Outputs, the number of tokens in the prediction that did
    /// not appear in the completion. However, like reasoning tokens, these tokens are
    /// still counted in the total completion tokens for purposes of billing, output,
    /// and context window limits.
    pub rejected_prediction_tokens: Option<u64>,
    /// vLLM: how many of the completion tokens were produced by speculative
    /// decoding. Not part of the OpenAI schema.
    #[cfg(feature = "vllm")]
    pub num_speculative_tokens: Option<u64>,
}

#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
pub struct PromptTokensDetails {
    /// Audio input tokens present in the prompt.
    pub audio_tokens: Option<u64>,
    /// Cached tokens present in the prompt.
    pub cached_tokens: Option<u64>,
}

impl FromStr for ChatCompletion {
    type Err = crate::errors::OapiError;

    fn from_str(content: &str) -> Result<Self, Self::Err> {
        let parse_result: Result<ChatCompletion, _> = serde_json::from_str(content)
            .map_err(|e| OapiError::DeserializationError(e.to_string()));
        parse_result
    }
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn service_tier_parses_official_values() {
        for (raw, is_fast) in [
            (r#""auto""#, false),
            (r#""default""#, false),
            (r#""flex""#, false),
            (r#""scale""#, false),
            (r#""priority""#, false),
            (r#""fast""#, true),
        ] {
            let tier: ServiceTier =
                serde_json::from_str(raw).unwrap_or_else(|e| panic!("failed to parse {raw}: {e}"));
            assert_eq!(matches!(tier, ServiceTier::Fast), is_fast, "raw: {raw}");
        }
    }

    #[test]
    fn no_streaming_example_deepseek() {
        let json = r#"{
          "id": "30f6413a-a827-4cf3-9898-f13a8634b798",
          "object": "chat.completion",
          "created": 1757944111,
          "model": "deepseek-chat",
          "choices": [
            {
              "index": 0,
              "message": {
                "role": "assistant",
                "content": "Hello! How can I help you today? 😊"
              },
              "logprobs": null,
              "finish_reason": "stop"
            }
          ],
          "usage": {
            "prompt_tokens": 10,
            "completion_tokens": 11,
            "total_tokens": 21,
            "prompt_tokens_details": {
              "cached_tokens": 0
            },
            "prompt_cache_hit_tokens": 0,
            "prompt_cache_miss_tokens": 10
          },
          "system_fingerprint": "fp_08f168e49b_prod0820_fp8_kvcache"
        }"#;

        let parsed = ChatCompletion::from_str(json);
        match parsed {
            Ok(_) => {}
            Err(e) => {
                panic!("Failed to deserialize: {}", e);
            }
        }
    }

    #[test]
    fn no_streaming_example_qwen() {
        let json = r#"{
            "choices": [
                {
                    "message": {
                        "role": "assistant",
                        "content": "我是阿里云开发的一款超大规模语言模型，我叫通义千问。"
                    },
                    "finish_reason": "stop",
                    "index": 0,
                    "logprobs": null
                }
            ],
            "object": "chat.completion",
            "usage": {
                "prompt_tokens": 3019,
                "completion_tokens": 104,
                "total_tokens": 3123,
                "prompt_tokens_details": {
                    "cached_tokens": 2048
                }
            },
            "created": 1735120033,
            "system_fingerprint": null,
            "model": "qwen-plus",
            "id": "chatcmpl-6ada9ed2-7f33-9de2-8bb0-78bd4035025a"
        }"#;

        let parsed = ChatCompletion::from_str(json);
        match parsed {
            Ok(_) => {}
            Err(e) => {
                panic!("Failed to deserialize: {}", e);
            }
        }
    }
}