openai_interface/audio/

transcriptions.rs

//! Transcribes audio into the input language.
//!
//! Endpoint: `POST /audio/transcriptions` (multipart/form-data request).
//!
//! Response shapes depend on `response_format`:
//!
//! - `json` and `verbose_json` deserialize into the typed
//!   [`TranscriptionResponse`] via `get_response`.
//! - `text`, `srt`, and `vtt` return plain text; use
//!   `get_response_string` for those.
//!
//! Streaming transcriptions (`stream`) is not supported yet.
//!
//! > ![warn] This module is untested!
//! > No OpenAI-compatible provider accessible to this project implements
//! > this endpoint, and no OpenAI API key was available for testing. If you
//! > encounter any issues, please report them on the repository.

use std::path::PathBuf;

use serde::{Deserialize, Serialize};
use url::Url;

use crate::{
    audio::AudioResponseFormat,
    errors::OapiError,
    rest::post::{Post, PostNoStream},
};

/// Transcribes audio into the input language.
///
/// The `keywords`, `languages`, `known_speaker_names`, and
/// `known_speaker_references` parameters are not covered by this type yet.
#[derive(Debug, Serialize, Default, Clone)]
pub struct TranscriptionRequest {
    /// The audio file (as a path) to transcribe, in one of these formats:
    /// flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm. The request must
    /// include enough format metadata for the file to be identified; an
    /// extension-bearing filename satisfies this.
    #[serde(skip_serializing)]
    pub file: PathBuf,
    /// ID of the model to use. The options are `gpt-transcribe`,
    /// `gpt-4o-transcribe`, `gpt-4o-mini-transcribe`, `whisper-1` (which is
    /// powered by the open source Whisper V2 model), and
    /// `gpt-4o-transcribe-diarize`.
    pub model: String,
    /// The language of the input audio. Supplying the input language in
    /// [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes)
    /// (e.g. `en`) format will improve accuracy and latency.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub language: Option<String>,
    /// An optional text to guide the model's style or continue a previous
    /// audio segment. The prompt should match the audio language.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub prompt: Option<String>,
    /// The format of the output, in one of these options: `json`, `text`,
    /// `srt`, `verbose_json`, or `vtt`. For `gpt-4o-transcribe` and
    /// `gpt-4o-mini-transcribe`, the only supported format is `json`.
    ///
    /// With `json` / `verbose_json`, use `get_response`; with `text` /
    /// `srt` / `vtt`, use `get_response_string`.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub response_format: Option<AudioResponseFormat>,
    /// The sampling temperature, between 0 and 1. Higher values like 0.8
    /// will make the output more random, while lower values like 0.2 will
    /// make it more focused and deterministic.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub temperature: Option<f32>,
    /// The timestamp granularities to populate for this transcription.
    /// `response_format` must be set to `verbose_json` to use timestamp
    /// granularities. Either or both of these options are supported: `word`,
    /// or `segment`.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub timestamp_granularities: Option<Vec<TimestampGranularity>>,
    /// Controls how the audio is cut into chunks.
    ///
    /// When set to `auto`, the server first normalizes loudness and then
    /// uses voice activity detection (VAD) to choose boundaries. A
    /// `server_vad` object can be provided to tweak VAD detection
    /// parameters manually. If unset, the audio is transcribed as a single
    /// block.
    #[serde(skip_serializing)]
    pub chunking_strategy: Option<ChunkingStrategy>,
    /// Additional information to include in the transcription response.
    /// `logprobs` will return the log probabilities of the tokens in the
    /// response.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub include: Option<Vec<Include>>,
}

/// The timestamp granularities to populate for a transcription.
#[derive(Debug, Serialize, Clone, Copy)]
#[serde(rename_all = "snake_case")]
pub enum TimestampGranularity {
    Word,
    Segment,
}

/// Additional information to include in the transcription response.
#[derive(Debug, Serialize, Clone, Copy)]
#[serde(rename_all = "snake_case")]
pub enum Include {
    Logprobs,
}

/// Controls how the audio is cut into chunks.
#[derive(Debug, Clone)]
pub enum ChunkingStrategy {
    /// The server normalizes loudness and then uses voice activity detection
    /// (VAD) to choose boundaries.
    Auto,
    /// Tweak VAD detection parameters manually.
    ServerVad(ServerVadConfig),
}

/// Manual VAD detection parameters, sent as the `server_vad` chunking
/// strategy.
#[derive(Debug, Clone, Default)]
pub struct ServerVadConfig {
    /// Amount of audio to include before the VAD detected speech (in
    /// milliseconds).
    pub prefix_padding_ms: Option<u32>,
    /// Duration of silence to detect speech stop (in milliseconds).
    pub silence_duration_ms: Option<u32>,
    /// Sensitivity threshold (0.0 to 1.0) for voice activity detection.
    pub threshold: Option<f32>,
}

/// The typed transcription response: `json` yields a plain
/// [`Transcription`], `verbose_json` yields a
/// [`TranscriptionVerbose`].
#[derive(Debug, Deserialize, Clone)]
#[serde(untagged)]
pub enum TranscriptionResponse {
    /// The `verbose_json` response shape (requires `duration` and
    /// `language`).
    Verbose(TranscriptionVerbose),
    /// The `json` response shape.
    Plain(Transcription),
}

/// Represents a transcription response returned by the model.
#[derive(Debug, Deserialize, Clone)]
pub struct Transcription {
    /// The transcribed text.
    pub text: String,
    /// The languages detected in the audio.
    ///
    /// Returned by `gpt-transcribe`. An empty array indicates that no
    /// language could be reliably detected.
    pub languages: Option<Vec<TranscriptionLanguage>>,
    /// The log probabilities of the tokens in the transcription.
    ///
    /// Only returned with the models `gpt-4o-transcribe` and
    /// `gpt-4o-mini-transcribe` if `logprobs` is added to the `include`
    /// array.
    pub logprobs: Option<Vec<TranscriptionLogprob>>,
    /// Usage statistics for the request.
    pub usage: Option<TranscriptionUsage>,
}

/// A language detected in transcribed audio.
#[derive(Debug, Deserialize, Clone, PartialEq)]
pub struct TranscriptionLanguage {
    /// The code of a language detected in the audio.
    pub code: String,
}

/// The log probability of a token in the transcription.
#[derive(Debug, Deserialize, Clone, PartialEq)]
pub struct TranscriptionLogprob {
    /// The token in the transcription.
    pub token: Option<String>,
    /// The bytes of the token.
    pub bytes: Option<Vec<f32>>,
    /// The log probability of the token.
    pub logprob: Option<f32>,
}

/// Usage statistics for a transcription request. Billed either by token
/// usage or by audio input duration, discriminated by `type`.
#[derive(Debug, Deserialize, Clone, PartialEq)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum TranscriptionUsage {
    /// Usage statistics for models billed by token usage.
    Tokens {
        /// Number of input tokens billed for this request.
        input_tokens: u64,
        /// Number of output tokens generated.
        output_tokens: u64,
        /// Total number of tokens used (input + output).
        total_tokens: u64,
        /// Details about the input tokens billed for this request.
        input_token_details: Option<UsageTokensInputTokenDetails>,
    },
    /// Usage statistics for models billed by audio input duration.
    Duration {
        /// Duration of the input audio in seconds.
        seconds: f64,
    },
}

/// Details about the input tokens billed for a request.
#[derive(Debug, Deserialize, Clone, PartialEq)]
pub struct UsageTokensInputTokenDetails {
    /// Number of audio tokens billed for this request.
    pub audio_tokens: Option<u64>,
    /// Number of text tokens billed for this request.
    pub text_tokens: Option<u64>,
}

/// Represents a verbose json transcription response.
#[derive(Debug, Deserialize, Clone)]
pub struct TranscriptionVerbose {
    /// The duration of the input audio.
    pub duration: f64,
    /// The language of the input audio.
    pub language: String,
    /// The transcribed text.
    pub text: String,
    /// Segments of the transcribed text and their corresponding details.
    pub segments: Option<Vec<TranscriptionSegment>>,
    /// Usage statistics for models billed by audio input duration.
    pub usage: Option<TranscriptionVerboseUsage>,
    /// Extracted words and their corresponding timestamps.
    pub words: Option<Vec<TranscriptionWord>>,
}

/// Usage statistics for models billed by audio input duration.
#[derive(Debug, Deserialize, Clone)]
pub struct TranscriptionVerboseUsage {
    /// Duration of the input audio in seconds.
    pub seconds: f64,
}

/// A segment of the transcribed text and its corresponding details.
#[derive(Debug, Deserialize, Clone)]
pub struct TranscriptionSegment {
    /// Unique identifier of the segment.
    pub id: u64,
    /// Average logprob of the segment.
    ///
    /// If the value is lower than -1, consider the logprobs failed.
    pub avg_logprob: f64,
    /// Compression ratio of the segment.
    ///
    /// If the value is greater than 2.4, consider the compression failed.
    pub compression_ratio: f64,
    /// End time of the segment in seconds.
    pub end: f64,
    /// Probability of no speech in the segment.
    pub no_speech_prob: f64,
    /// Seek offset of the segment.
    pub seek: u64,
    /// Start time of the segment in seconds.
    pub start: f64,
    /// Temperature parameter used for generating the segment.
    pub temperature: f64,
    /// Text content of the segment.
    pub text: String,
    /// Array of token IDs for the text content.
    pub tokens: Vec<u64>,
}

/// An extracted word and its corresponding timestamp.
#[derive(Debug, Deserialize, Clone)]
pub struct TranscriptionWord {
    /// End time of the word in seconds.
    pub end: f64,
    /// Start time of the word in seconds.
    pub start: f64,
    /// The text content of the word.
    pub word: String,
}

crate::impl_from_str!(TranscriptionResponse);

impl Post for TranscriptionRequest {
    #[inline]
    fn is_streaming(&self) -> bool {
        false
    }

    /// Builds the URL for the request.
    ///
    /// `base_url` should be like <https://api.openai.com/v1>
    fn build_url(&self, base_url: &str) -> Result<String, OapiError> {
        let mut url = Url::parse(base_url.trim_end_matches('/')).map_err(OapiError::UrlError)?;
        url.path_segments_mut()
            .map_err(|_| OapiError::UrlCannotBeBase(base_url.to_string()))?
            .push("audio")
            .push("transcriptions");

        Ok(url.to_string())
    }
}

impl PostNoStream for TranscriptionRequest {
    type Response = TranscriptionResponse;

    /// Sends a transcription POST request using multipart/form-data format,
    /// following the field layout of the official SDK.
    async fn get_response_string(
        &self,
        client: &reqwest::Client,
        url: &str,
        key: &str,
    ) -> Result<String, OapiError> {
        if !self.file.exists() {
            return Err(OapiError::FileNotFoundError(self.file.clone()));
        }

        let content = tokio::fs::read(&self.file).await?;
        let file_name = self
            .file
            .file_name()
            .and_then(|name| name.to_str())
            .ok_or_else(|| OapiError::ResponseError("Invalid file name".to_string()))?
            .to_string();

        let file_part = reqwest::multipart::Part::bytes(content).file_name(file_name);
        let mut form = reqwest::multipart::Form::new().part("file", file_part);

        form = form.text("model", self.model.clone());

        if let Some(language) = &self.language {
            form = form.text("language", language.clone());
        }
        if let Some(prompt) = &self.prompt {
            form = form.text("prompt", prompt.clone());
        }
        if let Some(response_format) = self.response_format {
            let literal = crate::audio::enum_to_literal(&response_format)?;
            form = form.text("response_format", literal);
        }
        if let Some(temperature) = self.temperature {
            form = form.text("temperature", temperature.to_string());
        }
        // List parameters are sent as repeated `name[]` parts, matching the
        // official SDK.
        if let Some(granularities) = &self.timestamp_granularities {
            for granularity in granularities {
                let literal = crate::audio::enum_to_literal(granularity)?;
                form = form.text("timestamp_granularities[]", literal);
            }
        }
        if let Some(include) = &self.include {
            for item in include {
                let literal = crate::audio::enum_to_literal(item)?;
                form = form.text("include[]", literal);
            }
        }
        if let Some(chunking_strategy) = &self.chunking_strategy {
            let value = match chunking_strategy {
                ChunkingStrategy::Auto => "auto".to_string(),
                ChunkingStrategy::ServerVad(config) => {
                    let mut map = serde_json::Map::new();
                    map.insert("type".to_string(), "server_vad".into());
                    if let Some(v) = config.prefix_padding_ms {
                        map.insert("prefix_padding_ms".to_string(), v.into());
                    }
                    if let Some(v) = config.silence_duration_ms {
                        map.insert("silence_duration_ms".to_string(), v.into());
                    }
                    if let Some(v) = config.threshold {
                        map.insert("threshold".to_string(), v.into());
                    }
                    serde_json::to_string(&map).map_err(|e| {
                        OapiError::ResponseError(format!(
                            "Failed to serialize chunking_strategy: {e}"
                        ))
                    })?
                }
            };
            form = form.text("chunking_strategy", value);
        }

        let response = client
            .post(url)
            .header("Accept", "application/json")
            .bearer_auth(key)
            .multipart(form)
            .send()
            .await?;

        crate::rest::response_text_checked(response).await
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_build_url() {
        let request = TranscriptionRequest::default();
        let url = request.build_url("https://api.openai.com/v1/").unwrap();
        assert_eq!(url, "https://api.openai.com/v1/audio/transcriptions");
    }

    /// Enum literals serialize to their official wire values.
    #[test]
    fn enum_literals() {
        assert_eq!(
            crate::audio::enum_to_literal(&AudioResponseFormat::VerboseJson).unwrap(),
            "verbose_json"
        );
        assert_eq!(
            crate::audio::enum_to_literal(&TimestampGranularity::Word).unwrap(),
            "word"
        );
        assert_eq!(
            crate::audio::enum_to_literal(&Include::Logprobs).unwrap(),
            "logprobs"
        );
    }

    /// Deserializes a `json` response (plain transcription shape).
    ///
    /// No accessible provider implements this endpoint, so this fixture is
    /// NOT captured from a live response. The structure follows the schema
    /// of openai-python `types/audio/transcription.py`; the values are
    /// constructed for the test.
    #[test]
    fn parse_plain_response() {
        let content = r#"{
            "text": "The quick brown fox jumped over the lazy dog."
        }"#;

        let response: TranscriptionResponse = content.parse().unwrap();
        let TranscriptionResponse::Plain(transcription) = response else {
            panic!("expected plain transcription");
        };
        assert_eq!(
            transcription.text,
            "The quick brown fox jumped over the lazy dog."
        );
        assert_eq!(transcription.languages, None);
        assert_eq!(transcription.logprobs, None);
        assert_eq!(transcription.usage, None);
    }

    /// Deserializes a `verbose_json` response with segments, words, and
    /// duration-billed usage.
    ///
    /// No accessible provider implements this endpoint, so this fixture is
    /// NOT captured from a live response. The structure follows the schema
    /// of openai-python `types/audio/transcription_verbose.py` +
    /// `transcription_segment.py` + `transcription_word.py`; the values are
    /// constructed for the test.
    #[test]
    fn parse_verbose_response() {
        let content = r#"{
            "duration": 8.47,
            "language": "english",
            "text": "The quick brown fox jumped over the lazy dog.",
            "segments": [
                {
                    "id": 0,
                    "avg_logprob": -0.2365,
                    "compression_ratio": 1.7174,
                    "end": 3.48,
                    "no_speech_prob": 0.01485,
                    "seek": 0,
                    "start": 0.0,
                    "temperature": 0.0,
                    "text": " The quick brown fox jumped over the lazy dog.",
                    "tokens": [464, 2069, 7586, 21831, 18045, 625, 262, 16931, 3290, 13]
                }
            ],
            "words": [
                {
                    "end": 0.36,
                    "start": 0.06,
                    "word": "The"
                }
            ],
            "usage": {
                "type": "duration",
                "seconds": 8.47
            }
        }"#;

        let response: TranscriptionResponse = content.parse().unwrap();
        let TranscriptionResponse::Verbose(verbose) = response else {
            panic!("expected verbose transcription");
        };
        assert_eq!(verbose.duration, 8.47);
        assert_eq!(verbose.language, "english");
        assert_eq!(
            verbose.text,
            "The quick brown fox jumped over the lazy dog."
        );
        let segments = verbose.segments.unwrap();
        assert_eq!(segments.len(), 1);
        assert_eq!(segments[0].id, 0);
        assert_eq!(segments[0].start, 0.0);
        assert_eq!(segments[0].end, 3.48);
        assert_eq!(segments[0].tokens.len(), 10);
        let words = verbose.words.unwrap();
        assert_eq!(words[0].word, "The");
        assert_eq!(words[0].start, 0.06);
        let usage = verbose.usage.unwrap();
        assert_eq!(usage.seconds, 8.47);
    }

    /// Deserializes a token-billed usage object (discriminated by `type`).
    ///
    /// No accessible provider implements this endpoint, so this fixture is
    /// NOT captured from a live response. The structure follows the schema
    /// of openai-python `types/audio/transcription.py` (`UsageTokens`);
    /// the values are constructed for the test.
    #[test]
    fn parse_tokens_usage() {
        let content = r#"{
            "text": "Hello.",
            "usage": {
                "type": "tokens",
                "input_tokens": 76,
                "output_tokens": 13,
                "total_tokens": 89,
                "input_token_details": {
                    "audio_tokens": 76,
                    "text_tokens": 0
                }
            }
        }"#;

        let response: TranscriptionResponse = content.parse().unwrap();
        let TranscriptionResponse::Plain(transcription) = response else {
            panic!("expected plain transcription");
        };
        let TranscriptionUsage::Tokens {
            input_tokens,
            output_tokens,
            total_tokens,
            input_token_details,
        } = transcription.usage.unwrap()
        else {
            panic!("expected tokens usage");
        };
        assert_eq!(input_tokens, 76);
        assert_eq!(output_tokens, 13);
        assert_eq!(total_tokens, 89);
        let details = input_token_details.unwrap();
        assert_eq!(details.audio_tokens, Some(76));
        assert_eq!(details.text_tokens, Some(0));
    }
}