outfox_openai/spec/audio/
audio_.rs

1use bytes::Bytes;
2use derive_builder::Builder;
3use serde::{Deserialize, Serialize};
4
5use crate::error::OpenAIError;
6use crate::spec::InputSource;
7use crate::spec::audio::{LogProbProperties, TranscriptTextUsageDuration, TranscriptionUsage};
8
9// openapi spec type: VoiceIdsShared
10#[derive(Debug, Default, Serialize, Deserialize, Clone, PartialEq)]
11#[serde(rename_all = "lowercase")]
12pub enum Voice {
13    #[default]
14    Alloy,
15    Ash,
16    Ballad,
17    Coral,
18    Echo,
19    Fable,
20    Onyx,
21    Nova,
22    Sage,
23    Shimmer,
24    Verse,
25    #[serde(untagged)]
26    Other(String),
27}
28
29#[derive(Debug, Default, Clone, PartialEq)]
30pub struct AudioInput {
31    pub source: InputSource,
32}
33
34#[derive(Debug, Serialize, Deserialize, Default, Clone, Copy, PartialEq)]
35#[serde(rename_all = "snake_case")]
36pub enum AudioResponseFormat {
37    #[default]
38    Json,
39    Text,
40    Srt,
41    VerboseJson,
42    Vtt,
43    DiarizedJson,
44}
45
46#[derive(Debug, Serialize, Deserialize, Default, Clone, Copy, PartialEq)]
47#[serde(rename_all = "snake_case")]
48pub enum TranslationResponseFormat {
49    #[default]
50    Json,
51    Text,
52    Srt,
53    VerboseJson,
54    Vtt,
55}
56
57#[derive(Debug, Serialize, Deserialize, Default, Clone, Copy, PartialEq)]
58#[serde(rename_all = "lowercase")]
59pub enum SpeechResponseFormat {
60    #[default]
61    Mp3,
62    Opus,
63    Aac,
64    Flac,
65    Pcm,
66    Wav,
67}
68
69#[derive(Debug, Default, Serialize, Deserialize, Clone, PartialEq)]
70pub enum SpeechModel {
71    #[default]
72    #[serde(rename = "tts-1")]
73    Tts1,
74    #[serde(rename = "tts-1-hd")]
75    Tts1Hd,
76    #[serde(rename = "gpt-4o-mini-tts")]
77    Gpt4oMiniTts,
78    #[serde(untagged)]
79    Other(String),
80}
81
82#[derive(Debug, Default, Serialize, Deserialize, Clone, PartialEq)]
83#[serde(rename_all = "lowercase")]
84pub enum TimestampGranularity {
85    Word,
86    #[default]
87    Segment,
88}
89
90#[derive(Clone, Default, Debug, Builder, PartialEq)]
91#[builder(name = "CreateTranscriptionRequestArgs")]
92#[builder(pattern = "mutable")]
93#[builder(setter(into, strip_option), default)]
94#[builder(derive(Debug))]
95#[builder(build_fn(error = "OpenAIError"))]
96pub struct CreateTranscriptionRequest {
97    /// The audio file object (not file name) to transcribe, in one of these formats:
98    /// flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.
99    pub file: AudioInput,
100
101    /// ID of the model to use. The options are `gpt-4o-transcribe`, `gpt-4o-mini-transcribe`,
102    /// `whisper-1` (which is powered by our open source Whisper V2 model), and
103    /// `gpt-4o-transcribe-diarize`.
104    pub model: String,
105
106    /// The language of the input audio. Supplying the input language in
107    /// [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) (e.g. `en`) format will improve
108    /// accuracy and latency.
109    pub language: Option<String>,
110
111    /// An optional text to guide the model's style or continue a previous audio segment. The
112    /// [prompt](https://platform.openai.com/docs/guides/speech-to-text#prompting) should match the audio
113    /// language. This field is not supported when using `gpt-4o-transcribe-diarize`.
114    pub prompt: Option<String>,
115
116    /// The format of the output, in one of these options: `json`, `text`, `srt`, `verbose_json`,
117    /// `vtt`, or `diarized_json`. For `gpt-4o-transcribe` and `gpt-4o-mini-transcribe`, the
118    /// only supported format is `json`. For `gpt-4o-transcribe-diarize`, the supported formats
119    /// are `json`, `text`, and `diarized_json`, with `diarized_json` required to receive
120    /// speaker annotations.
121    pub response_format: Option<AudioResponseFormat>,
122
123    /// TThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output
124    /// more random, while lower values like 0.2 will make it more focused and deterministic.
125    /// If set to 0, the model will use [log probability](https://en.wikipedia.org/wiki/Log_probability) to automatically
126    /// increase the temperature until certain thresholds are hit.
127    pub temperature: Option<f32>, // default: 0
128
129    /// Additional information to include in the transcription response.
130
131    /// `logprobs` will return the log probabilities of the tokens in the
132    /// response to understand the model's confidence in the transcription.
133    /// `logprobs` only works with response_format set to `json` and only with
134    /// the models `gpt-4o-transcribe` and `gpt-4o-mini-transcribe`. This field is not supported
135    /// when using `gpt-4o-transcribe-diarize`.
136    pub include: Option<Vec<TranscriptionInclude>>,
137
138    /// The timestamp granularities to populate for this transcription. `response_format` must be
139    /// set `verbose_json` to use timestamp granularities. Either or both of these options are
140    /// supported: `word`, or `segment`. Note: There is no additional latency for segment
141    /// timestamps, but generating word timestamps incurs additional latency. This option is
142    /// not available for `gpt-4o-transcribe-diarize`.
143    pub timestamp_granularities: Option<Vec<TimestampGranularity>>,
144
145    /// If set to true, the model response data will be streamed to the client
146    /// as it is generated using [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format).
147    /// See the [Streaming section of the Speech-to-Text guide](https://platform.openai.com/docs/guides/speech-to-text?lang=curl#streaming-transcriptions)
148    /// for more information.
149    /// Note: Streaming is not supported for the `whisper-1` model and will be ignored.    
150    pub stream: Option<bool>,
151
152    /// Controls how the audio is cut into chunks. When set to `"auto"`, the server first
153    /// normalizes loudness and then uses voice activity detection (VAD) to choose boundaries.
154    /// `server_vad` object can be provided to tweak VAD detection parameters manually. If
155    /// unset, the audio is transcribed as a single block. Required when using
156    /// `gpt-4o-transcribe-diarize` for inputs longer than 30 seconds.
157    pub chunking_strategy: Option<TranscriptionChunkingStrategy>,
158
159    /// Optional list of speaker names that correspond to the audio samples provided in
160    /// `known_speaker_references[]`. Each entry should be a short identifier (for example
161    /// `customer` or `agent`). Up to 4 speakers are supported.
162    pub known_speaker_names: Option<Vec<String>>,
163
164    /// Optional list of audio samples (as [data
165    /// URLs](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URLs)) that contain
166    /// known speaker references matching `known_speaker_names[]`. Each sample must be between 2
167    /// and 10 seconds, and can use any of the same input audio formats supported by `file`.
168    pub known_speaker_references: Option<Vec<String>>,
169}
170
171#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Default)]
172#[serde(rename_all = "snake_case")]
173pub enum TranscriptionChunkingStrategy {
174    #[default]
175    Auto,
176    #[serde(untagged)]
177    ServerVad(VadConfig),
178}
179
180#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Default)]
181#[serde(rename_all = "snake_case")]
182pub enum VadConfigType {
183    #[default]
184    ServerVad,
185}
186
187#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Default)]
188pub struct VadConfig {
189    pub kind: VadConfigType,
190    /// Amount of audio to include before the VAD detected speech (in milliseconds). Default: 300.
191    #[serde(skip_serializing_if = "Option::is_none")]
192    pub prefix_padding_ms: Option<u32>,
193
194    /// Duration of silence to detect speech stop (in milliseconds).
195    /// With shorter values the model will respond more quickly,
196    /// but may jump in on short pauses from the user. Default: 200.
197    #[serde(skip_serializing_if = "Option::is_none")]
198    pub silence_duration_ms: Option<u32>,
199
200    /// Sensitivity threshold (0.0 to 1.0) for voice activity detection. A
201    /// higher threshold will require louder audio to activate the model, and
202    /// thus might perform better in noisy environments. Default: 0.5.
203    #[serde(skip_serializing_if = "Option::is_none")]
204    pub threshold: Option<f32>,
205}
206
207#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
208#[serde(rename_all = "snake_case")]
209pub enum TranscriptionInclude {
210    Logprobs,
211}
212
213/// Represents a transcription response returned by model, based on the provided
214/// input.
215#[derive(Debug, Deserialize, Clone, Serialize)]
216pub struct CreateTranscriptionResponseJson {
217    /// The transcribed text.
218    pub text: String,
219
220    /// The log probabilities of the tokens in the transcription. Only returned with the models
221    /// `gpt-4o-transcribe` and `gpt-4o-mini-transcribe` if `logprobs` is added to the `include`
222    /// array.
223    pub logprobs: Option<Vec<LogProbProperties>>,
224
225    /// Token usage statistics for the request.
226    pub usage: TranscriptionUsage,
227}
228
229#[derive(Debug, Serialize, Deserialize, Clone)]
230#[serde(rename_all = "lowercase")]
231pub enum CreateTranscriptionResponseDiarizedJsonTask {
232    Transcribe,
233}
234
235#[derive(Debug, Serialize, Deserialize, Clone)]
236pub struct CreateTranscriptionResponseDiarizedJson {
237    /// The type of task that was run. Always `transcribe`.
238    pub task: Option<CreateTranscriptionResponseDiarizedJsonTask>,
239
240    /// Duration of the input audio in seconds.
241    pub duration: Option<f32>,
242
243    /// The concatenated transcript text for the entire audio input.
244    pub text: String,
245
246    /// Segments of the transcript annotated with timestamps and speaker labels.
247    pub segments: Vec<TranscriptionDiarizedSegment>,
248
249    /// Token or duration usage statistics for the request.
250    pub usage: TranscriptionUsage,
251}
252
253/// Represents a verbose json transcription response returned by model, based on
254/// the provided input.
255#[derive(Debug, Deserialize, Clone, Serialize)]
256pub struct CreateTranscriptionResponseVerboseJson {
257    /// The language of the input audio.
258    pub language: String,
259
260    /// The duration of the input audio.
261    pub duration: f32,
262
263    /// The transcribed text.
264    pub text: String,
265
266    /// Extracted words and their corresponding timestamps.
267    #[serde(skip_serializing_if = "Option::is_none")]
268    pub words: Option<Vec<TranscriptionWord>>,
269
270    /// Segments of the transcribed text and their corresponding details.
271    #[serde(skip_serializing_if = "Option::is_none")]
272    pub segments: Option<Vec<TranscriptionSegment>>,
273
274    /// Usage statistics for models billed by audio input duration.
275    pub usage: TranscriptTextUsageDuration,
276}
277
278#[derive(Debug, Deserialize, Clone, Serialize)]
279pub struct TranscriptionWord {
280    /// The text content of the word.
281    pub word: String,
282
283    /// Start time of the word in seconds.
284    pub start: f32,
285
286    /// End time of the word in seconds.
287    pub end: f32,
288}
289
290#[derive(Debug, Serialize, Deserialize, Clone)]
291pub enum TranscriptionDiarizedSegmentType {
292    #[serde(rename = "transcript.text.segment")]
293    TranscriptTextSegment,
294}
295
296#[derive(Debug, Serialize, Deserialize, Clone)]
297pub struct TranscriptionDiarizedSegment {
298    /// The type of the segment. Always `transcript.text.segment`.
299    pub kind: TranscriptionDiarizedSegmentType,
300
301    /// Unique identifier for the segment.
302    pub id: String,
303
304    /// Start timestamp of the segment in seconds.
305    pub start: f32,
306
307    /// End timestamp of the segment in seconds.
308    pub end: f32,
309
310    /// Transcript text for this segment.
311    pub text: String,
312
313    /// Speaker label for this segment.
314    /// When known speakers are provided, the label matches known_speaker_names[].
315    /// Otherwise speakers are labeled sequentially using capital letters (`A`, `B`, ...).
316    pub speaker: String,
317}
318
319#[derive(Debug, Deserialize, Clone, Serialize)]
320pub struct TranscriptionSegment {
321    /// Unique identifier of the segment.
322    pub id: u32,
323
324    // Seek offset of the segment.
325    pub seek: u32,
326
327    /// Start time of the segment in seconds.
328    pub start: f32,
329
330    /// End time of the segment in seconds.
331    pub end: f32,
332
333    /// Text content of the segment.
334    pub text: String,
335
336    /// Array of token IDs for the text content.
337    pub tokens: Vec<u32>,
338
339    /// Temperature parameter used for generating the segment.
340    pub temperature: f32,
341
342    /// Average logprob of the segment. If the value is lower than -1, consider
343    /// the logprobs failed.
344    pub avg_logprob: f32,
345
346    /// Compression ratio of the segment. If the value is greater than 2.4,
347    /// consider the compression failed.
348    pub compression_ratio: f32,
349
350    /// Probability of no speech in the segment. If the value is higher than 1.0
351    /// and the `avg_logprob` is below -1, consider this segment silent.
352    pub no_speech_prob: f32,
353}
354
355#[derive(Clone, Default, Debug, Builder, PartialEq, Serialize, Deserialize)]
356#[builder(name = "CreateSpeechRequestArgs")]
357#[builder(pattern = "mutable")]
358#[builder(setter(into, strip_option), default)]
359#[builder(derive(Debug))]
360#[builder(build_fn(error = "OpenAIError"))]
361pub struct CreateSpeechRequest {
362    /// The text to generate audio for. The maximum length is 4096 characters.
363    pub input: String,
364
365    /// One of the available [TTS models](https://platform.openai.com/docs/models#tts): `tts-1`,
366    /// `tts-1-hd` or `gpt-4o-mini-tts`.
367    pub model: SpeechModel,
368
369    /// The voice to use when generating the audio. Supported voices are `alloy`, `ash`, `coral`,
370    /// `echo`, `fable`, `onyx`, `nova`, `sage`, `shimmer` and `verse`.
371
372    /// The voice to use when generating the audio. Supported voices are `alloy`, `ash`, `ballad`,
373    /// `coral`, `echo`, `fable`, `onyx`, `nova`, `sage`, `shimmer`, and `verse`. Previews of the
374    /// voices are available in the [Text to speech guide](https://platform.openai.com/docs/guides/text-to-speech#voice-options).
375    pub voice: Voice,
376
377    /// Control the voice of your generated audio with additional instructions.
378    /// Does not work with `tts-1` or `tts-1-hd`.
379    #[serde(skip_serializing_if = "Option::is_none")]
380    pub instructions: Option<String>,
381
382    /// The format to audio in. Supported formats are `mp3`, `opus`, `aac`, `flac`, `wav`, and
383    /// `pcm`.
384    #[serde(skip_serializing_if = "Option::is_none")]
385    pub response_format: Option<SpeechResponseFormat>,
386
387    /// The speed of the generated audio. Select a value from 0.25 to 4.0. 1.0 is the default.
388    #[serde(skip_serializing_if = "Option::is_none")]
389    pub speed: Option<f32>, // default: 1.0
390
391    /// The format to stream the audio in. Supported formats are `sse` and `audio`. `sse` is not
392    /// supported for `tts-1` or `tts-1-hd`.
393    #[serde(skip_serializing_if = "Option::is_none")]
394    pub stream_format: Option<StreamFormat>,
395}
396
397#[derive(Debug, Serialize, Deserialize, Default, Clone, Copy, PartialEq)]
398#[serde(rename_all = "snake_case")]
399pub enum StreamFormat {
400    #[default]
401    #[serde(rename = "sse")]
402    SSE,
403    #[serde(rename = "audio")]
404    Audio,
405}
406
407#[derive(Clone, Default, Debug, Builder, PartialEq)]
408#[builder(name = "CreateTranslationRequestArgs")]
409#[builder(pattern = "mutable")]
410#[builder(setter(into, strip_option), default)]
411#[builder(derive(Debug))]
412#[builder(build_fn(error = "OpenAIError"))]
413pub struct CreateTranslationRequest {
414    /// The audio file object (not file name) translate, in one of these
415    /// formats: flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.
416    pub file: AudioInput,
417
418    /// ID of the model to use. Only `whisper-1` (which is powered by our open
419    /// source Whisper V2 model) is currently available.
420    pub model: String,
421
422    /// An optional text to guide the model's style or continue a previous audio
423    /// segment. The [prompt](https://platform.openai.com/docs/guides/speech-to-text#prompting) should be in English.
424    pub prompt: Option<String>,
425
426    /// The format of the transcript output, in one of these options: json, text, srt,
427    /// verbose_json, or vtt.
428    pub response_format: Option<TranslationResponseFormat>,
429
430    /// The sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use [log probability](https://en.wikipedia.org/wiki/Log_probability) to automatically increase the temperature until certain thresholds are hit.
431    pub temperature: Option<f32>, // default: 0
432}
433
434#[derive(Debug, Deserialize, Clone, PartialEq, Serialize)]
435pub struct CreateTranslationResponseJson {
436    pub text: String,
437}
438
439#[derive(Debug, Deserialize, Clone, Serialize)]
440pub struct CreateTranslationResponseVerboseJson {
441    /// The language of the output translation (always `english`).
442    pub language: String,
443    /// The duration of the input audio.
444    pub duration: String,
445    /// The translated text.
446    pub text: String,
447    /// Segments of the translated text and their corresponding details.
448    #[serde(skip_serializing_if = "Option::is_none")]
449    pub segments: Option<Vec<TranscriptionSegment>>,
450}
451
452#[derive(Debug, Clone)]
453pub struct CreateSpeechResponse {
454    pub bytes: Bytes,
455}
456
457/// A consent recording used to authorize creation of a custom voice.
458#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
459pub struct VoiceConsentResource {
460    /// The object type, which is always `audio.voice_consent`.
461    pub object: String,
462    /// The consent recording identifier.
463    pub id: String,
464    /// The label provided when the consent recording was uploaded.
465    pub name: String,
466    /// The BCP 47 language tag for the consent phrase (for example, `en-US`).
467    pub language: String,
468    /// The Unix timestamp (in seconds) for when the consent recording was created.
469    pub created_at: u64,
470}
471
472/// Request to create a voice consent recording.
473#[derive(Clone, Default, Debug, Builder, PartialEq)]
474#[builder(name = "CreateVoiceConsentRequestArgs")]
475#[builder(pattern = "mutable")]
476#[builder(setter(into, strip_option), default)]
477#[builder(derive(Debug))]
478#[builder(build_fn(error = "OpenAIError"))]
479pub struct CreateVoiceConsentRequest {
480    /// The label to use for this consent recording.
481    pub name: String,
482    /// The consent audio recording file. Maximum size is 10 MiB.
483    /// Supported MIME types: `audio/mpeg`, `audio/wav`, `audio/x-wav`, `audio/ogg`,
484    /// `audio/aac`, `audio/flac`, `audio/webm`, `audio/mp4`.
485    pub recording: AudioInput,
486    /// The BCP 47 language tag for the consent phrase (for example, `en-US`).
487    pub language: String,
488}
489
490/// Request to update a voice consent recording (metadata only).
491#[derive(Clone, Serialize, Default, Debug, Deserialize, Builder, PartialEq)]
492#[builder(name = "UpdateVoiceConsentRequestArgs")]
493#[builder(pattern = "mutable")]
494#[builder(setter(into, strip_option), default)]
495#[builder(derive(Debug))]
496#[builder(build_fn(error = "OpenAIError"))]
497pub struct UpdateVoiceConsentRequest {
498    /// The updated label for this consent recording.
499    pub name: String,
500}
501
502/// The voice consent deletion object.
503#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
504pub struct VoiceConsentDeletedResource {
505    /// The consent recording identifier.
506    pub id: String,
507    /// The object type, which is always `audio.voice_consent`.
508    pub object: String,
509    /// Whether the consent recording was deleted.
510    pub deleted: bool,
511}
512
513/// The voice consent list object.
514#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
515pub struct VoiceConsentListResource {
516    /// The object type, which is always `list`.
517    pub object: String,
518    /// The list of voice consent recordings.
519    pub data: Vec<VoiceConsentResource>,
520    /// The ID of the first voice consent recording in the list.
521    #[serde(skip_serializing_if = "Option::is_none")]
522    pub first_id: Option<String>,
523    /// The ID of the last voice consent recording in the list.
524    #[serde(skip_serializing_if = "Option::is_none")]
525    pub last_id: Option<String>,
526    /// Whether there are more voice consent recordings available.
527    pub has_more: bool,
528}
529
530/// A custom voice that can be used for audio output.
531#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
532pub struct VoiceResource {
533    /// The object type, which is always `audio.voice`.
534    pub object: String,
535    /// The voice identifier, which can be referenced in API endpoints.
536    pub id: String,
537    /// The name of the voice.
538    pub name: String,
539    /// The Unix timestamp (in seconds) for when the voice was created.
540    pub created_at: u64,
541}
542
543/// Request to create a custom voice.
544#[derive(Clone, Default, Debug, Builder, PartialEq)]
545#[builder(name = "CreateVoiceRequestArgs")]
546#[builder(pattern = "mutable")]
547#[builder(setter(into, strip_option), default)]
548#[builder(derive(Debug))]
549#[builder(build_fn(error = "OpenAIError"))]
550pub struct CreateVoiceRequest {
551    /// The name of the new voice.
552    pub name: String,
553    /// The sample audio recording file. Maximum size is 10 MiB.
554    /// Supported MIME types: `audio/mpeg`, `audio/wav`, `audio/x-wav`, `audio/ogg`,
555    /// `audio/aac`, `audio/flac`, `audio/webm`, `audio/mp4`.
556    pub audio_sample: AudioInput,
557    /// The consent recording ID (for example, `cons_1234`).
558    pub consent: String,
559}
outfox_openai/spec/audio/audio_.rs

outfox_openai/spec/audio/
audio_.rs
