foundry_local_sdk/openai/
audio_client.rs

1//! OpenAI-compatible audio transcription client.
2#![allow(deprecated)] // this module implements the deprecated OpenAI facade
3
4use std::path::Path;
5
6use serde_json::{json, Value};
7
8use crate::detail::native::NativeModel;
9use crate::detail::session::{run_openai_json_streaming, NativeSession};
10use crate::detail::task::spawn_blocking;
11use crate::error::{FoundryLocalError, Result};
12
13use super::json_stream::JsonStream;
14use super::live_audio_session::LiveAudioTranscriptionSession;
15
16/// A segment of a transcription, as returned by the OpenAI-compatible API.
17#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
18pub struct TranscriptionSegment {
19    /// Segment index.
20    pub id: i32,
21    /// Seek offset of the segment.
22    pub seek: i32,
23    /// Start time of the segment in seconds.
24    pub start: f64,
25    /// End time of the segment in seconds.
26    pub end: f64,
27    /// Transcribed text of the segment.
28    pub text: String,
29    /// Token IDs corresponding to the text.
30    #[serde(skip_serializing_if = "Option::is_none")]
31    pub tokens: Option<Vec<i32>>,
32    /// Temperature used for generation.
33    #[serde(skip_serializing_if = "Option::is_none")]
34    pub temperature: Option<f64>,
35    /// Average log probability of the segment.
36    #[serde(skip_serializing_if = "Option::is_none")]
37    pub avg_logprob: Option<f64>,
38    /// Compression ratio of the segment.
39    #[serde(skip_serializing_if = "Option::is_none")]
40    pub compression_ratio: Option<f64>,
41    /// Probability of no speech in the segment.
42    #[serde(skip_serializing_if = "Option::is_none")]
43    pub no_speech_prob: Option<f64>,
44}
45
46/// A word with timing information, as returned by the OpenAI-compatible API.
47#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
48pub struct TranscriptionWord {
49    /// The word text.
50    pub word: String,
51    /// Start time of the word in seconds.
52    pub start: f64,
53    /// End time of the word in seconds.
54    pub end: f64,
55}
56
57/// OpenAI-compatible audio transcription response.
58#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
59pub struct AudioTranscriptionResponse {
60    /// The transcribed text.
61    pub text: String,
62    /// The language of the input audio (if detected).
63    #[serde(skip_serializing_if = "Option::is_none")]
64    pub language: Option<String>,
65    /// Duration of the input audio in seconds (if available).
66    #[serde(skip_serializing_if = "Option::is_none")]
67    pub duration: Option<f64>,
68    /// Segments of the transcription (if available).
69    #[serde(skip_serializing_if = "Option::is_none")]
70    pub segments: Option<Vec<TranscriptionSegment>>,
71    /// Words with timestamps (if available).
72    #[serde(skip_serializing_if = "Option::is_none")]
73    pub words: Option<Vec<TranscriptionWord>>,
74}
75
76/// Tuning knobs for audio transcription requests.
77///
78/// Use the chainable setter methods to configure, e.g.:
79///
80/// ```ignore
81/// let client = model.create_audio_client()
82///     .language("en")
83///     .temperature(0.2);
84/// ```
85#[derive(Debug, Clone, Default)]
86pub struct AudioClientSettings {
87    language: Option<String>,
88    temperature: Option<f64>,
89}
90
91impl AudioClientSettings {
92    fn serialize(&self, model_id: &str, file_name: &str) -> Value {
93        let mut map = serde_json::Map::new();
94
95        map.insert("model".into(), json!(model_id));
96        map.insert("filename".into(), json!(file_name));
97
98        if let Some(ref lang) = self.language {
99            map.insert("language".into(), json!(lang));
100        }
101        if let Some(temp) = self.temperature {
102            map.insert("temperature".into(), json!(temp));
103        }
104
105        Value::Object(map)
106    }
107}
108
109/// A stream of [`AudioTranscriptionResponse`] chunks.
110///
111/// Returned by [`AudioClient::transcribe_streaming`].
112pub type AudioTranscriptionStream = JsonStream<AudioTranscriptionResponse>;
113
114/// Client for OpenAI-compatible audio transcription backed by a local model.
115#[deprecated(
116    since = "2.0.0",
117    note = "The OpenAI direct clients are deprecated; use the Session API instead \
118            (`AudioSession::new(&model)`)."
119)]
120pub struct AudioClient {
121    model_id: String,
122    model: NativeModel,
123    settings: AudioClientSettings,
124}
125
126impl AudioClient {
127    pub(crate) fn new(model_id: &str, model: NativeModel) -> Self {
128        Self {
129            model_id: model_id.to_owned(),
130            model,
131            settings: AudioClientSettings::default(),
132        }
133    }
134
135    /// Set the language hint for transcription.
136    pub fn language(mut self, lang: impl Into<String>) -> Self {
137        self.settings.language = Some(lang.into());
138        self
139    }
140
141    /// Set the sampling temperature.
142    pub fn temperature(mut self, v: f64) -> Self {
143        self.settings.temperature = Some(v);
144        self
145    }
146
147    /// Transcribe an audio file.
148    pub async fn transcribe(
149        &self,
150        audio_file_path: impl AsRef<Path>,
151    ) -> Result<AudioTranscriptionResponse> {
152        let path_str =
153            audio_file_path
154                .as_ref()
155                .to_str()
156                .ok_or_else(|| FoundryLocalError::Validation {
157                    reason: "audio file path is not valid UTF-8".into(),
158                })?;
159        Self::validate_path(path_str)?;
160
161        let request = self.settings.serialize(&self.model_id, path_str);
162        let request_json = serde_json::to_string(&request)?;
163        let model = self.model.clone();
164
165        let raw = spawn_blocking(move || {
166            let session = NativeSession::create(&model)?;
167            session.run_openai_json(&request_json)
168        })
169        .await?;
170
171        let parsed: AudioTranscriptionResponse = serde_json::from_str(&raw)?;
172        Ok(parsed)
173    }
174
175    /// Transcribe an audio file with streaming results, returning an
176    /// [`AudioTranscriptionStream`].
177    pub async fn transcribe_streaming(
178        &self,
179        audio_file_path: impl AsRef<Path>,
180    ) -> Result<AudioTranscriptionStream> {
181        let path_str =
182            audio_file_path
183                .as_ref()
184                .to_str()
185                .ok_or_else(|| FoundryLocalError::Validation {
186                    reason: "audio file path is not valid UTF-8".into(),
187                })?;
188        Self::validate_path(path_str)?;
189
190        let request = self.settings.serialize(&self.model_id, path_str);
191        let request_json = serde_json::to_string(&request)?;
192        let model = self.model.clone();
193
194        let session = spawn_blocking(move || NativeSession::create(&model)).await?;
195        let rx = run_openai_json_streaming(session, request_json, Box::new(Some));
196        Ok(AudioTranscriptionStream::new(rx))
197    }
198
199    /// Create a [`LiveAudioTranscriptionSession`] for real-time audio
200    /// streaming transcription.
201    ///
202    /// Configure the session's [`settings`](LiveAudioTranscriptionSession::settings)
203    /// before calling [`start`](LiveAudioTranscriptionSession::start).
204    #[deprecated(
205        since = "2.0.0",
206        note = "The OpenAI direct clients are deprecated; use `AudioSession::new(&model)` \
207                with streaming instead."
208    )]
209    #[allow(deprecated)]
210    pub fn create_live_transcription_session(&self) -> LiveAudioTranscriptionSession {
211        LiveAudioTranscriptionSession::new(&self.model_id, self.model.clone())
212    }
213
214    fn validate_path(path: &str) -> Result<()> {
215        if path.trim().is_empty() {
216            return Err(FoundryLocalError::Validation {
217                reason: "audio_file_path must be a non-empty string".into(),
218            });
219        }
220        Ok(())
221    }
222}
foundry_local_sdk/openai/audio_client.rs

foundry_local_sdk/openai/
audio_client.rs
