ferrin_spec/
speech_translation_model.rs

1//! Speech translation model interface (streaming only).
2
3use std::future::Future;
4
5use bytes::Bytes;
6use chrono::DateTime;
7use chrono::Utc;
8use serde::Deserialize;
9use serde::Serialize;
10use tokio_util::sync::CancellationToken;
11
12use crate::dynamic::BoxStream;
13use crate::error::ProviderError;
14use crate::json::JsonValue;
15use crate::language_model::RequestMetadata;
16use crate::language_model::ResponseMetadata;
17use crate::language_model::StreamError;
18use crate::shared::AudioFormat;
19use crate::shared::Headers;
20use crate::shared::ModelId;
21use crate::shared::ProviderId;
22use crate::shared::ProviderMetadata;
23use crate::shared::ProviderOptions;
24use crate::shared::Warning;
25use crate::shared::base64_bytes;
26
27/// A model that translates streamed speech into text (and optionally audio).
28pub trait SpeechTranslationModel: Send + Sync + 'static {
29    /// Provider identifier.
30    fn provider(&self) -> &ProviderId;
31
32    /// Model identifier.
33    fn model_id(&self) -> &ModelId;
34
35    /// Translates a live audio stream.
36    fn do_stream(
37        &self,
38        options: SpeechTranslationStreamOptions,
39    ) -> impl Future<Output = Result<SpeechTranslationStreamResult, ProviderError>> + Send;
40}
41
42/// Options for a speech translation stream.
43pub struct SpeechTranslationStreamOptions {
44    /// Audio chunks.
45    pub audio: BoxStream<'static, Bytes>,
46    /// Format of the audio chunks.
47    pub input_audio_format: AudioFormat,
48    /// Target language code.
49    pub target_language: String,
50    /// Source language code, if known.
51    pub source_language: Option<String>,
52    /// Requested output audio format, if audio output is wanted.
53    pub output_audio_format: Option<AudioFormat>,
54    /// Provider-specific options keyed by provider name.
55    pub provider_options: ProviderOptions,
56    /// Additional request headers.
57    pub headers: Headers,
58    /// Whether to include `Raw` parts.
59    pub include_raw_chunks: bool,
60    /// Cancellation token.
61    pub cancellation: CancellationToken,
62}
63
64impl std::fmt::Debug for SpeechTranslationStreamOptions {
65    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
66        f.debug_struct("SpeechTranslationStreamOptions")
67            .field("audio", &"<stream>")
68            .field("input_audio_format", &self.input_audio_format)
69            .field("target_language", &self.target_language)
70            .field("source_language", &self.source_language)
71            .field("output_audio_format", &self.output_audio_format)
72            .field("provider_options", &self.provider_options)
73            .field("headers", &self.headers)
74            .field("include_raw_chunks", &self.include_raw_chunks)
75            .finish_non_exhaustive()
76    }
77}
78
79/// Usage of a speech translation stream.
80#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)]
81pub struct SpeechTranslationUsage {
82    /// Seconds of input audio.
83    #[serde(default, skip_serializing_if = "Option::is_none")]
84    pub input_audio_seconds: Option<f64>,
85    /// Input audio tokens.
86    #[serde(default, skip_serializing_if = "Option::is_none")]
87    pub input_audio_tokens: Option<u64>,
88    /// Output audio tokens.
89    #[serde(default, skip_serializing_if = "Option::is_none")]
90    pub output_audio_tokens: Option<u64>,
91    /// Input text tokens.
92    #[serde(default, skip_serializing_if = "Option::is_none")]
93    pub input_text_tokens: Option<u64>,
94    /// Output text tokens.
95    #[serde(default, skip_serializing_if = "Option::is_none")]
96    pub output_text_tokens: Option<u64>,
97}
98
99/// A part of a speech translation stream, tagged by `type`.
100#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
101#[serde(tag = "type", rename_all = "kebab-case")]
102#[non_exhaustive]
103pub enum SpeechTranslationStreamPart {
104    /// First part of every stream.
105    StreamStart {
106        /// Warnings.
107        #[serde(default)]
108        warnings: Vec<Warning>,
109    },
110    /// Translated audio chunk.
111    Audio {
112        /// Part id.
113        #[serde(default, skip_serializing_if = "Option::is_none")]
114        id: Option<String>,
115        /// Audio bytes.
116        #[serde(with = "base64_bytes")]
117        audio: Bytes,
118        /// Provider-specific metadata.
119        #[serde(default, skip_serializing_if = "Option::is_none")]
120        provider_metadata: Option<ProviderMetadata>,
121    },
122    /// Translated text increment.
123    OutputTextDelta {
124        /// Part id.
125        #[serde(default, skip_serializing_if = "Option::is_none")]
126        id: Option<String>,
127        /// Appended text.
128        delta: String,
129        /// Provider-specific metadata.
130        #[serde(default, skip_serializing_if = "Option::is_none")]
131        provider_metadata: Option<ProviderMetadata>,
132    },
133    /// Final translated text for a part.
134    OutputTextFinal {
135        /// Part id.
136        #[serde(default, skip_serializing_if = "Option::is_none")]
137        id: Option<String>,
138        /// The text.
139        text: String,
140        /// Provider-specific metadata.
141        #[serde(default, skip_serializing_if = "Option::is_none")]
142        provider_metadata: Option<ProviderMetadata>,
143    },
144    /// Source transcript increment.
145    SourceTranscriptDelta {
146        /// Part id.
147        #[serde(default, skip_serializing_if = "Option::is_none")]
148        id: Option<String>,
149        /// Appended text.
150        delta: String,
151        /// Provider-specific metadata.
152        #[serde(default, skip_serializing_if = "Option::is_none")]
153        provider_metadata: Option<ProviderMetadata>,
154    },
155    /// Interim source transcript.
156    SourceTranscriptPartial {
157        /// Part id.
158        #[serde(default, skip_serializing_if = "Option::is_none")]
159        id: Option<String>,
160        /// Interim text.
161        text: String,
162        /// Start time in seconds.
163        #[serde(default, skip_serializing_if = "Option::is_none")]
164        start_second: Option<f64>,
165        /// End time in seconds.
166        #[serde(default, skip_serializing_if = "Option::is_none")]
167        end_second: Option<f64>,
168        /// Audio channel index.
169        #[serde(default, skip_serializing_if = "Option::is_none")]
170        channel_index: Option<u32>,
171        /// Provider-specific metadata.
172        #[serde(default, skip_serializing_if = "Option::is_none")]
173        provider_metadata: Option<ProviderMetadata>,
174    },
175    /// Final source transcript segment.
176    SourceTranscriptFinal {
177        /// Part id.
178        #[serde(default, skip_serializing_if = "Option::is_none")]
179        id: Option<String>,
180        /// Final text.
181        text: String,
182        /// Start time in seconds.
183        #[serde(default, skip_serializing_if = "Option::is_none")]
184        start_second: Option<f64>,
185        /// End time in seconds.
186        #[serde(default, skip_serializing_if = "Option::is_none")]
187        end_second: Option<f64>,
188        /// Audio channel index.
189        #[serde(default, skip_serializing_if = "Option::is_none")]
190        channel_index: Option<u32>,
191        /// Provider-specific metadata.
192        #[serde(default, skip_serializing_if = "Option::is_none")]
193        provider_metadata: Option<ProviderMetadata>,
194    },
195    /// Response metadata.
196    ResponseMetadata {
197        /// Timestamp.
198        #[serde(default, skip_serializing_if = "Option::is_none")]
199        timestamp: Option<DateTime<Utc>>,
200        /// Model id.
201        #[serde(default, skip_serializing_if = "Option::is_none")]
202        model_id: Option<ModelId>,
203        /// Headers.
204        #[serde(default, skip_serializing_if = "Option::is_none")]
205        headers: Option<Headers>,
206        /// Body.
207        #[serde(default, skip_serializing_if = "Option::is_none")]
208        body: Option<JsonValue>,
209    },
210    /// Last part of a successful stream.
211    Finish {
212        /// Full source transcript.
213        source_text: String,
214        /// Full translated text.
215        output_text: String,
216        /// Audio duration in seconds.
217        #[serde(default, skip_serializing_if = "Option::is_none")]
218        duration_in_seconds: Option<f64>,
219        /// Usage.
220        #[serde(default, skip_serializing_if = "Option::is_none")]
221        usage: Option<SpeechTranslationUsage>,
222        /// Provider-specific metadata.
223        #[serde(default, skip_serializing_if = "Option::is_none")]
224        provider_metadata: Option<ProviderMetadata>,
225    },
226    /// A raw provider chunk.
227    Raw {
228        /// The chunk as JSON.
229        raw_value: JsonValue,
230    },
231    /// An error; the stream ends after this part.
232    Error {
233        /// The error.
234        error: StreamError,
235    },
236}
237
238/// Result of a speech translation stream call.
239pub struct SpeechTranslationStreamResult {
240    /// The stream of parts.
241    pub stream: BoxStream<'static, SpeechTranslationStreamPart>,
242    /// Request metadata.
243    pub request: RequestMetadata,
244    /// Response metadata known at stream start.
245    pub response: ResponseMetadata,
246}
247
248impl std::fmt::Debug for SpeechTranslationStreamResult {
249    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
250        f.debug_struct("SpeechTranslationStreamResult")
251            .field("stream", &"<stream>")
252            .field("request", &self.request)
253            .field("response", &self.response)
254            .finish()
255    }
256}
ferrin_spec/speech_translation_model.rs

ferrin_spec/
speech_translation_model.rs
