ferrin_google/
speech.rs

1//! Gemini text-to-speech model (`generateContent` with `AUDIO` modality).
2
3use base64::Engine;
4use bytes::Bytes;
5use ferrin_provider_util::http::ResponseHandlers;
6use ferrin_provider_util::http::json_response_handler;
7use ferrin_provider_util::http::post_json;
8use ferrin_spec::JsonObject;
9use ferrin_spec::JsonValue;
10use ferrin_spec::MediaType;
11use ferrin_spec::ModelId;
12use ferrin_spec::ProviderId;
13use ferrin_spec::ResponseMetadata;
14use ferrin_spec::error::ProviderError;
15use ferrin_spec::language_model::RequestMetadata;
16use ferrin_spec::shared::Warning;
17use ferrin_spec::speech_model::SpeechModel;
18use ferrin_spec::speech_model::SpeechOptions;
19use ferrin_spec::speech_model::SpeechResult;
20use serde::Deserialize;
21use serde_json::json;
22
23use crate::api_types::GenerateContentResponse;
24use crate::config::SharedConfig;
25use crate::error::failed_response_handler;
26use crate::options::parse_merged;
27use crate::output::OutputMapper;
28
29/// Provider id family.
30pub const FAMILY: &str = "speech";
31
32/// Voice used when none is requested.
33pub const DEFAULT_VOICE: &str = "Kore";
34
35/// Sample rate assumed when the response media type carries none.
36pub const DEFAULT_SAMPLE_RATE: u32 = 24_000;
37
38/// Speech options (`provider_options["google"]`).
39#[derive(Debug, Clone, Default, PartialEq, Deserialize)]
40#[serde(rename_all = "camelCase", deny_unknown_fields)]
41pub struct GoogleSpeechOptions {
42    /// Multi-speaker voice configuration
43    /// (`{speakerVoiceConfigs: [{speaker, voiceConfig: {prebuiltVoiceConfig: {voiceName}}}]}`).
44    #[serde(default)]
45    pub multi_speaker_voice_config: Option<JsonObject>,
46}
47
48/// Wraps signed 16-bit little-endian mono PCM in a 44-byte WAV header.
49#[must_use]
50pub fn add_wav_header(pcm: &[u8], sample_rate: u32) -> Bytes {
51    let channels: u16 = 1;
52    let bits_per_sample: u16 = 16;
53    let block_align = channels * bits_per_sample / 8;
54    let byte_rate = sample_rate * u32::from(block_align);
55    let data_size = u32::try_from(pcm.len()).unwrap_or(u32::MAX);
56    let mut out = Vec::with_capacity(44 + pcm.len());
57    out.extend_from_slice(b"RIFF");
58    out.extend_from_slice(&(36u32.saturating_add(data_size)).to_le_bytes());
59    out.extend_from_slice(b"WAVE");
60    out.extend_from_slice(b"fmt ");
61    out.extend_from_slice(&16u32.to_le_bytes());
62    out.extend_from_slice(&1u16.to_le_bytes());
63    out.extend_from_slice(&channels.to_le_bytes());
64    out.extend_from_slice(&sample_rate.to_le_bytes());
65    out.extend_from_slice(&byte_rate.to_le_bytes());
66    out.extend_from_slice(&block_align.to_le_bytes());
67    out.extend_from_slice(&bits_per_sample.to_le_bytes());
68    out.extend_from_slice(b"data");
69    out.extend_from_slice(&data_size.to_le_bytes());
70    out.extend_from_slice(pcm);
71    Bytes::from(out)
72}
73
74/// Sample rate encoded in a media type such as `audio/L16;codec=pcm;rate=24000`.
75#[must_use]
76pub fn parse_sample_rate(media_type: &str) -> Option<u32> {
77    media_type
78        .split(';')
79        .map(str::trim)
80        .find_map(|parameter| parameter.strip_prefix("rate="))
81        .and_then(|rate| rate.parse().ok())
82}
83
84/// Text-to-speech model backed by the Gemini TTS models.
85#[derive(Debug, Clone)]
86pub struct GoogleSpeechModel {
87    config: SharedConfig,
88    provider: ProviderId,
89    model_id: ModelId,
90}
91
92/// A prepared speech request.
93#[derive(Debug, Clone)]
94pub struct PreparedSpeechRequest {
95    /// Request body.
96    pub body: JsonValue,
97    /// Warnings.
98    pub warnings: Vec<Warning>,
99    /// Whether raw PCM is returned instead of WAV.
100    pub raw_pcm: bool,
101}
102
103impl GoogleSpeechModel {
104    /// Creates the model.
105    #[must_use]
106    pub fn new(config: SharedConfig, model_id: impl Into<ModelId>) -> Self {
107        Self {
108            provider: config.provider_id(FAMILY),
109            config,
110            model_id: model_id.into(),
111        }
112    }
113
114    /// Builds the request body for `options`.
115    ///
116    /// # Errors
117    ///
118    /// Returns [`ProviderError::InvalidArgument`] for invalid provider options.
119    pub fn prepare_request(
120        &self,
121        options: &SpeechOptions,
122    ) -> Result<PreparedSpeechRequest, ProviderError> {
123        let google = parse_merged::<GoogleSpeechOptions>(
124            &self.config,
125            &options.provider_options,
126            |canonical, custom| {
127                if custom.multi_speaker_voice_config.is_some() {
128                    custom
129                } else {
130                    canonical
131                }
132            },
133        )?;
134        let mut warnings = Vec::new();
135        let speech_config = match &google.multi_speaker_voice_config {
136            Some(multi) => json!({"multiSpeakerVoiceConfig": multi}),
137            None => json!({"voiceConfig": {"prebuiltVoiceConfig": {
138                "voiceName": options.voice.as_deref().unwrap_or(DEFAULT_VOICE)
139            }}}),
140        };
141        let mut prompt = options.text.clone();
142        if let Some(instructions) = &options.instructions {
143            if google.multi_speaker_voice_config.is_some() {
144                warnings.push(Warning::unsupported_with_details(
145                    "instructions",
146                    "Google Gemini TTS ignores `instructions` when `multiSpeakerVoiceConfig` is set, because prepending them would break multi-speaker transcript parsing.",
147                ));
148            } else {
149                prompt = format!("{instructions}: {}", options.text);
150            }
151        }
152        if options.speed.is_some() {
153            warnings.push(Warning::unsupported_with_details(
154                "speed",
155                "Google Gemini TTS models do not support the `speed` option. It was ignored.",
156            ));
157        }
158        if options.language.is_some() {
159            warnings.push(Warning::unsupported_with_details(
160                "language",
161                "Google Gemini TTS models do not support the `language` option. Language is detected automatically from the input text.",
162            ));
163        }
164        let raw_pcm = match options.output_format.as_deref() {
165            Some("pcm") => true,
166            None | Some("wav") => false,
167            Some(other) => {
168                warnings.push(Warning::unsupported_with_details(
169                    "outputFormat",
170                    format!("Unsupported output format: {other}. Using wav instead."),
171                ));
172                false
173            }
174        };
175        let body = json!({
176            "contents": [{"role": "user", "parts": [{"text": prompt}]}],
177            "generationConfig": {
178                "responseModalities": ["AUDIO"],
179                "speechConfig": speech_config,
180            },
181        });
182        Ok(PreparedSpeechRequest {
183            body,
184            warnings,
185            raw_pcm,
186        })
187    }
188}
189
190impl SpeechModel for GoogleSpeechModel {
191    fn provider(&self) -> &ProviderId {
192        &self.provider
193    }
194
195    fn model_id(&self) -> &ModelId {
196        &self.model_id
197    }
198
199    #[tracing::instrument(skip_all, fields(model = %self.model_id))]
200    async fn do_generate(&self, options: SpeechOptions) -> Result<SpeechResult, ProviderError> {
201        let prepared = self.prepare_request(&options)?;
202        let mut warnings = prepared.warnings;
203        let handlers = ResponseHandlers::new(
204            json_response_handler::<GenerateContentResponse>(),
205            failed_response_handler(),
206        );
207        let response = post_json(
208            self.config.transport.as_ref(),
209            self.config
210                .model_url(self.model_id.as_str(), "generateContent"),
211            self.config.headers(&options.headers)?,
212            &prepared.body,
213            &handlers,
214            options.cancellation.clone(),
215        )
216        .await?;
217        let inline = response
218            .value
219            .candidates
220            .iter()
221            .flatten()
222            .flat_map(|candidate| candidate.parts().iter())
223            .find_map(|part| {
224                part.inline_data
225                    .as_ref()
226                    .filter(|data| !data.data.is_empty())
227            });
228        let mime_type = inline.map(|data| data.mime_type.clone());
229        let sample_rate = mime_type
230            .as_deref()
231            .and_then(parse_sample_rate)
232            .unwrap_or(DEFAULT_SAMPLE_RATE);
233        let pcm = match inline {
234            Some(data) => base64::engine::general_purpose::STANDARD
235                .decode(&data.data)
236                .map_err(|error| {
237                    ProviderError::InvalidResponseData(Box::new(
238                        ferrin_spec::error::InvalidResponseDataError::new(
239                            format!("invalid base64 audio data: {error}"),
240                            JsonValue::Null,
241                        ),
242                    ))
243                })?,
244            None => Vec::new(),
245        };
246        let (audio, media_type) = if prepared.raw_pcm || pcm.is_empty() {
247            if prepared.raw_pcm && !pcm.is_empty() {
248                warnings.push(Warning::unsupported_with_details(
249                    "outputFormat",
250                    format!(
251                        "Returning raw PCM audio (signed 16-bit little-endian, mono, {sample_rate} Hz). These bytes have no container header and are not directly playable; see providerMetadata.google for the sample rate and mime type."
252                    ),
253                ));
254            }
255            let media_type = mime_type.clone().map(MediaType::new);
256            (Bytes::from(pcm), media_type)
257        } else {
258            (
259                add_wav_header(&pcm, sample_rate),
260                Some(MediaType::new("audio/wav")),
261            )
262        };
263        let mapper = OutputMapper::new(self.config.clone(), Default::default());
264        let mut metadata = JsonObject::new();
265        metadata.insert("sampleRate".to_owned(), JsonValue::from(sample_rate));
266        metadata.insert(
267            "mimeType".to_owned(),
268            mime_type.map_or(JsonValue::Null, JsonValue::from),
269        );
270        Ok(SpeechResult {
271            audio,
272            media_type,
273            warnings,
274            request: RequestMetadata::with_body(prepared.body),
275            response: ResponseMetadata {
276                id: response.value.response_id.clone(),
277                timestamp: Some(chrono::Utc::now()),
278                model_id: Some(self.model_id.clone()),
279                headers: Some(response.response_headers),
280                body: response.raw,
281            },
282            provider_metadata: Some(mapper.metadata(metadata)),
283        })
284    }
285}
ferrin_google/speech.rs

ferrin_google/
speech.rs
