rig_core/providers/gemini/
audio_generation.rs

1//! Text-to-speech through the Interactions API. A request speaks its text
2//! in its voice; a [`Script`] speaks turns, each with an optional speaker
3//! and style, in one voice or one voice per speaker. [`format_of`] reads the
4//! format of the audio a reply carries.
5//!
6//! ```no_run
7//! use rig_core::audio_generation::AudioGenerationRequestBuilder;
8//! use rig_core::providers::gemini::Gemini;
9//! use rig_core::providers::gemini::audio_generation::{
10//!     AudioFormat, GEMINI_3_8_FLASH_TTS, Script, Turn,
11//! };
12//!
13//! # async fn run() -> Result<(), Box<dyn std::error::Error>> {
14//! let model = Gemini::from_env()?.audio_generation(GEMINI_3_8_FLASH_TTS);
15//! let hello = AudioGenerationRequestBuilder::new("Have a wonderful day!", "Kore").build();
16//! let wav = model.call(hello).await?.audio;
17//!
18//! let podcast = Script::speakers([("Joe", "Puck"), ("Jane", "Kore")])
19//!     .turn(Turn::new("How's it going, Jane?").speaker("Joe").style("cheerful"))
20//!     .turn(Turn::new("Not bad at all.").speaker("Jane"))
21//!     .format(AudioFormat::pcm(24_000))
22//!     .build();
23//! let pcm = model.call(podcast).await?.audio;
24//! # let _ = (wav, pcm);
25//! # Ok(())
26//! # }
27//! ```
28
29use base64::Engine;
30use base64::prelude::BASE64_STANDARD;
31use serde::Serialize;
32use serde_json::{Value, json};
33
34use super::interactions_api::streaming::usage_of;
35use super::interactions_api::{InteractionSpeech, InteractionSpeechConfig};
36use crate::audio_generation::{AudioGenerationRequest, AudioGenerationResponse};
37use crate::error::{EncodeError, ProviderError};
38use crate::json_utils::Lenient;
39use crate::operation::AudioGeneration;
40use crate::providers::internal::wire::classify_marker_keyed_frame;
41use crate::wire::{
42    Decoder, Descriptor, Encoded, Flow, Framing, Mode, Out, Wire, WireEvent, WireFrame,
43};
44
45/// `gemini-3.8-flash-tts`: the most expressive speech model.
46pub const GEMINI_3_8_FLASH_TTS: &str = "gemini-3.8-flash-tts";
47/// `gemini-3.8-flash-lite-tts`: the fast, low-cost speech model.
48pub const GEMINI_3_8_FLASH_LITE_TTS: &str = "gemini-3.8-flash-lite-tts";
49
50/// One turn of a [`Script`]: its text, read verbatim, and how to read it.
51#[derive(Clone, Debug, Default, PartialEq, Eq)]
52pub struct Turn {
53    /// The words to speak. Inline tags such as `<laugh>` mark momentary
54    /// sounds.
55    pub text: String,
56    /// The speaker, one of the script's speakers. Every turn of a
57    /// multi-speaker script names one.
58    pub speaker: Option<String>,
59    /// How the whole turn sounds, such as `"whispered urgently"`.
60    pub style: Option<String>,
61}
62
63impl Turn {
64    /// A turn that speaks `text`.
65    pub fn new(text: impl Into<String>) -> Self {
66        Self {
67            text: text.into(),
68            ..Self::default()
69        }
70    }
71
72    /// The speaker.
73    pub fn speaker(mut self, speaker: impl Into<String>) -> Self {
74        self.speaker = Some(speaker.into());
75        self
76    }
77
78    /// The style.
79    pub fn style(mut self, style: impl Into<String>) -> Self {
80        self.style = Some(style.into());
81        self
82    }
83}
84
85impl Serialize for Turn {
86    /// A text content item, with a `speech_metadata` annotation when the
87    /// turn names a speaker or a style.
88    fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
89        #[derive(Serialize)]
90        struct Metadata<'a> {
91            #[serde(rename = "type")]
92            kind: &'static str,
93            #[serde(skip_serializing_if = "Option::is_none")]
94            speaker: Option<&'a str>,
95            #[serde(skip_serializing_if = "Option::is_none")]
96            style: Option<&'a str>,
97        }
98        #[derive(Serialize)]
99        struct Text<'a> {
100            #[serde(rename = "type")]
101            kind: &'static str,
102            text: &'a str,
103            #[serde(skip_serializing_if = "Vec::is_empty")]
104            annotations: Vec<Metadata<'a>>,
105        }
106        let annotations = match (&self.speaker, &self.style) {
107            (None, None) => Vec::new(),
108            (speaker, style) => vec![Metadata {
109                kind: "speech_metadata",
110                speaker: speaker.as_deref(),
111                style: style.as_deref(),
112            }],
113        };
114        Text {
115            kind: "text",
116            text: &self.text,
117            annotations,
118        }
119        .serialize(serializer)
120    }
121}
122
123/// The encoding of the audio a reply carries.
124#[non_exhaustive]
125#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
126pub enum AudioEncoding {
127    /// `audio/wav`, the API's default: 16-bit PCM with a RIFF header.
128    #[serde(rename = "audio/wav")]
129    Wav,
130    /// `audio/l16`: headerless 16-bit signed little-endian PCM, which
131    /// joins by concatenation.
132    #[serde(rename = "audio/l16")]
133    L16,
134    /// `audio/mulaw`: 8-bit G.711 mu-law.
135    #[serde(rename = "audio/mulaw")]
136    Mulaw,
137    /// `audio/alaw`: 8-bit G.711 A-law.
138    #[serde(rename = "audio/alaw")]
139    Alaw,
140    /// `audio/mp3`.
141    #[serde(rename = "audio/mp3")]
142    Mp3,
143    /// `audio/ogg_opus`.
144    #[serde(rename = "audio/ogg_opus")]
145    OggOpus,
146}
147
148/// The audio format to answer in: the request's `response_format`. A field
149/// left unset takes the API's default.
150#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize)]
151pub struct AudioFormat {
152    /// `mime_type`.
153    #[serde(skip_serializing_if = "Option::is_none")]
154    pub mime_type: Option<AudioEncoding>,
155    /// `sample_rate`, in hertz.
156    #[serde(skip_serializing_if = "Option::is_none")]
157    pub sample_rate: Option<u32>,
158}
159
160impl AudioFormat {
161    /// Audio in `encoding`.
162    pub fn new(encoding: AudioEncoding) -> Self {
163        Self {
164            mime_type: Some(encoding),
165            sample_rate: None,
166        }
167    }
168
169    /// Headerless PCM ([`AudioEncoding::L16`]) at `sample_rate` hertz.
170    pub fn pcm(sample_rate: u32) -> Self {
171        Self::new(AudioEncoding::L16).sample_rate(sample_rate)
172    }
173
174    /// `sample_rate`, in hertz.
175    pub fn sample_rate(mut self, hertz: u32) -> Self {
176        self.sample_rate = Some(hertz);
177        self
178    }
179}
180
181/// `response_format` for audio.
182#[derive(Serialize)]
183struct ResponseFormat {
184    #[serde(rename = "type")]
185    kind: &'static str,
186    #[serde(flatten)]
187    format: AudioFormat,
188}
189
190impl From<AudioFormat> for ResponseFormat {
191    fn from(format: AudioFormat) -> Self {
192        Self {
193            kind: "audio",
194            format,
195        }
196    }
197}
198
199/// One `user_input` step of `turns`.
200fn input(turns: &[Turn]) -> Value {
201    json!([{ "type": "user_input", "content": turns }])
202}
203
204/// Turns spoken in one voice or one voice per speaker, in a format: a
205/// speech request [`build`](Self::build) turns into an
206/// [`AudioGenerationRequest`] for the [`Speech`] wire.
207#[derive(Clone, Debug, PartialEq, Eq)]
208pub struct Script {
209    speech: InteractionSpeechConfig,
210    turns: Vec<Turn>,
211    format: Option<AudioFormat>,
212}
213
214impl Script {
215    /// A script spoken in `voice`, such as `"Kore"` or a custom voice id.
216    pub fn voice(voice: impl Into<String>) -> Self {
217        Self::with(InteractionSpeechConfig::voice(InteractionSpeech::voice(
218            voice,
219        )))
220    }
221
222    /// A script whose turns each speaker of the `(speaker, voice)` pairs
223    /// speaks in its voice. The API takes up to two speakers.
224    pub fn speakers<S: Into<String>, V: Into<String>>(
225        speakers: impl IntoIterator<Item = (S, V)>,
226    ) -> Self {
227        Self::with(InteractionSpeechConfig::speakers(speakers))
228    }
229
230    /// A script spoken as `speech` configures.
231    pub fn with(speech: InteractionSpeechConfig) -> Self {
232        Self {
233            speech,
234            turns: Vec::new(),
235            format: None,
236        }
237    }
238
239    /// Add a turn.
240    pub fn turn(mut self, turn: Turn) -> Self {
241        self.turns.push(turn);
242        self
243    }
244
245    /// The audio format.
246    pub fn format(mut self, format: AudioFormat) -> Self {
247        self.format = Some(format);
248        self
249    }
250
251    /// The request that speaks this script. Its text is the turns' text,
252    /// one per line, and its `additional_params` carry the turns, the
253    /// voices and the format.
254    pub fn build(self) -> AudioGenerationRequest {
255        let text = self
256            .turns
257            .iter()
258            .map(|turn| turn.text.as_str())
259            .collect::<Vec<_>>()
260            .join("\n");
261        let voice = match &self.speech {
262            InteractionSpeechConfig::Voice([voice]) => voice.voice.clone().unwrap_or_default(),
263            InteractionSpeechConfig::Speakers { .. } => String::new(),
264        };
265        let mut params = json!({
266            "input": input(&self.turns),
267            "generation_config": { "speech_config": self.speech },
268        });
269        if let (Some(format), Some(fields)) = (self.format, params.as_object_mut()) {
270            fields.insert(
271                "response_format".to_owned(),
272                json!(ResponseFormat::from(format)),
273            );
274        }
275        crate::audio_generation::AudioGenerationRequestBuilder::new(text, voice)
276            .additional_params(params)
277            .build()
278    }
279}
280
281/// The audio content item a reply ends with: the last `audio` item of its
282/// model output steps.
283fn last_audio(reply: &Value) -> Option<&Value> {
284    reply
285        .arr("steps")
286        .iter()
287        .filter(|step| step.str("type") == Some("model_output"))
288        .flat_map(|step| step.arr("content"))
289        .rfind(|content| content.str("type") == Some("audio"))
290}
291
292/// The format of the audio a reply carries, as the API states it.
293#[derive(Clone, Debug, PartialEq, Eq)]
294pub struct SpeechFormat {
295    /// The media type, with its parameters, such as
296    /// `audio/l16; rate=24000; channels=1`.
297    pub mime_type: Option<String>,
298    /// The sample rate in hertz, which the API states for headerless
299    /// audio.
300    pub sample_rate: Option<u64>,
301    /// The number of channels, which the API states for headerless audio.
302    pub channels: Option<u64>,
303}
304
305/// The format of the audio in `response`, read from its reply document.
306pub fn format_of(response: &AudioGenerationResponse) -> Option<SpeechFormat> {
307    let audio = last_audio(&response.raw)?;
308    Some(SpeechFormat {
309        mime_type: audio.str("mime_type").map(str::to_owned),
310        sample_rate: audio.u64("sample_rate"),
311        channels: audio.u64("channels"),
312    })
313}
314
315/// The text-to-speech wire: `POST /v1beta/interactions` with audio output.
316///
317/// Both [`Mode`]s send a unary request and decode the whole interaction.
318#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)]
319pub struct Speech {
320    /// The provider this wire speaks to.
321    pub provider: super::GeminiConfig,
322    /// Name of the model, for example [`GEMINI_3_8_FLASH_TTS`].
323    pub model: String,
324}
325
326impl Speech {
327    /// The text-to-speech wire for `model`.
328    pub fn new(provider: super::GeminiConfig, model: impl Into<String>) -> Self {
329        Self {
330            provider,
331            model: model.into(),
332        }
333    }
334}
335
336/// The interaction `request` creates: its text as one turn in its voice,
337/// with `additional_params` merged over it key by key. An `input` there,
338/// such as a [`Script`]'s, takes the place of the text.
339fn create_request_body(model: &str, request: AudioGenerationRequest) -> Value {
340    let mut body = json!({
341        "model": model,
342        "input": input(&[Turn::new(request.text)]),
343        "response_format": ResponseFormat::from(AudioFormat::default()),
344    });
345    if let (false, Some(fields)) = (request.voice.is_empty(), body.as_object_mut()) {
346        let speech = InteractionSpeechConfig::voice(InteractionSpeech::voice(request.voice));
347        fields.insert(
348            "generation_config".to_owned(),
349            json!({ "speech_config": speech }),
350        );
351    }
352    match (&mut body, request.additional_params) {
353        (Value::Object(body), Some(Value::Object(params))) => {
354            crate::completion::options::deep_merge(body, params);
355        }
356        (body, Some(params)) => *body = params,
357        (_, None) => {}
358    }
359    body
360}
361
362impl Wire for Speech {
363    type Op = AudioGeneration;
364    type Payload = Encoded;
365    type Frame = WireFrame;
366    type Event = Value;
367    type Decoder<'id> = SpeechDecoder;
368    type Reassembler = crate::wire::document::Unreassembled;
369
370    fn describe(&self) -> Descriptor<'_> {
371        Descriptor::new(super::PROVIDER_NAME).model(self.model.as_str())
372    }
373
374    /// # Errors
375    ///
376    /// When the request asks for a speed other than 1.0: the API has none,
377    /// and a turn's style sets the pace instead.
378    fn encode(&self, request: AudioGenerationRequest, _mode: Mode) -> Result<Encoded, EncodeError> {
379        if request.speed != 1.0 {
380            return Err(EncodeError::request(
381                "Gemini speech takes no speed; ask for the pace in a turn's style",
382            ));
383        }
384        let body = serde_json::to_vec(&create_request_body(&self.model, request))?;
385        let request = http::Request::post(self.provider.interactions_uri("/v1beta/interactions"))
386            .header(http::header::CONTENT_TYPE, "application/json")
387            .header(
388                super::GeminiConfig::INTERACTIONS_KEY_HEADER,
389                self.provider.api_key.expose(),
390            )
391            .body(crate::wire::Body::Bytes(body))?;
392        // Gemini reports no transport request-id header.
393        Ok(Encoded::new(request, Framing::Whole))
394    }
395
396    fn decoder<'id>(&self) -> Self::Decoder<'id> {
397        SpeechDecoder
398    }
399}
400
401/// Decode the last audio item of an interaction, with its usage and
402/// identity. An interaction with no inline audio is a response error.
403#[derive(Default)]
404pub struct SpeechDecoder;
405
406impl<'id> Decoder<'id, AudioGeneration> for SpeechDecoder {
407    type Event = Value;
408
409    fn classify(&self, frame: WireFrame) -> WireEvent<Self::Event> {
410        classify_marker_keyed_frame(&frame.as_str(), &["steps", "status", "usage", "id"])
411    }
412
413    fn decode(
414        &mut self,
415        event: Self::Event,
416        out: Out<'id, AudioGeneration>,
417    ) -> Result<Flow, ProviderError> {
418        let data = last_audio(&event)
419            .and_then(|audio| audio.str("data"))
420            .ok_or_else(|| {
421                ProviderError::Response(format!(
422                    "the Gemini speech interaction ({}) carried no inline audio",
423                    event.str("status").unwrap_or("no status")
424                ))
425            })?;
426        let audio = BASE64_STANDARD.decode(data).map_err(|err| {
427            ProviderError::Response(format!("Gemini audio data was not valid base64: {err}"))
428        })?;
429        Ok(out.end(AudioGenerationResponse {
430            model: event.str("model").map(str::to_owned),
431            response_id: event.str("id").map(str::to_owned),
432            usage: event.get("usage").map(usage_of).unwrap_or_default(),
433            ..AudioGenerationResponse::new(audio)
434        }))
435    }
436}
437
438impl super::GeminiConfig {
439    /// The text-to-speech wire.
440    pub(crate) fn audio_generation(&self, model: impl Into<String>) -> Speech {
441        Speech::new(self.clone(), model)
442    }
443}
444
445#[cfg(test)]
446mod tests;
rig_core/providers/gemini/audio_generation.rs

rig_core/providers/gemini/
audio_generation.rs
