aither_gemini/
audio.rs

1use aither_core::audio::{AudioGenerator, AudioTranscriber, Data as AudioData};
2use futures_lite::StreamExt;
3
4use crate::{
5    client::call_generate,
6    config::{Gemini, GeminiConfig},
7    error::GeminiError,
8    types::{
9        GeminiContent, GenerateContentRequest, GenerationConfig, Part, PrebuiltVoiceConfig,
10        SpeechConfig, ThinkingConfig, VoiceConfig,
11    },
12};
13
14impl AudioGenerator for Gemini {
15    type Error = GeminiError;
16
17    fn generate(
18        &self,
19        prompt: &str,
20    ) -> impl futures_core::Stream<Item = Result<AudioData, Self::Error>> + Send {
21        let cfg = self.config();
22        let text = prompt.to_owned();
23        futures_lite::stream::iter(vec![synthesize_audio(cfg, text)]).then(|fut| fut)
24    }
25}
26
27impl AudioTranscriber for Gemini {
28    type Error = GeminiError;
29
30    fn transcribe(
31        &self,
32        audio: &[u8],
33    ) -> impl futures_core::Stream<Item = Result<String, Self::Error>> + Send {
34        let cfg = self.config();
35        let payload = audio.to_vec();
36        futures_lite::stream::iter(vec![transcribe_audio(cfg, payload)]).then(|fut| fut)
37    }
38}
39
40async fn synthesize_audio(cfg: &GeminiConfig, text: String) -> Result<Vec<u8>, GeminiError> {
41    let model = cfg.tts_model.clone().ok_or_else(|| {
42        GeminiError::Api("audio generation is disabled for this Gemini backend".into())
43    })?;
44    let request = GenerateContentRequest {
45        system_instruction: None,
46        contents: vec![GeminiContent::text("user", text)],
47        generation_config: Some(GenerationConfig {
48            response_modalities: Some(vec!["AUDIO".into()]),
49            speech_config: Some(SpeechConfig {
50                voice_config: Some(VoiceConfig {
51                    prebuilt: PrebuiltVoiceConfig {
52                        voice_name: cfg.tts_voice.clone(),
53                    },
54                }),
55            }),
56            thinking_config: Some(ThinkingConfig {
57                include_thoughts: Some(false),
58                thinking_level: None,
59            }),
60            ..GenerationConfig::default()
61        }),
62        tools: Vec::new(),
63        tool_config: None,
64        safety_settings: Vec::new(),
65        cached_content: None,
66    };
67    let response = call_generate(cfg, &model, request).await?;
68    if let Some(candidate) = response.primary_candidate()
69        && let Some(content) = &candidate.content
70    {
71        for part in &content.parts {
72            if let Some(inline) = &part.inline_data {
73                let bytes = inline.decode()?;
74                return Ok(bytes);
75            }
76        }
77    }
78    Err(GeminiError::Api(
79        "Gemini did not return audio data in the response".into(),
80    ))
81}
82
83async fn transcribe_audio(cfg: &GeminiConfig, audio: Vec<u8>) -> Result<String, GeminiError> {
84    let mut parts = vec![Part::inline_audio(audio)];
85    parts.push(Part::text(
86        "Transcribe the audio verbatim in the original language.",
87    ));
88    let request = GenerateContentRequest {
89        system_instruction: None,
90        contents: vec![GeminiContent::with_parts("user", parts)],
91        generation_config: Some(GenerationConfig {
92            thinking_config: Some(ThinkingConfig {
93                include_thoughts: Some(false),
94                thinking_level: None,
95            }),
96            ..GenerationConfig::default()
97        }),
98        tools: Vec::new(),
99        tool_config: None,
100        safety_settings: Vec::new(),
101        cached_content: None,
102    };
103    let response = call_generate(cfg, &cfg.text_model, request).await?;
104    if let Some(candidate) = response.primary_candidate()
105        && let Some(content) = &candidate.content
106    {
107        let text = content.text_chunks().join("");
108        return Ok(text);
109    }
110    Err(GeminiError::Api(
111        "Gemini did not return transcription text".into(),
112    ))
113}
aither_gemini/audio.rs

aither_gemini/
audio.rs
