Skip to main content

rig_core/providers/gemini/
transcription.rs

1use crate::wire::Flow;
2use std::path::Path;
3
4use base64::{Engine, prelude::BASE64_STANDARD};
5use mime_guess;
6use serde_json::{Map, Value};
7
8use crate::error::EncodeError;
9use crate::error::ProviderError;
10use crate::{
11    completion::Usage,
12    operation::Transcription,
13    providers::gemini::completion::gemini_api_types::{
14        Blob, Content, GenerateContentRequest, GenerationConfig, Part, PartKind, Role,
15        visible_text_parts,
16    },
17    providers::internal::wire::classify_marker_keyed_frame,
18    transcription::{self, NormalizeTranscriptionResponse},
19    wire::{Body, Decoder, Descriptor, Encoded, Framing, Mode, Out, Wire, WireEvent, WireFrame},
20};
21
22use super::completion::gemini_api_types::GenerateContentResponse;
23
24const TRANSCRIPTION_PREAMBLE: &str =
25    "Translate the provided audio exactly. Do not add additional information.";
26
27/// Encode audio as inline base64 with a transcription system instruction.
28/// Reject invalid generation parameters or JSON serialization failures.
29fn transcription_body(
30    request: transcription::TranscriptionRequest,
31) -> Result<Vec<u8>, EncodeError> {
32    let additional_params = request
33        .additional_params
34        .unwrap_or_else(|| Value::Object(Map::new()));
35    let mut generation_config = serde_json::from_value::<GenerationConfig>(additional_params)?;
36
37    // A temperature named on the request outranks one carried inside
38    // `additional_params`.
39    if let Some(temp) = request.temperature {
40        generation_config.temperature = Some(temp);
41    }
42
43    let system_instruction = Some(Content {
44        parts: vec![TRANSCRIPTION_PREAMBLE.into()],
45        role: Some(Role::Model),
46    });
47
48    // The request supplies no explicit MIME type, so infer it from the filename.
49    let mime_type = mime_guess::from_path(Path::new(&request.filename))
50        .first()
51        .map_or_else(|| "audio/mpeg".to_string(), |mime| mime.to_string());
52
53    let body = GenerateContentRequest {
54        contents: vec![Content {
55            parts: vec![Part {
56                thought: Some(false),
57                thought_signature: None,
58                part: PartKind::InlineData(Blob {
59                    mime_type,
60                    data: BASE64_STANDARD.encode(request.data),
61                }),
62                additional_params: None,
63            }],
64            role: Some(Role::User),
65        }],
66        generation_config: Some(generation_config),
67        safety_settings: None,
68        tools: None,
69        tool_config: None,
70        system_instruction,
71        cached_content: None,
72        additional_params: None,
73    };
74
75    tracing::trace!(
76        target: "rig::transcription",
77        "Sending completion request to Gemini API {}",
78        serde_json::to_string_pretty(&body)?
79    );
80
81    Ok(serde_json::to_vec(&body)?)
82}
83
84/// The transcription wire: `POST /v1beta/models/{model}:generateContent`.
85///
86/// Both [`Mode`]s send inline audio in JSON and read a whole response.
87#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)]
88pub struct Transcriptions {
89    /// The provider this wire speaks to.
90    pub provider: super::GeminiConfig,
91    /// The model transcribing, for example
92    /// [`GEMINI_2_0_FLASH`](super::completion::GEMINI_2_0_FLASH).
93    pub model: String,
94}
95
96impl Transcriptions {
97    /// The transcription wire for `model`.
98    pub fn new(provider: super::GeminiConfig, model: impl Into<String>) -> Self {
99        Self {
100            provider,
101            model: model.into(),
102        }
103    }
104}
105
106impl Wire for Transcriptions {
107    type Op = Transcription;
108    type Payload = crate::wire::Encoded;
109    type Frame = crate::wire::WireFrame;
110    type Decoder<'id> = TranscriptionsDecoder;
111
112    fn describe(&self) -> Descriptor<'_> {
113        Descriptor::new(super::PROVIDER_NAME).model(self.model.as_str())
114    }
115
116    fn encode(
117        &self,
118        request: transcription::TranscriptionRequest,
119        _mode: Mode,
120    ) -> Result<Encoded, EncodeError> {
121        let body = transcription_body(request)?;
122        let request = http::Request::post(format!(
123            "{}/v1beta/models/{}:generateContent?key={}",
124            self.provider.base_url,
125            self.model,
126            self.provider.api_key.expose()
127        ))
128        .header(http::header::CONTENT_TYPE, "application/json")
129        .body(Body::Bytes(body))?;
130        // Gemini reports no transport request-id header.
131        Ok(Encoded::new(request, Framing::Whole))
132    }
133
134    fn decoder<'id>(&self) -> Self::Decoder<'id> {
135        TranscriptionsDecoder
136    }
137}
138
139/// Decode visible text from the first `generateContent` candidate.
140/// Missing candidates or visible text produce response errors.
141#[derive(Default)]
142pub struct TranscriptionsDecoder;
143
144impl<'id> Decoder<'id, Transcription> for TranscriptionsDecoder {
145    type Event = GenerateContentResponse;
146
147    fn classify(&self, frame: WireFrame) -> WireEvent<Self::Event> {
148        classify_marker_keyed_frame(
149            &frame.as_str(),
150            &["candidates", "promptFeedback", "usageMetadata"],
151        )
152    }
153
154    fn decode(
155        &mut self,
156        event: Self::Event,
157        out: Out<'id, Transcription>,
158    ) -> Result<Flow, ProviderError> {
159        Ok(out.end(event.normalize()?))
160    }
161}
162
163impl NormalizeTranscriptionResponse for GenerateContentResponse {
164    fn normalize(self) -> Result<transcription::TranscriptionResponse, ProviderError> {
165        let candidate = self
166            .candidates
167            .first()
168            .ok_or_else(|| ProviderError::Response("No response candidates in response".into()))?;
169
170        let mut parts = candidate
171            .content
172            .as_ref()
173            .map(visible_text_parts)
174            .into_iter()
175            .flatten()
176            .peekable();
177        if parts.peek().is_none() {
178            return Err(ProviderError::Response(
179                "Response content contains no text".to_string(),
180            ));
181        }
182        let text = parts.collect::<String>();
183
184        let usage = self
185            .usage_metadata
186            .as_ref()
187            .map(Usage::from)
188            .unwrap_or_default();
189
190        Ok(transcription::TranscriptionResponse {
191            model: self.model_version,
192            response_id: Some(self.response_id),
193            usage,
194            ..transcription::TranscriptionResponse::new(text)
195        })
196    }
197}
198
199#[cfg(test)]
200mod tests;