rig_core/providers/gemini/
transcription.rs1use crate::wire::Flow;
2use std::path::Path;
3
4use base64::{Engine, prelude::BASE64_STANDARD};
5use mime_guess;
6use serde_json::{Map, Value};
7
8use crate::error::EncodeError;
9use crate::error::ProviderError;
10use crate::{
11 completion::Usage,
12 operation::Transcription,
13 providers::gemini::completion::gemini_api_types::{
14 Blob, Content, GenerateContentRequest, GenerationConfig, Part, PartKind, Role,
15 visible_text_parts,
16 },
17 providers::internal::wire::classify_marker_keyed_frame,
18 transcription::{self, NormalizeTranscriptionResponse},
19 wire::{Body, Decoder, Descriptor, Encoded, Framing, Mode, Out, Wire, WireEvent, WireFrame},
20};
21
22use super::completion::gemini_api_types::GenerateContentResponse;
23
24const TRANSCRIPTION_PREAMBLE: &str =
25 "Translate the provided audio exactly. Do not add additional information.";
26
27fn transcription_body(
30 request: transcription::TranscriptionRequest,
31) -> Result<Vec<u8>, EncodeError> {
32 let additional_params = request
33 .additional_params
34 .unwrap_or_else(|| Value::Object(Map::new()));
35 let mut generation_config = serde_json::from_value::<GenerationConfig>(additional_params)?;
36
37 if let Some(temp) = request.temperature {
40 generation_config.temperature = Some(temp);
41 }
42
43 let system_instruction = Some(Content {
44 parts: vec![TRANSCRIPTION_PREAMBLE.into()],
45 role: Some(Role::Model),
46 });
47
48 let mime_type = mime_guess::from_path(Path::new(&request.filename))
50 .first()
51 .map_or_else(|| "audio/mpeg".to_string(), |mime| mime.to_string());
52
53 let body = GenerateContentRequest {
54 contents: vec![Content {
55 parts: vec![Part {
56 thought: Some(false),
57 thought_signature: None,
58 part: PartKind::InlineData(Blob {
59 mime_type,
60 data: BASE64_STANDARD.encode(request.data),
61 }),
62 additional_params: None,
63 }],
64 role: Some(Role::User),
65 }],
66 generation_config: Some(generation_config),
67 safety_settings: None,
68 tools: None,
69 tool_config: None,
70 system_instruction,
71 cached_content: None,
72 additional_params: None,
73 };
74
75 tracing::trace!(
76 target: "rig::transcription",
77 "Sending completion request to Gemini API {}",
78 serde_json::to_string_pretty(&body)?
79 );
80
81 Ok(serde_json::to_vec(&body)?)
82}
83
84#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)]
88pub struct Transcriptions {
89 pub provider: super::GeminiConfig,
91 pub model: String,
94}
95
96impl Transcriptions {
97 pub fn new(provider: super::GeminiConfig, model: impl Into<String>) -> Self {
99 Self {
100 provider,
101 model: model.into(),
102 }
103 }
104}
105
106impl Wire for Transcriptions {
107 type Op = Transcription;
108 type Payload = crate::wire::Encoded;
109 type Frame = crate::wire::WireFrame;
110 type Decoder<'id> = TranscriptionsDecoder;
111
112 fn describe(&self) -> Descriptor<'_> {
113 Descriptor::new(super::PROVIDER_NAME).model(self.model.as_str())
114 }
115
116 fn encode(
117 &self,
118 request: transcription::TranscriptionRequest,
119 _mode: Mode,
120 ) -> Result<Encoded, EncodeError> {
121 let body = transcription_body(request)?;
122 let request = http::Request::post(format!(
123 "{}/v1beta/models/{}:generateContent?key={}",
124 self.provider.base_url,
125 self.model,
126 self.provider.api_key.expose()
127 ))
128 .header(http::header::CONTENT_TYPE, "application/json")
129 .body(Body::Bytes(body))?;
130 Ok(Encoded::new(request, Framing::Whole))
132 }
133
134 fn decoder<'id>(&self) -> Self::Decoder<'id> {
135 TranscriptionsDecoder
136 }
137}
138
139#[derive(Default)]
142pub struct TranscriptionsDecoder;
143
144impl<'id> Decoder<'id, Transcription> for TranscriptionsDecoder {
145 type Event = GenerateContentResponse;
146
147 fn classify(&self, frame: WireFrame) -> WireEvent<Self::Event> {
148 classify_marker_keyed_frame(
149 &frame.as_str(),
150 &["candidates", "promptFeedback", "usageMetadata"],
151 )
152 }
153
154 fn decode(
155 &mut self,
156 event: Self::Event,
157 out: Out<'id, Transcription>,
158 ) -> Result<Flow, ProviderError> {
159 Ok(out.end(event.normalize()?))
160 }
161}
162
163impl NormalizeTranscriptionResponse for GenerateContentResponse {
164 fn normalize(self) -> Result<transcription::TranscriptionResponse, ProviderError> {
165 let candidate = self
166 .candidates
167 .first()
168 .ok_or_else(|| ProviderError::Response("No response candidates in response".into()))?;
169
170 let mut parts = candidate
171 .content
172 .as_ref()
173 .map(visible_text_parts)
174 .into_iter()
175 .flatten()
176 .peekable();
177 if parts.peek().is_none() {
178 return Err(ProviderError::Response(
179 "Response content contains no text".to_string(),
180 ));
181 }
182 let text = parts.collect::<String>();
183
184 let usage = self
185 .usage_metadata
186 .as_ref()
187 .map(Usage::from)
188 .unwrap_or_default();
189
190 Ok(transcription::TranscriptionResponse {
191 model: self.model_version,
192 response_id: Some(self.response_id),
193 usage,
194 ..transcription::TranscriptionResponse::new(text)
195 })
196 }
197}
198
199#[cfg(test)]
200mod tests;