rig_experimental/providers/
elevenlabs.rs

1//! The module for Eleven Labs.
2//!
3
4use std::fmt::{self, Debug};
5
6use rig::{
7    audio_generation::{self, AudioGenerationError},
8    client::{AudioGenerationClient, ProviderClient},
9    impl_conversion_traits,
10};
11
12use bytes::Bytes;
13use serde::{Deserialize, Serialize};
14
15#[derive(Clone)]
16pub struct Client {
17    base_url: String,
18    api_key: String,
19    http_client: reqwest::Client,
20}
21
22impl Debug for Client {
23    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
24        f.debug_struct("Client")
25            .field("base_url", &self.base_url)
26            .field("http_client", &self.http_client)
27            .field("api_key", b"<REDACTED>")
28            .finish()
29    }
30}
31
32const ELEVENLABS_API_BASE_URL: &str = "https://api.elevenlabs.io/v1";
33
34impl Client {
35    pub fn new(api_key: &str) -> Self {
36        Self::from_url(api_key, ELEVENLABS_API_BASE_URL)
37    }
38
39    fn from_url(api_key: &str, url: &str) -> Self {
40        Self {
41            base_url: url.to_string(),
42            api_key: api_key.to_string(),
43            http_client: reqwest::Client::builder()
44                .build()
45                .expect("The ElevenLabs client should always build correctly"),
46        }
47    }
48
49    pub fn with_custom_client(mut self, client: reqwest::Client) -> Self {
50        self.http_client = client;
51        self
52    }
53
54    async fn post<T>(&self, path: &str, body: &T) -> Result<reqwest::Response, reqwest::Error>
55    where
56        T: serde::Serialize,
57    {
58        let mut url = self.base_url.clone();
59        url.push_str(path);
60
61        self.http_client
62            .post(&url)
63            .header("xi-api-key", &self.api_key)
64            .json(body)
65            .send()
66            .await
67    }
68}
69
70impl ProviderClient for Client {
71    fn from_env() -> Self
72    where
73        Self: Sized,
74    {
75        let api_key = std::env::var("ELEVENLABS_API_KEY")
76            .expect("expected ELEVENLABS_API_KEY to exist as an environment variable");
77
78        Self::new(&api_key)
79    }
80}
81
82impl AudioGenerationClient for Client {
83    type AudioGenerationModel = AudioGenerationModel;
84    /// Create an audio generation model with the given name.
85    ///
86    /// # Example
87    /// ```
88    /// use rig::providers::openai::{Client, self};
89    ///
90    /// // Initialize the OpenAI client
91    /// let openai = Client::new("your-open-ai-api-key");
92    ///
93    /// let gpt4 = openai.audio_generation_model(openai::TTS_1);
94    /// ```
95    fn audio_generation_model(&self, model: &str) -> Self::AudioGenerationModel {
96        AudioGenerationModel::new(self.clone(), model)
97    }
98}
99
100impl_conversion_traits!(AsCompletion, AsEmbeddings, AsTranscription, AsImageGeneration for Client);
101
102#[derive(Clone, Debug)]
103pub struct AudioGenerationModel {
104    client: Client,
105    model: String,
106}
107
108impl AudioGenerationModel {
109    fn new(client: Client, model: &str) -> Self {
110        Self {
111            client,
112            model: model.to_owned(),
113        }
114    }
115}
116
117#[derive(Clone, Debug, Serialize, Deserialize)]
118pub struct AudioGenerationRequest {
119    /// The text that will be used to generate speech
120    pub text: String,
121    /// The audio generation model that will be used to generate speech
122    pub model_id: String,
123    /// The ID of the voice that will be used to generate speech
124    pub voice_id: String,
125    #[serde(flatten)]
126    pub params: ElevenLabsParams,
127}
128
129#[derive(Clone, Debug, Serialize, Deserialize, Default)]
130pub struct ElevenLabsParams {
131    /// The audio output format
132    pub output_format: AudioOutputFormat,
133    /// The language code (ISO 936-1) that the model will use
134    #[serde(skip_serializing_if = "Option::is_none")]
135    pub language_code: Option<String>,
136    /// Voice settings to be used by the model.
137    #[serde(skip_serializing_if = "Option::is_none")]
138    pub voice_settings: Option<VoiceSettings>,
139    /// Use a pre-generated seed to help deterministically create audio samples. However, determinism is not guaranteed.
140    #[serde(skip_serializing_if = "Option::is_none")]
141    pub seed: Option<u64>,
142    /// The text that came before the text of the current request. Can be used to improve speech continuity.
143    #[serde(skip_serializing_if = "Option::is_none")]
144    pub previous_text: Option<String>,
145    /// The text that came after the text of the current request. Can be used to improve speech continuity.
146    #[serde(skip_serializing_if = "Option::is_none")]
147    pub next_text: Option<String>,
148    /// A list of requests that were generated before this generation. Can be used to improve speech continuity.
149    #[serde(skip_serializing_if = "Option::is_none")]
150    pub previous_request_ids: Option<String>,
151    /// A list of samples that come after this generation. Can be used to improve speech continuity.
152    #[serde(skip_serializing_if = "Option::is_none")]
153    pub next_request_ids: Option<Vec<String>>,
154    /// Whether or not to apply text normalization. If an option is not supplied (or you've set it to Auto), the Elevenlabs API will decide whether or not to apply it.
155    #[serde(skip_serializing_if = "Option::is_none")]
156    pub apply_text_normalization: Option<ApplyTextNormalization>,
157    /// Controls language text normalization - helps with proper pronounciation of text in some supported languages. This parameter can heavily increase the latency of the request and is so far only supported in Japanese.
158    #[serde(skip_serializing_if = "Option::is_none")]
159    pub apply_language_text_normalization: Option<bool>,
160}
161
162impl ElevenLabsParams {
163    pub fn into_json(self) -> Result<serde_json::Value, serde_json::Error> {
164        serde_json::to_value(self)
165    }
166}
167
168impl TryFrom<(String, audio_generation::AudioGenerationRequest)> for AudioGenerationRequest {
169    type Error = AudioGenerationError;
170    fn try_from(
171        (model_name, req): (String, audio_generation::AudioGenerationRequest),
172    ) -> Result<Self, Self::Error> {
173        let audio_generation::AudioGenerationRequest {
174            text,
175            voice,
176            speed,
177            additional_params,
178        } = req;
179
180        let Some(params) = additional_params else {
181            return Err(AudioGenerationError::ProviderError("You need to use additional parameters to be able to insert required variables for this provider!".into()));
182        };
183
184        let mut params: ElevenLabsParams = serde_json::from_value(params)?;
185        let voice_settings = {
186            let mut settings = params.voice_settings.unwrap_or_default();
187
188            settings.speed = Some(speed as f64);
189            settings
190        };
191        params.voice_settings = Some(voice_settings);
192
193        // let params = params.voice_settings.map(|x| x.speed = Some(speed as f64));
194
195        Ok(Self {
196            text,
197            voice_id: voice,
198            model_id: model_name,
199            params,
200        })
201    }
202}
203
204impl TryFrom<(&str, audio_generation::AudioGenerationRequest)> for AudioGenerationRequest {
205    type Error = AudioGenerationError;
206    fn try_from(
207        (model_name, req): (&str, audio_generation::AudioGenerationRequest),
208    ) -> Result<Self, Self::Error> {
209        let model_name = model_name.to_string();
210        Self::try_from((model_name, req))
211    }
212}
213
214#[derive(Clone, Debug, Serialize, Deserialize, Default)]
215pub enum ApplyTextNormalization {
216    #[default]
217    Auto,
218    On,
219    Off,
220}
221
222#[derive(Clone, Debug, Serialize, Deserialize, Default)]
223pub struct VoiceSettings {
224    pub stability: Option<f64>,
225    pub use_speaker_boost: Option<bool>,
226    pub similarity_boost: Option<f64>,
227    pub style: Option<f64>,
228    pub speed: Option<f64>,
229}
230
231#[derive(Clone, Debug, Serialize, Deserialize, Default)]
232pub enum AudioOutputFormat {
233    /// MP3 with 22.05kHz sample at 32kbs
234    #[serde(rename = "mp3_22050_32")]
235    #[default]
236    Mp3_22050_32,
237
238    /// MP3 with 44.1kHz sample at 32kbs
239    #[serde(rename = "mp3_44100_32")]
240    Mp3_44100_32,
241
242    /// MP3 with 44.1kHz sample at 64kbs
243    #[serde(rename = "mp3_44100_64")]
244    Mp3_44100_64,
245
246    /// MP3 with 44.1kHz sample at 96kbs
247    #[serde(rename = "mp3_44100_96")]
248    Mp3_44100_96,
249
250    /// MP3 with 44.1kHz sample at 128kbs
251    #[serde(rename = "mp3_44100_128")]
252    Mp3_44100_128,
253
254    /// MP3 with 44.1kHz sample at 192kbs
255    #[serde(rename = "mp3_44100_192")]
256    Mp3_44100_192,
257
258    /// PCM with 8kHz sample
259    #[serde(rename = "pcm_8000")]
260    Pcm8000,
261
262    /// PCM with 16kHz sample
263    #[serde(rename = "pcm_16000")]
264    Pcm16000,
265
266    /// PCM with 22.05kHz sample
267    #[serde(rename = "pcm_22050")]
268    Pcm22050,
269
270    /// PCM with 44.1kHz sample
271    #[serde(rename = "pcm_44100")]
272    Pcm44100,
273
274    /// PCM with 48kHz sample
275    #[serde(rename = "pcm_48000")]
276    Pcm48000,
277
278    /// ULaw with 8kHz sample
279    #[serde(rename = "ulaw_8000")]
280    Ulaw8000,
281
282    /// ALaw with 8kHz sample
283    #[serde(rename = "alaw_8000")]
284    Alaw8000,
285
286    /// Opus with 48kHz sample at 32kbs
287    #[serde(rename = "opus_48000_32")]
288    Opus4800032,
289
290    /// Opus with 48kHz sample at 64kbs
291    #[serde(rename = "opus_48000_64")]
292    Opus4800064,
293
294    /// Opus with 48kHz sample at 96kbs
295    #[serde(rename = "opus_48000_96")]
296    Opus4800096,
297
298    /// Opus with 48kHz sample at 128kbs
299    #[serde(rename = "opus_48000_128")]
300    Opus48000128,
301
302    /// Opus with 48kHz sample at 192kbs
303    #[serde(rename = "opus_48000_192")]
304    Opus48000192,
305}
306
307impl audio_generation::AudioGenerationModel for AudioGenerationModel {
308    type Response = Bytes;
309
310    async fn audio_generation(
311        &self,
312        request: audio_generation::AudioGenerationRequest,
313    ) -> Result<
314        audio_generation::AudioGenerationResponse<Self::Response>,
315        audio_generation::AudioGenerationError,
316    > {
317        let req: AudioGenerationRequest =
318            AudioGenerationRequest::try_from((self.model.as_ref(), request))?;
319        let url = format!(
320            "/text-to-speech/{voice_id}?output_format={output}",
321            voice_id = req.voice_id,
322            output =
323                serde_json::to_string(&req.params.output_format).expect("This should never fail")
324        );
325
326        let response = self.client.post(&url, &req).await.unwrap().bytes().await?;
327
328        Ok(audio_generation::AudioGenerationResponse {
329            audio: response.to_vec(),
330            response,
331        })
332    }
333}
334
335/// The ElevenLabs eleven_multilingual_v2 model.
336pub const ELEVEN_MULTILINGUAL_V2: &str = "eleven_multilingual_v2";
337
338/// The ElevenLabs eleven_v3 model.
339pub const ELEVEN_V3: &str = "eleven_v3";
340
341/// The ElevenLabs eleven_flash_v2 model.
342pub const ELEVEN_FLASH_V2: &str = "eleven_flash_v2";
343
344/// The ElevenLabs eleven_turbo_v2_5 model.
345pub const ELEVEN_TURBO_V2_5: &str = "eleven_turbo_v2_5";
346
347/// The ElevenLabs scribe_v1 model for usage with transcription.
348pub const SCRIBE_V1: &str = "scribe_v1";
rig_experimental/providers/elevenlabs.rs

rig_experimental/providers/
elevenlabs.rs
