Skip to main content

rig_core/providers/minimax/
voice_clone.rs

1//! MiniMax voice cloning: upload a sample, clone it under a voice id, then
2//! speak with that id through the client's `audio_generation`. MiniMax
3//! deletes a cloned voice that goes unused for 7 days.
4//!
5//! ```no_run
6//! use rig_core::audio_generation::AudioGenerationRequestBuilder;
7//! use rig_core::providers::minimax::{self, voice_clone::{CloneAudio, VoiceClone}};
8//!
9//! # async fn run(sample: Vec<u8>) -> Result<(), Box<dyn std::error::Error>> {
10//! let client = minimax::from_env()?;
11//! let cloning = minimax::voice_cloning(&client);
12//! let file = cloning.upload(CloneAudio::sample("sample.mp3", sample)).await?;
13//! let voice = cloning.clone_voice(VoiceClone::new(file.file_id, "MyVoice001")).await?;
14//! let request = AudioGenerationRequestBuilder::new("Hello", voice.voice_id).build();
15//! let audio = client.audio_generation(minimax::SPEECH_2_8_HD).call(request).await?;
16//! # let _ = audio;
17//! # Ok(())
18//! # }
19//! ```
20
21use serde::{Deserialize, Serialize};
22use serde_json::Value;
23
24use crate::driver::Model;
25use crate::error::{EncodeError, ProviderError};
26use crate::operation::Whole;
27use crate::providers::openai::OpenAI;
28use crate::providers::openai::wire::OpenAIConfig;
29use crate::wire::{
30    Body, Call, Decoder, Descriptor, Encoded, Flow, Free, Mode, Operation, Out, Wire, WireEvent,
31    WireFrame,
32};
33
34const FILES_UPLOAD_PATH: &str = "/files/upload";
35const VOICE_CLONE_PATH: &str = "/voice_clone";
36
37/// The voice-cloning model of a MiniMax chat-completions `client`. Calls
38/// through a client of another provider fail before they are sent.
39pub fn voice_cloning(client: &OpenAI) -> Model<VoiceClones> {
40    client.model(VoiceClones::new(client.config().clone()))
41}
42
43/// What an uploaded audio file is for.
44#[derive(Clone, Copy, Debug, PartialEq, Eq)]
45pub enum AudioPurpose {
46    /// The voice to clone (`voice_clone`).
47    VoiceClone,
48    /// A short prompt clip that improves similarity (`prompt_audio`).
49    PromptAudio,
50}
51
52impl AudioPurpose {
53    /// The `purpose` form field.
54    pub fn as_str(self) -> &'static str {
55        match self {
56            Self::VoiceClone => "voice_clone",
57            Self::PromptAudio => "prompt_audio",
58        }
59    }
60}
61
62/// An mp3, m4a or wav file to upload, at most 20 MB. Other extensions are
63/// refused before the request is sent.
64#[derive(Clone, Debug, PartialEq, Eq)]
65pub struct CloneAudio {
66    /// What the file is for.
67    pub purpose: AudioPurpose,
68    /// The file name, whose extension names the format.
69    pub filename: String,
70    /// The file's bytes.
71    pub data: Vec<u8>,
72}
73
74impl CloneAudio {
75    /// The voice to clone: 10 seconds to 5 minutes of audio.
76    pub fn sample(filename: impl Into<String>, data: impl Into<Vec<u8>>) -> Self {
77        Self {
78            purpose: AudioPurpose::VoiceClone,
79            filename: filename.into(),
80            data: data.into(),
81        }
82    }
83
84    /// A prompt clip under 8 seconds, for [`VoiceClone::prompt`].
85    pub fn prompt(filename: impl Into<String>, data: impl Into<Vec<u8>>) -> Self {
86        Self {
87            purpose: AudioPurpose::PromptAudio,
88            filename: filename.into(),
89            data: data.into(),
90        }
91    }
92}
93
94/// A file MiniMax stored.
95#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
96pub struct UploadedFile {
97    /// The id a clone request names the file by.
98    pub file_id: i64,
99    /// Size in bytes.
100    pub bytes: Option<u64>,
101    /// Unix time of the upload, in seconds.
102    pub created_at: Option<u64>,
103    /// The uploaded file name.
104    pub filename: Option<String>,
105    /// The `purpose` it was uploaded for.
106    pub purpose: Option<String>,
107}
108
109/// A prompt clip and its transcript, which make the cloned voice closer and
110/// steadier.
111#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
112pub struct ClonePrompt {
113    /// The `file_id` of a [`CloneAudio::prompt`] upload.
114    pub prompt_audio: i64,
115    /// What the clip says, ending with punctuation.
116    pub prompt_text: String,
117}
118
119/// A request to clone an uploaded sample as `voice_id`.
120#[derive(Clone, Debug, PartialEq, Serialize)]
121pub struct VoiceClone {
122    /// The `file_id` of a [`CloneAudio::sample`] upload.
123    pub file_id: i64,
124    /// The new voice's id: 8 to 256 letters, digits, `-` and `_`, starting
125    /// with a letter, not ending with `-` or `_`, and not already in use.
126    pub voice_id: String,
127    /// An optional prompt clip.
128    #[serde(skip_serializing_if = "Option::is_none")]
129    pub clone_prompt: Option<ClonePrompt>,
130    /// Text the cloned voice reads for a preview, billed as speech.
131    #[serde(skip_serializing_if = "Option::is_none")]
132    pub text: Option<String>,
133    /// The speech model that reads the preview.
134    #[serde(skip_serializing_if = "Option::is_none")]
135    pub model: Option<String>,
136    /// Other request fields, such as `need_noise_reduction` or
137    /// `language_boost`, merged over the ones above.
138    #[serde(skip)]
139    pub additional_params: Option<Value>,
140}
141
142impl VoiceClone {
143    /// Clone the sample `file_id` as `voice_id`.
144    pub fn new(file_id: i64, voice_id: impl Into<String>) -> Self {
145        Self {
146            file_id,
147            voice_id: voice_id.into(),
148            clone_prompt: None,
149            text: None,
150            model: None,
151            additional_params: None,
152        }
153    }
154
155    /// Guide the clone with the uploaded prompt clip `prompt_audio`, which
156    /// says `prompt_text`.
157    pub fn prompt(mut self, prompt_audio: i64, prompt_text: impl Into<String>) -> Self {
158        self.clone_prompt = Some(ClonePrompt {
159            prompt_audio,
160            prompt_text: prompt_text.into(),
161        });
162        self
163    }
164
165    /// Have `model` read `text` in the new voice, returning a preview URL.
166    pub fn preview(mut self, text: impl Into<String>, model: impl Into<String>) -> Self {
167        self.text = Some(text.into());
168        self.model = Some(model.into());
169        self
170    }
171
172    /// The voice id the request creates: a `voice_id` in
173    /// [`Self::additional_params`] replaces [`Self::voice_id`].
174    pub fn created_voice_id(&self) -> &str {
175        self.additional_params
176            .as_ref()
177            .and_then(|params| params.get("voice_id"))
178            .and_then(Value::as_str)
179            .unwrap_or(&self.voice_id)
180    }
181
182    /// Merge `params` over earlier ones, key by key for JSON objects; `None`
183    /// clears them.
184    pub fn additional_params(mut self, params: impl Into<Option<Value>>) -> Self {
185        self.additional_params =
186            crate::json_utils::merge_params(self.additional_params.take(), params.into());
187        self
188    }
189}
190
191/// A cloned voice.
192#[derive(Clone, Debug, PartialEq)]
193pub struct ClonedVoice {
194    /// The id to speak with, as [`VoiceClone::created_voice_id`] names it.
195    pub voice_id: String,
196    /// A URL of the preview, when one was asked for.
197    pub demo_audio: Option<String>,
198    /// The preview's length, size and billed characters, verbatim.
199    pub extra_info: Option<Value>,
200}
201
202/// One voice-cloning request.
203#[derive(Clone, Debug, PartialEq)]
204pub enum VoiceCloneRequest {
205    /// `POST /v1/files/upload`.
206    Upload(CloneAudio),
207    /// `POST /v1/voice_clone`.
208    Clone(VoiceClone),
209}
210
211/// The reply to either request. An upload carries `file`; a clone carries
212/// the preview fields.
213#[derive(Clone, Debug, Default, Deserialize, Serialize)]
214pub struct VoiceCloneReply {
215    /// The stored file.
216    pub file: Option<UploadedFile>,
217    /// A URL of the preview audio, empty when none was asked for.
218    pub demo_audio: Option<String>,
219    /// The preview's length, size and billed characters.
220    #[serde(default, skip_serializing_if = "Option::is_none")]
221    pub extra_info: Option<Value>,
222}
223
224/// The voice-cloning operation.
225#[derive(Debug, Clone, Copy, PartialEq, Eq)]
226pub struct VoiceCloning;
227
228impl Operation for VoiceCloning {
229    type Request = VoiceCloneRequest;
230    type Event = std::convert::Infallible;
231    type End = VoiceCloneReply;
232    type Response = VoiceCloneReply;
233    type Fold = Whole<Self>;
234    type Emit = Free;
235
236    fn fold(_request: &Self::Request, _call: &mut Call<'_>) -> Self::Fold {
237        Whole::new()
238    }
239}
240
241/// The voice-cloning wire: file upload and voice clone on one MiniMax
242/// configuration.
243#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
244pub struct VoiceClones {
245    /// Which MiniMax endpoint, and the credential.
246    pub provider: OpenAIConfig,
247}
248
249impl VoiceClones {
250    /// The voice-cloning wire for `provider`.
251    pub fn new(provider: OpenAIConfig) -> Self {
252        Self { provider }
253    }
254}
255
256impl Wire for VoiceClones {
257    type Op = VoiceCloning;
258    type Event = Vec<u8>;
259    type Payload = Encoded;
260    type Frame = WireFrame;
261    type Decoder<'id> = VoiceClonesDecoder;
262    type Reassembler = crate::wire::document::Unreassembled;
263
264    fn describe(&self) -> Descriptor<'_> {
265        Descriptor::new(self.provider.dialect.name)
266    }
267
268    fn encode(&self, request: VoiceCloneRequest, _mode: Mode) -> Result<Encoded, EncodeError> {
269        if self.provider.dialect.name != super::PROVIDER_NAME {
270            return Err(EncodeError::request(format!(
271                "voice cloning is a MiniMax endpoint, not one {} serves",
272                self.provider.dialect.name
273            )));
274        }
275        let provider = &self.provider;
276        match request {
277            VoiceCloneRequest::Upload(audio) => {
278                use crate::http_client::MultipartForm;
279                use crate::http_client::multipart::Part;
280
281                let format = std::path::Path::new(&audio.filename)
282                    .extension()
283                    .and_then(std::ffi::OsStr::to_str)
284                    .map(str::to_ascii_lowercase);
285                if !matches!(format.as_deref(), Some("mp3" | "m4a" | "wav")) {
286                    return Err(EncodeError::request(format!(
287                        "MiniMax clones voices from mp3, m4a or wav files, not `{}`",
288                        audio.filename
289                    )));
290                }
291
292                let form = MultipartForm::new()
293                    .text("purpose", audio.purpose.as_str())
294                    .part(Part::bytes("file", audio.data).filename(audio.filename));
295                let builder = http::Request::post(provider.uri(FILES_UPLOAD_PATH, None));
296                crate::providers::openai::wire::encoded(provider, builder, Body::Multipart(form))
297            }
298            VoiceCloneRequest::Clone(mut clone) => {
299                let additional_params = clone.additional_params.take();
300                let mut body = serde_json::to_value(clone)?;
301                if let Some(additional_params) = additional_params {
302                    body = crate::json_utils::merge(body, additional_params);
303                }
304                crate::providers::openai::wire::json_post(provider, VOICE_CLONE_PATH, None, &body)
305            }
306        }
307    }
308
309    fn decoder<'id>(&self) -> Self::Decoder<'id> {
310        VoiceClonesDecoder
311    }
312}
313
314/// Decodes either voice-cloning reply after checking its status envelope.
315pub struct VoiceClonesDecoder;
316
317impl<'id> Decoder<'id, VoiceCloning> for VoiceClonesDecoder {
318    type Event = Vec<u8>;
319
320    fn classify(&self, frame: WireFrame) -> WireEvent<Self::Event> {
321        WireEvent::Known(frame.into_bytes())
322    }
323
324    fn decode(
325        &mut self,
326        body: Vec<u8>,
327        out: Out<'id, VoiceCloning>,
328    ) -> Result<Flow, ProviderError> {
329        Ok(out.end(super::decode_reply(&body)?))
330    }
331}