rig_core/providers/minimax/
voice_clone.rs

1//! MiniMax voice cloning: upload a sample, clone it under a voice id, then
2//! speak with that id through the client's `audio_generation`. MiniMax
3//! deletes a cloned voice that goes unused for 7 days.
4//!
5//! ```no_run
6//! use rig_core::audio_generation::AudioGenerationRequestBuilder;
7//! use rig_core::providers::minimax::{self, voice_clone::{CloneAudio, VoiceClone}};
8//!
9//! # async fn run(sample: Vec<u8>) -> Result<(), Box<dyn std::error::Error>> {
10//! let client = minimax::from_env()?;
11//! let cloning = minimax::voice_cloning(&client);
12//! let file = cloning.upload(CloneAudio::sample("sample.mp3", sample)).await?;
13//! let voice = cloning.clone_voice(VoiceClone::new(file.file_id, "MyVoice001")).await?;
14//! let request = AudioGenerationRequestBuilder::new("Hello", voice.voice_id).build();
15//! let audio = client.audio_generation(minimax::SPEECH_2_8_HD).call(request).await?;
16//! # let _ = audio;
17//! # Ok(())
18//! # }
19//! ```
20
21use serde::{Deserialize, Serialize};
22use serde_json::Value;
23
24use crate::driver::Model;
25use crate::error::{EncodeError, ProviderError};
26use crate::operation::Whole;
27use crate::providers::openai::OpenAI;
28use crate::providers::openai::wire::OpenAIConfig;
29use crate::wire::{
30    Body, Call, Decoder, Descriptor, Encoded, Flow, Free, Mode, Operation, Out, Wire, WireEvent,
31    WireFrame,
32};
33
34const FILES_UPLOAD_PATH: &str = "/files/upload";
35const VOICE_CLONE_PATH: &str = "/voice_clone";
36
37/// The voice-cloning model of a MiniMax chat-completions `client`. Calls
38/// through a client of another provider fail before they are sent.
39pub fn voice_cloning(client: &OpenAI) -> Model<VoiceClones> {
40    client.model(VoiceClones::new(client.config().clone()))
41}
42
43/// What an uploaded audio file is for.
44#[derive(Clone, Copy, Debug, PartialEq, Eq)]
45pub enum AudioPurpose {
46    /// The voice to clone (`voice_clone`).
47    VoiceClone,
48    /// A short prompt clip that improves similarity (`prompt_audio`).
49    PromptAudio,
50}
51
52impl AudioPurpose {
53    /// The `purpose` form field.
54    pub fn as_str(self) -> &'static str {
55        match self {
56            Self::VoiceClone => "voice_clone",
57            Self::PromptAudio => "prompt_audio",
58        }
59    }
60}
61
62/// An mp3, m4a or wav file to upload, at most 20 MB. Other extensions are
63/// refused before the request is sent.
64#[derive(Clone, Debug, PartialEq, Eq)]
65pub struct CloneAudio {
66    /// What the file is for.
67    pub purpose: AudioPurpose,
68    /// The file name, whose extension names the format.
69    pub filename: String,
70    /// The file's bytes.
71    pub data: Vec<u8>,
72}
73
74impl CloneAudio {
75    /// The voice to clone: 10 seconds to 5 minutes of audio.
76    pub fn sample(filename: impl Into<String>, data: impl Into<Vec<u8>>) -> Self {
77        Self {
78            purpose: AudioPurpose::VoiceClone,
79            filename: filename.into(),
80            data: data.into(),
81        }
82    }
83
84    /// A prompt clip under 8 seconds, for [`VoiceClone::prompt`].
85    pub fn prompt(filename: impl Into<String>, data: impl Into<Vec<u8>>) -> Self {
86        Self {
87            purpose: AudioPurpose::PromptAudio,
88            filename: filename.into(),
89            data: data.into(),
90        }
91    }
92}
93
94/// A file MiniMax stored.
95#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)]
96pub struct UploadedFile {
97    /// The id a clone request names the file by.
98    pub file_id: i64,
99    /// Size in bytes.
100    #[serde(default)]
101    pub bytes: Option<u64>,
102    /// Unix time of the upload, in seconds.
103    #[serde(default)]
104    pub created_at: Option<u64>,
105    /// The uploaded file name.
106    #[serde(default)]
107    pub filename: Option<String>,
108    /// The `purpose` it was uploaded for.
109    #[serde(default)]
110    pub purpose: Option<String>,
111}
112
113/// A prompt clip and its transcript, which make the cloned voice closer and
114/// steadier.
115#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
116pub struct ClonePrompt {
117    /// The `file_id` of a [`CloneAudio::prompt`] upload.
118    pub prompt_audio: i64,
119    /// What the clip says, ending with punctuation.
120    pub prompt_text: String,
121}
122
123/// A request to clone an uploaded sample as `voice_id`.
124#[derive(Clone, Debug, PartialEq, Serialize)]
125pub struct VoiceClone {
126    /// The `file_id` of a [`CloneAudio::sample`] upload.
127    pub file_id: i64,
128    /// The new voice's id: 8 to 256 letters, digits, `-` and `_`, starting
129    /// with a letter, not ending with `-` or `_`, and not already in use.
130    pub voice_id: String,
131    /// An optional prompt clip.
132    #[serde(skip_serializing_if = "Option::is_none")]
133    pub clone_prompt: Option<ClonePrompt>,
134    /// Text the cloned voice reads for a preview, billed as speech.
135    #[serde(skip_serializing_if = "Option::is_none")]
136    pub text: Option<String>,
137    /// The speech model that reads the preview.
138    #[serde(skip_serializing_if = "Option::is_none")]
139    pub model: Option<String>,
140    /// Other request fields, such as `need_noise_reduction` or
141    /// `language_boost`, merged over the ones above.
142    #[serde(skip)]
143    pub additional_params: Option<Value>,
144}
145
146impl VoiceClone {
147    /// Clone the sample `file_id` as `voice_id`.
148    pub fn new(file_id: i64, voice_id: impl Into<String>) -> Self {
149        Self {
150            file_id,
151            voice_id: voice_id.into(),
152            clone_prompt: None,
153            text: None,
154            model: None,
155            additional_params: None,
156        }
157    }
158
159    /// Guide the clone with the uploaded prompt clip `prompt_audio`, which
160    /// says `prompt_text`.
161    pub fn prompt(mut self, prompt_audio: i64, prompt_text: impl Into<String>) -> Self {
162        self.clone_prompt = Some(ClonePrompt {
163            prompt_audio,
164            prompt_text: prompt_text.into(),
165        });
166        self
167    }
168
169    /// Have `model` read `text` in the new voice, returning a preview URL.
170    pub fn preview(mut self, text: impl Into<String>, model: impl Into<String>) -> Self {
171        self.text = Some(text.into());
172        self.model = Some(model.into());
173        self
174    }
175
176    /// The voice id the request creates: a `voice_id` in
177    /// [`Self::additional_params`] replaces [`Self::voice_id`].
178    pub fn created_voice_id(&self) -> &str {
179        self.additional_params
180            .as_ref()
181            .and_then(|params| params.get("voice_id"))
182            .and_then(Value::as_str)
183            .unwrap_or(&self.voice_id)
184    }
185
186    /// Merge `params` over earlier ones, key by key for JSON objects; `None`
187    /// clears them.
188    pub fn additional_params(mut self, params: impl Into<Option<Value>>) -> Self {
189        self.additional_params =
190            crate::json_utils::merge_params(self.additional_params.take(), params.into());
191        self
192    }
193}
194
195/// A cloned voice.
196#[derive(Clone, Debug, PartialEq)]
197pub struct ClonedVoice {
198    /// The id to speak with, as [`VoiceClone::created_voice_id`] names it.
199    pub voice_id: String,
200    /// A URL of the preview, when one was asked for.
201    pub demo_audio: Option<String>,
202    /// The preview's length, size and billed characters, verbatim.
203    pub extra_info: Option<Value>,
204}
205
206/// One voice-cloning request.
207#[derive(Clone, Debug, PartialEq)]
208pub enum VoiceCloneRequest {
209    /// `POST /v1/files/upload`.
210    Upload(CloneAudio),
211    /// `POST /v1/voice_clone`.
212    Clone(VoiceClone),
213}
214
215/// The reply to either request. An upload carries `file`; a clone carries
216/// the preview fields.
217#[derive(Clone, Debug, Default, Deserialize, Serialize)]
218pub struct VoiceCloneReply {
219    /// The stored file.
220    #[serde(default)]
221    pub file: Option<UploadedFile>,
222    /// A URL of the preview audio, empty when none was asked for.
223    #[serde(default)]
224    pub demo_audio: Option<String>,
225    /// The preview's length, size and billed characters.
226    #[serde(default, skip_serializing_if = "Option::is_none")]
227    pub extra_info: Option<Value>,
228}
229
230/// The voice-cloning operation.
231#[derive(Debug, Clone, Copy, PartialEq, Eq)]
232pub struct VoiceCloning;
233
234impl Operation for VoiceCloning {
235    type Request = VoiceCloneRequest;
236    type Event = std::convert::Infallible;
237    type End = VoiceCloneReply;
238    type Response = VoiceCloneReply;
239    type Fold = Whole<Self>;
240    type Emit = Free;
241
242    fn fold(_request: &Self::Request, _call: &mut Call<'_>) -> Self::Fold {
243        Whole::new()
244    }
245}
246
247/// The voice-cloning wire: file upload and voice clone on one MiniMax
248/// configuration.
249#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
250pub struct VoiceClones {
251    /// Which MiniMax endpoint, and the credential.
252    pub provider: OpenAIConfig,
253}
254
255impl VoiceClones {
256    /// The voice-cloning wire for `provider`.
257    pub fn new(provider: OpenAIConfig) -> Self {
258        Self { provider }
259    }
260}
261
262impl Wire for VoiceClones {
263    type Op = VoiceCloning;
264    type Event = Vec<u8>;
265    type Payload = Encoded;
266    type Frame = WireFrame;
267    type Decoder<'id> = VoiceClonesDecoder;
268    type Reassembler = crate::wire::document::Unreassembled;
269
270    fn describe(&self) -> Descriptor<'_> {
271        Descriptor::new(self.provider.dialect.name)
272    }
273
274    fn encode(&self, request: VoiceCloneRequest, _mode: Mode) -> Result<Encoded, EncodeError> {
275        if self.provider.dialect.name != super::PROVIDER_NAME {
276            return Err(EncodeError::request(format!(
277                "voice cloning is a MiniMax endpoint, not one {} serves",
278                self.provider.dialect.name
279            )));
280        }
281        let provider = &self.provider;
282        match request {
283            VoiceCloneRequest::Upload(audio) => {
284                use crate::http_client::MultipartForm;
285                use crate::http_client::multipart::Part;
286
287                let format = std::path::Path::new(&audio.filename)
288                    .extension()
289                    .and_then(std::ffi::OsStr::to_str)
290                    .map(str::to_ascii_lowercase);
291                if !matches!(format.as_deref(), Some("mp3" | "m4a" | "wav")) {
292                    return Err(EncodeError::request(format!(
293                        "MiniMax clones voices from mp3, m4a or wav files, not `{}`",
294                        audio.filename
295                    )));
296                }
297
298                let form = MultipartForm::new()
299                    .text("purpose", audio.purpose.as_str())
300                    .part(Part::bytes("file", audio.data).filename(audio.filename));
301                let builder = http::Request::post(provider.uri(FILES_UPLOAD_PATH, None));
302                crate::providers::openai::wire::encoded(provider, builder, Body::Multipart(form))
303            }
304            VoiceCloneRequest::Clone(mut clone) => {
305                let additional_params = clone.additional_params.take();
306                let mut body = serde_json::to_value(clone)?;
307                if let Some(additional_params) = additional_params {
308                    crate::json_utils::merge_inplace(&mut body, additional_params);
309                }
310                crate::providers::openai::wire::json_post(provider, VOICE_CLONE_PATH, None, &body)
311            }
312        }
313    }
314
315    fn decoder<'id>(&self) -> Self::Decoder<'id> {
316        VoiceClonesDecoder
317    }
318}
319
320/// Decodes either voice-cloning reply after checking its status envelope.
321pub struct VoiceClonesDecoder;
322
323impl<'id> Decoder<'id, VoiceCloning> for VoiceClonesDecoder {
324    type Event = Vec<u8>;
325
326    fn classify(&self, frame: WireFrame) -> WireEvent<Self::Event> {
327        WireEvent::Known(match frame {
328            WireFrame::Text(text) => text.into_bytes(),
329            WireFrame::Bytes(bytes) => bytes,
330        })
331    }
332
333    fn decode(
334        &mut self,
335        body: Vec<u8>,
336        out: Out<'id, VoiceCloning>,
337    ) -> Result<Flow, ProviderError> {
338        Ok(out.end(super::decode_reply(&body)?))
339    }
340}
rig_core/providers/minimax/voice_clone.rs

rig_core/providers/minimax/
voice_clone.rs
