Skip to main content

active_call/synthesis/
mod.rs

1use anyhow::Result;
2use async_trait::async_trait;
3use bytes::Bytes;
4use futures::stream::BoxStream;
5use serde::{Deserialize, Serialize};
6use std::collections::HashMap;
7use tokio::sync::mpsc;
8
9mod aliyun;
10mod deepgram;
11mod sixtydb;
12mod tencent_cloud;
13mod tencent_cloud_basic;
14
15#[cfg(feature = "offline")]
16mod supertonic;
17
18pub use aliyun::AliyunTtsClient;
19pub use deepgram::DeepegramTtsClient;
20pub use sixtydb::SixtyDBTtsClient;
21pub use tencent_cloud::TencentCloudTtsClient;
22pub use tencent_cloud_basic::TencentCloudTtsBasicClient;
23
24#[cfg(feature = "offline")]
25pub use supertonic::SupertonicTtsClient;
26
27#[derive(Clone, Default)]
28pub struct SynthesisCommand {
29    pub text: String,
30    pub speaker: Option<String>,
31    pub play_id: Option<String>,
32    pub streaming: bool,
33    pub end_of_stream: bool,
34    pub option: SynthesisOption,
35    pub base64: bool,
36    pub cache_key: Option<String>,
37    pub auto_hangup: Option<bool>,
38}
39pub type SynthesisCommandSender = mpsc::UnboundedSender<SynthesisCommand>;
40pub type SynthesisCommandReceiver = mpsc::UnboundedReceiver<SynthesisCommand>;
41
42#[derive(Debug, Clone, Serialize, Hash, Eq, PartialEq)]
43pub enum SynthesisType {
44    #[serde(rename = "tencent")]
45    TencentCloud,
46    #[serde(rename = "aliyun")]
47    Aliyun,
48    #[serde(rename = "deepgram")]
49    Deepgram,
50    #[serde(rename = "60db")]
51    SixtyDB,
52    #[cfg(feature = "offline")]
53    #[serde(rename = "supertonic")]
54    Supertonic,
55    #[serde(rename = "other")]
56    Other(String),
57}
58
59impl std::fmt::Display for SynthesisType {
60    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
61        match self {
62            SynthesisType::TencentCloud => write!(f, "tencent"),
63            SynthesisType::Aliyun => write!(f, "aliyun"),
64            SynthesisType::Deepgram => write!(f, "deepgram"),
65            SynthesisType::SixtyDB => write!(f, "60db"),
66            #[cfg(feature = "offline")]
67            SynthesisType::Supertonic => write!(f, "supertonic"),
68            SynthesisType::Other(provider) => write!(f, "{}", provider),
69        }
70    }
71}
72
73impl<'de> Deserialize<'de> for SynthesisType {
74    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
75    where
76        D: serde::Deserializer<'de>,
77    {
78        let value = String::deserialize(deserializer)?;
79        match value.as_str() {
80            "tencent" => Ok(SynthesisType::TencentCloud),
81            "aliyun" => Ok(SynthesisType::Aliyun),
82            "deepgram" => Ok(SynthesisType::Deepgram),
83            "60db" => Ok(SynthesisType::SixtyDB),
84            #[cfg(feature = "offline")]
85            "supertonic" => Ok(SynthesisType::Supertonic),
86            _ => Ok(SynthesisType::Other(value)),
87        }
88    }
89}
90
91#[cfg(test)]
92mod tests;
93#[derive(Debug, Clone, Deserialize, Serialize)]
94#[serde(rename_all = "camelCase")]
95#[serde(default)]
96pub struct SynthesisOption {
97    pub samplerate: Option<i32>,
98    pub provider: Option<SynthesisType>,
99    pub speed: Option<f32>,
100    pub app_id: Option<String>,
101    pub secret_id: Option<String>,
102    #[serde(alias = "apiKey")]
103    pub secret_key: Option<String>,
104    pub volume: Option<i32>,
105    pub speaker: Option<String>,
106    pub codec: Option<String>,
107    pub subtitle: Option<bool>,
108    #[serde(alias = "voice")]
109    pub model: Option<String>,
110    pub language: Option<String>,
111    /// emotion: neutral、sad、happy、angry、fear、news、story、radio、poetry、
112    /// call、sajiao、disgusted、amaze、peaceful、exciting、aojiao、jieshuo
113    pub emotion: Option<String>,
114    pub endpoint: Option<String>,
115    pub extra: Option<HashMap<String, String>>,
116    pub max_concurrent_tasks: Option<usize>,
117    pub session_id: Option<String>,
118}
119
120impl SynthesisOption {
121    pub fn merge_with(&self, option: Option<SynthesisOption>) -> Self {
122        if let Some(other) = option {
123            Self {
124                samplerate: other.samplerate.or(self.samplerate),
125                provider: other.provider.or(self.provider.clone()),
126                speed: other.speed.or(self.speed),
127                app_id: other.app_id.or(self.app_id.clone()),
128                secret_id: other.secret_id.or(self.secret_id.clone()),
129                secret_key: other.secret_key.or(self.secret_key.clone()),
130                volume: other.volume.or(self.volume),
131                speaker: other.speaker.or(self.speaker.clone()),
132                codec: other.codec.or(self.codec.clone()),
133                subtitle: other.subtitle.or(self.subtitle),
134                model: other.model.or(self.model.clone()),
135                language: other.language.or(self.language.clone()),
136                emotion: other.emotion.or(self.emotion.clone()),
137                endpoint: other.endpoint.or(self.endpoint.clone()),
138                extra: other.extra.or(self.extra.clone()),
139                max_concurrent_tasks: other.max_concurrent_tasks.or(self.max_concurrent_tasks),
140                session_id: other.session_id.or(self.session_id.clone()),
141            }
142        } else {
143            self.clone()
144        }
145    }
146}
147
148#[derive(Debug)]
149pub enum SynthesisEvent {
150    /// Raw audio data chunk
151    AudioChunk(Bytes),
152    /// Progress information including completion status
153    Subtitles(Vec<Subtitle>),
154    Finished,
155}
156
157#[derive(Debug, Clone)]
158pub struct Subtitle {
159    pub text: String,
160    pub begin_time: u32,
161    pub end_time: u32,
162    pub begin_index: u32,
163    pub end_index: u32,
164}
165
166impl Subtitle {
167    pub fn new(
168        text: String,
169        begin_time: u32,
170        end_time: u32,
171        begin_index: u32,
172        end_index: u32,
173    ) -> Self {
174        Self {
175            text,
176            begin_time,
177            end_time,
178            begin_index,
179            end_index,
180        }
181    }
182}
183
184// calculate audio duration from bytes size and sample rate
185pub fn bytes_size_to_duration(bytes: usize, sample_rate: u32) -> u32 {
186    (500.0 * bytes as f32 / sample_rate as f32) as u32
187}
188
189#[async_trait]
190pub trait SynthesisClient: Send {
191    // provider of the synthesis client.
192    fn provider(&self) -> SynthesisType;
193
194    // connect to the synthesis service.
195    // (cmd_seq, result), return the cmd_seq that passed from `synthesize`
196    async fn start(
197        &mut self,
198    ) -> Result<BoxStream<'static, (Option<usize>, Result<SynthesisEvent>)>>;
199
200    // send text to the synthesis service.
201    // `cmd_seq` and `option` are used for non streaming mode
202    // for streaming mode, `cmd_seq` and `option` are None
203    async fn synthesize(
204        &mut self,
205        text: &str,
206        cmd_seq: Option<usize>,
207        option: Option<SynthesisOption>,
208    ) -> Result<()>;
209
210    async fn stop(&mut self) -> Result<()>;
211}
212
213impl Default for SynthesisOption {
214    fn default() -> Self {
215        Self {
216            samplerate: Some(16000),
217            provider: None,
218            speed: Some(1.0),
219            app_id: None,
220            secret_id: None,
221            secret_key: None,
222            volume: Some(5), // 0-10
223            speaker: None,
224            codec: Some("pcm".to_string()),
225            subtitle: None,
226            model: None,
227            language: None,
228            emotion: None,
229            endpoint: None,
230            extra: None,
231            max_concurrent_tasks: None,
232            session_id: None,
233        }
234    }
235}
236
237impl SynthesisOption {
238    pub fn check_default(&mut self) {
239        if let Some(provider) = &self.provider {
240            match provider.to_string().as_str() {
241                "tencent" | "tencent_basic" => {
242                    if self.app_id.is_none() {
243                        self.app_id = std::env::var("TENCENT_APPID").ok();
244                    }
245                    if self.secret_id.is_none() {
246                        self.secret_id = std::env::var("TENCENT_SECRET_ID").ok();
247                    }
248                    if self.secret_key.is_none() {
249                        self.secret_key = std::env::var("TENCENT_SECRET_KEY").ok();
250                    }
251                }
252                "voiceapi" => {
253                    // Set the endpoint from environment variable if not already set
254                    if self.endpoint.is_none() {
255                        self.endpoint = std::env::var("VOICEAPI_ENDPOINT")
256                            .ok()
257                            .or_else(|| Some("http://localhost:8000".to_string()));
258                    }
259                    // Set speaker ID from environment variable if not already set
260                    if self.speaker.is_none() {
261                        self.speaker = std::env::var("VOICEAPI_SPEAKER_ID")
262                            .ok()
263                            .or_else(|| Some("0".to_string()));
264                    }
265                }
266                "aliyun" => {
267                    if self.secret_key.is_none() {
268                        self.secret_key = std::env::var("DASHSCOPE_API_KEY").ok();
269                    }
270                }
271                "60db" => {
272                    if self.secret_key.is_none() {
273                        self.secret_key = std::env::var("SIXTYDB_API_KEY").ok();
274                    }
275                    if self.speaker.is_none() {
276                        self.speaker = std::env::var("SIXTYDB_VOICE_ID").ok();
277                    }
278                }
279                "deepgram" => {
280                    if self.secret_key.is_none() {
281                        self.secret_key = std::env::var("DEEPGRAM_API_KEY").ok();
282                    }
283                }
284                _ => {}
285            }
286        }
287    }
288}