active_call/synthesis/
mod.rs

1use anyhow::Result;
2use async_trait::async_trait;
3use bytes::Bytes;
4use futures::stream::BoxStream;
5use serde::{Deserialize, Serialize};
6use std::collections::HashMap;
7use tokio::sync::mpsc;
8
9mod aliyun;
10mod deepgram;
11mod tencent_cloud;
12mod tencent_cloud_basic;
13
14#[cfg(feature = "offline")]
15mod supertonic;
16
17pub use aliyun::AliyunTtsClient;
18pub use deepgram::DeepegramTtsClient;
19pub use tencent_cloud::TencentCloudTtsClient;
20pub use tencent_cloud_basic::TencentCloudTtsBasicClient;
21
22#[cfg(feature = "offline")]
23pub use supertonic::SupertonicTtsClient;
24
25#[derive(Clone, Default)]
26pub struct SynthesisCommand {
27    pub text: String,
28    pub speaker: Option<String>,
29    pub play_id: Option<String>,
30    pub streaming: bool,
31    pub end_of_stream: bool,
32    pub option: SynthesisOption,
33    pub base64: bool,
34    pub cache_key: Option<String>,
35    pub auto_hangup: Option<bool>,
36}
37pub type SynthesisCommandSender = mpsc::UnboundedSender<SynthesisCommand>;
38pub type SynthesisCommandReceiver = mpsc::UnboundedReceiver<SynthesisCommand>;
39
40#[derive(Debug, Clone, Serialize, Hash, Eq, PartialEq)]
41pub enum SynthesisType {
42    #[serde(rename = "tencent")]
43    TencentCloud,
44    #[serde(rename = "aliyun")]
45    Aliyun,
46    #[serde(rename = "deepgram")]
47    Deepgram,
48    #[cfg(feature = "offline")]
49    #[serde(rename = "supertonic")]
50    Supertonic,
51    #[serde(rename = "other")]
52    Other(String),
53}
54
55impl std::fmt::Display for SynthesisType {
56    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
57        match self {
58            SynthesisType::TencentCloud => write!(f, "tencent"),
59            SynthesisType::Aliyun => write!(f, "aliyun"),
60            SynthesisType::Deepgram => write!(f, "deepgram"),
61            #[cfg(feature = "offline")]
62            SynthesisType::Supertonic => write!(f, "supertonic"),
63            SynthesisType::Other(provider) => write!(f, "{}", provider),
64        }
65    }
66}
67
68impl<'de> Deserialize<'de> for SynthesisType {
69    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
70    where
71        D: serde::Deserializer<'de>,
72    {
73        let value = String::deserialize(deserializer)?;
74        match value.as_str() {
75            "tencent" => Ok(SynthesisType::TencentCloud),
76            "aliyun" => Ok(SynthesisType::Aliyun),
77            "deepgram" => Ok(SynthesisType::Deepgram),
78            #[cfg(feature = "offline")]
79            "supertonic" => Ok(SynthesisType::Supertonic),
80            _ => Ok(SynthesisType::Other(value)),
81        }
82    }
83}
84
85#[cfg(test)]
86mod tests;
87#[derive(Debug, Clone, Deserialize, Serialize)]
88#[serde(rename_all = "camelCase")]
89#[serde(default)]
90pub struct SynthesisOption {
91    pub samplerate: Option<i32>,
92    pub provider: Option<SynthesisType>,
93    pub speed: Option<f32>,
94    pub app_id: Option<String>,
95    pub secret_id: Option<String>,
96    #[serde(alias = "apiKey")]
97    pub secret_key: Option<String>,
98    pub volume: Option<i32>,
99    pub speaker: Option<String>,
100    pub codec: Option<String>,
101    pub subtitle: Option<bool>,
102    #[serde(alias = "voice")]
103    pub model: Option<String>,
104    pub language: Option<String>,
105    /// emotion: neutral、sad、happy、angry、fear、news、story、radio、poetry、
106    /// call、sajiao、disgusted、amaze、peaceful、exciting、aojiao、jieshuo
107    pub emotion: Option<String>,
108    pub endpoint: Option<String>,
109    pub extra: Option<HashMap<String, String>>,
110    pub max_concurrent_tasks: Option<usize>,
111    pub session_id: Option<String>,
112}
113
114impl SynthesisOption {
115    pub fn merge_with(&self, option: Option<SynthesisOption>) -> Self {
116        if let Some(other) = option {
117            Self {
118                samplerate: other.samplerate.or(self.samplerate),
119                provider: other.provider.or(self.provider.clone()),
120                speed: other.speed.or(self.speed),
121                app_id: other.app_id.or(self.app_id.clone()),
122                secret_id: other.secret_id.or(self.secret_id.clone()),
123                secret_key: other.secret_key.or(self.secret_key.clone()),
124                volume: other.volume.or(self.volume),
125                speaker: other.speaker.or(self.speaker.clone()),
126                codec: other.codec.or(self.codec.clone()),
127                subtitle: other.subtitle.or(self.subtitle),
128                model: other.model.or(self.model.clone()),
129                language: other.language.or(self.language.clone()),
130                emotion: other.emotion.or(self.emotion.clone()),
131                endpoint: other.endpoint.or(self.endpoint.clone()),
132                extra: other.extra.or(self.extra.clone()),
133                max_concurrent_tasks: other.max_concurrent_tasks.or(self.max_concurrent_tasks),
134                session_id: other.session_id.or(self.session_id.clone()),
135            }
136        } else {
137            self.clone()
138        }
139    }
140}
141
142#[derive(Debug)]
143pub enum SynthesisEvent {
144    /// Raw audio data chunk
145    AudioChunk(Bytes),
146    /// Progress information including completion status
147    Subtitles(Vec<Subtitle>),
148    Finished,
149}
150
151#[derive(Debug, Clone)]
152pub struct Subtitle {
153    pub text: String,
154    pub begin_time: u32,
155    pub end_time: u32,
156    pub begin_index: u32,
157    pub end_index: u32,
158}
159
160impl Subtitle {
161    pub fn new(
162        text: String,
163        begin_time: u32,
164        end_time: u32,
165        begin_index: u32,
166        end_index: u32,
167    ) -> Self {
168        Self {
169            text,
170            begin_time,
171            end_time,
172            begin_index,
173            end_index,
174        }
175    }
176}
177
178// calculate audio duration from bytes size and sample rate
179pub fn bytes_size_to_duration(bytes: usize, sample_rate: u32) -> u32 {
180    (500.0 * bytes as f32 / sample_rate as f32) as u32
181}
182
183#[async_trait]
184pub trait SynthesisClient: Send {
185    // provider of the synthesis client.
186    fn provider(&self) -> SynthesisType;
187
188    // connect to the synthesis service.
189    // (cmd_seq, result), return the cmd_seq that passed from `synthesize`
190    async fn start(
191        &mut self,
192    ) -> Result<BoxStream<'static, (Option<usize>, Result<SynthesisEvent>)>>;
193
194    // send text to the synthesis service.
195    // `cmd_seq` and `option` are used for non streaming mode
196    // for streaming mode, `cmd_seq` and `option` are None
197    async fn synthesize(
198        &mut self,
199        text: &str,
200        cmd_seq: Option<usize>,
201        option: Option<SynthesisOption>,
202    ) -> Result<()>;
203
204    async fn stop(&mut self) -> Result<()>;
205}
206
207impl Default for SynthesisOption {
208    fn default() -> Self {
209        Self {
210            samplerate: Some(16000),
211            provider: None,
212            speed: Some(1.0),
213            app_id: None,
214            secret_id: None,
215            secret_key: None,
216            volume: Some(5), // 0-10
217            speaker: None,
218            codec: Some("pcm".to_string()),
219            subtitle: None,
220            model: None,
221            language: None,
222            emotion: None,
223            endpoint: None,
224            extra: None,
225            max_concurrent_tasks: None,
226            session_id: None,
227        }
228    }
229}
230
231impl SynthesisOption {
232    pub fn check_default(&mut self) {
233        if let Some(provider) = &self.provider {
234            match provider.to_string().as_str() {
235                "tencent" | "tencent_basic" => {
236                    if self.app_id.is_none() {
237                        self.app_id = std::env::var("TENCENT_APPID").ok();
238                    }
239                    if self.secret_id.is_none() {
240                        self.secret_id = std::env::var("TENCENT_SECRET_ID").ok();
241                    }
242                    if self.secret_key.is_none() {
243                        self.secret_key = std::env::var("TENCENT_SECRET_KEY").ok();
244                    }
245                }
246                "voiceapi" => {
247                    // Set the endpoint from environment variable if not already set
248                    if self.endpoint.is_none() {
249                        self.endpoint = std::env::var("VOICEAPI_ENDPOINT")
250                            .ok()
251                            .or_else(|| Some("http://localhost:8000".to_string()));
252                    }
253                    // Set speaker ID from environment variable if not already set
254                    if self.speaker.is_none() {
255                        self.speaker = std::env::var("VOICEAPI_SPEAKER_ID")
256                            .ok()
257                            .or_else(|| Some("0".to_string()));
258                    }
259                }
260                "aliyun" => {
261                    if self.secret_key.is_none() {
262                        self.secret_key = std::env::var("DASHSCOPE_API_KEY").ok();
263                    }
264                }
265                "deepgram" => {
266                    if self.secret_key.is_none() {
267                        self.secret_key = std::env::var("DEEPGRAM_API_KEY").ok();
268                    }
269                }
270                _ => {}
271            }
272        }
273    }
274}
active_call/synthesis/mod.rs

active_call/synthesis/
mod.rs
