1use super::{
2 INTERNAL_SAMPLERATE,
3 agc::AutomaticGainControl,
4 asr_processor::AsrProcessor,
5 denoiser::NoiseReducer,
6 processor::Processor,
7 track::{
8 Track, TrackPacketSender,
9 tts::{SynthesisHandle, TtsTrack},
10 },
11 vad::{VADOption, VadProcessor, VadType},
12};
13use crate::{
14 CallOption, EouOption,
15 event::EventSender,
16 media::TrackId,
17 synthesis::{
18 AliyunTtsClient, DeepegramTtsClient, SixtyDBTtsClient, SynthesisClient, SynthesisOption,
19 SynthesisType, TencentCloudTtsBasicClient, TencentCloudTtsClient,
20 },
21 transcription::{
22 AliyunAsrClientBuilder, DeepgramAsrClientBuilder, TencentCloudAsrClientBuilder,
23 TranscriptionClient, TranscriptionOption, TranscriptionType,
24 },
25};
26
27#[cfg(feature = "offline")]
28use crate::{synthesis::SupertonicTtsClient, transcription::SensevoiceAsrClientBuilder};
29
30use anyhow::Result;
31use std::{collections::HashMap, future::Future, pin::Pin, sync::Arc};
32use tokio::sync::mpsc;
33use tokio_util::sync::CancellationToken;
34use tracing::debug;
35#[cfg(feature = "ringback-detection")]
36use tracing::warn;
37
38pub type FnCreateVadProcessor = fn(
39 token: CancellationToken,
40 event_sender: EventSender,
41 option: VADOption,
42) -> Result<Box<dyn Processor>>;
43
44pub type FnCreateEouProcessor = fn(
45 token: CancellationToken,
46 event_sender: EventSender,
47 option: EouOption,
48) -> Result<Box<dyn Processor>>;
49
50pub type FnCreateAsrClient = Box<
51 dyn Fn(
52 TrackId,
53 CancellationToken,
54 TranscriptionOption,
55 EventSender,
56 ) -> Pin<Box<dyn Future<Output = Result<Box<dyn TranscriptionClient>>> + Send>>
57 + Send
58 + Sync,
59>;
60pub type FnCreateTtsClient =
61 fn(streaming: bool, option: &SynthesisOption) -> Result<Box<dyn SynthesisClient>>;
62
63pub type CreateProcessorsHook = Box<
65 dyn Fn(
66 Arc<StreamEngine>,
67 TrackId,
68 CancellationToken,
69 EventSender,
70 TrackPacketSender,
71 CallOption,
72 ) -> Pin<Box<dyn Future<Output = Result<Vec<Box<dyn Processor>>>> + Send>>
73 + Send
74 + Sync,
75>;
76
77pub struct StreamEngine {
78 vad_creators: HashMap<VadType, FnCreateVadProcessor>,
79 eou_creators: HashMap<String, FnCreateEouProcessor>,
80 asr_creators: HashMap<TranscriptionType, FnCreateAsrClient>,
81 tts_creators: HashMap<SynthesisType, FnCreateTtsClient>,
82 create_processors_hook: Arc<CreateProcessorsHook>,
83}
84
85impl Default for StreamEngine {
86 fn default() -> Self {
87 let mut engine = Self::new();
88 engine.register_vad(VadType::Silero, VadProcessor::create);
89 engine.register_vad(VadType::Other("nop".to_string()), VadProcessor::create_nop);
90
91 engine.register_asr(
92 TranscriptionType::TencentCloud,
93 Box::new(TencentCloudAsrClientBuilder::create),
94 );
95 engine.register_asr(
96 TranscriptionType::Aliyun,
97 Box::new(AliyunAsrClientBuilder::create),
98 );
99 engine.register_asr(
100 TranscriptionType::Deepgram,
101 Box::new(DeepgramAsrClientBuilder::create),
102 );
103
104 #[cfg(feature = "offline")]
105 engine.register_asr(
106 TranscriptionType::Sensevoice,
107 Box::new(SensevoiceAsrClientBuilder::create),
108 );
109
110 engine.register_tts(SynthesisType::Aliyun, AliyunTtsClient::create);
111 engine.register_tts(SynthesisType::TencentCloud, TencentCloudTtsClient::create);
112 engine.register_tts(
113 SynthesisType::Other("tencent_basic".to_string()),
114 TencentCloudTtsBasicClient::create,
115 );
116 engine.register_tts(SynthesisType::Deepgram, DeepegramTtsClient::create);
117 engine.register_tts(SynthesisType::SixtyDB, SixtyDBTtsClient::create);
118
119 #[cfg(feature = "offline")]
120 engine.register_tts(SynthesisType::Supertonic, SupertonicTtsClient::create);
121
122 engine
123 }
124}
125
126impl StreamEngine {
127 pub fn new() -> Self {
128 Self {
129 vad_creators: HashMap::new(),
130 asr_creators: HashMap::new(),
131 tts_creators: HashMap::new(),
132 eou_creators: HashMap::new(),
133 create_processors_hook: Arc::new(Box::new(Self::default_create_procesors_hook)),
134 }
135 }
136
137 pub fn register_vad(&mut self, vad_type: VadType, creator: FnCreateVadProcessor) -> &mut Self {
138 self.vad_creators.insert(vad_type, creator);
139 self
140 }
141
142 pub fn register_eou(&mut self, name: String, creator: FnCreateEouProcessor) -> &mut Self {
143 self.eou_creators.insert(name, creator);
144 self
145 }
146
147 pub fn register_asr(
148 &mut self,
149 asr_type: TranscriptionType,
150 creator: FnCreateAsrClient,
151 ) -> &mut Self {
152 self.asr_creators.insert(asr_type, creator);
153 self
154 }
155
156 pub fn register_tts(
157 &mut self,
158 tts_type: SynthesisType,
159 creator: FnCreateTtsClient,
160 ) -> &mut Self {
161 self.tts_creators.insert(tts_type, creator);
162 self
163 }
164
165 pub fn create_vad_processor(
166 &self,
167 token: CancellationToken,
168 event_sender: EventSender,
169 option: VADOption,
170 ) -> Result<Box<dyn Processor>> {
171 let creator = self.vad_creators.get(&option.r#type);
172 if let Some(creator) = creator {
173 creator(token, event_sender, option)
174 } else {
175 Err(anyhow::anyhow!("VAD type not found: {}", option.r#type))
176 }
177 }
178 pub fn create_eou_processor(
179 &self,
180 token: CancellationToken,
181 event_sender: EventSender,
182 option: EouOption,
183 ) -> Result<Box<dyn Processor>> {
184 let creator = self
185 .eou_creators
186 .get(&option.r#type.clone().unwrap_or_default());
187 if let Some(creator) = creator {
188 creator(token, event_sender, option)
189 } else {
190 Err(anyhow::anyhow!("EOU type not found: {:?}", option.r#type))
191 }
192 }
193
194 pub async fn create_asr_processor(
195 &self,
196 track_id: TrackId,
197 cancel_token: CancellationToken,
198 option: TranscriptionOption,
199 event_sender: EventSender,
200 ) -> Result<Box<dyn Processor>> {
201 let asr_client = match option.provider {
202 Some(ref provider) => {
203 let creator = self.asr_creators.get(&provider);
204 if let Some(creator) = creator {
205 creator(track_id, cancel_token, option, event_sender).await?
206 } else {
207 return Err(anyhow::anyhow!("ASR type not found: {}", provider));
208 }
209 }
210 None => return Err(anyhow::anyhow!("ASR type not found: {:?}", option.provider)),
211 };
212 Ok(Box::new(AsrProcessor { asr_client }))
213 }
214
215 pub async fn create_tts_client(
216 &self,
217 streaming: bool,
218 tts_option: &SynthesisOption,
219 ) -> Result<Box<dyn SynthesisClient>> {
220 match tts_option.provider {
221 Some(ref provider) => {
222 let creator = self.tts_creators.get(&provider);
223 if let Some(creator) = creator {
224 creator(streaming, tts_option)
225 } else {
226 Err(anyhow::anyhow!("TTS type not found: {}", provider))
227 }
228 }
229 None => Err(anyhow::anyhow!(
230 "TTS type not found: {:?}",
231 tts_option.provider
232 )),
233 }
234 }
235
236 pub async fn create_processors(
237 engine: Arc<StreamEngine>,
238 track_id: TrackId,
239 cancel_token: CancellationToken,
240 event_sender: EventSender,
241 packet_sender: TrackPacketSender,
242 option: &CallOption,
243 ) -> Result<Vec<Box<dyn Processor>>> {
244 (engine.clone().create_processors_hook)(
245 engine,
246 track_id,
247 cancel_token,
248 event_sender,
249 packet_sender,
250 option.clone(),
251 )
252 .await
253 }
254
255 pub async fn create_tts_track(
256 engine: Arc<StreamEngine>,
257 cancel_token: CancellationToken,
258 session_id: String,
259 track_id: TrackId,
260 ssrc: u32,
261 play_id: Option<String>,
262 streaming: bool,
263 tts_option: &SynthesisOption,
264 auto_hangup: Option<bool>,
265 ) -> Result<(SynthesisHandle, Box<dyn Track>)> {
266 let (tx, rx) = mpsc::unbounded_channel();
267 let new_handle = SynthesisHandle::new(tx, play_id.clone(), ssrc);
268 let tts_client = engine.create_tts_client(streaming, tts_option).await?;
269 let sample_rate = tts_option.samplerate.unwrap_or(16000) as u32;
270 let tts_track = TtsTrack::new(track_id, session_id, streaming, play_id, rx, tts_client)
271 .with_ssrc(ssrc)
272 .with_auto_hangup(auto_hangup)
273 .with_sample_rate(sample_rate)
274 .with_cancel_token(cancel_token);
275 Ok((new_handle, Box::new(tts_track) as Box<dyn Track>))
276 }
277
278 pub fn with_processor_hook(&mut self, hook_fn: CreateProcessorsHook) -> &mut Self {
279 self.create_processors_hook = Arc::new(Box::new(hook_fn));
280 self
281 }
282
283 pub fn default_create_procesors_hook(
284 engine: Arc<StreamEngine>,
285 track_id: TrackId,
286 cancel_token: CancellationToken,
287 event_sender: EventSender,
288 packet_sender: TrackPacketSender,
289 option: CallOption,
290 ) -> Pin<Box<dyn Future<Output = Result<Vec<Box<dyn Processor>>>> + Send>> {
291 Box::pin(async move {
292 let mut processors = vec![];
293 debug!(%track_id, "Creating processors for track");
294
295 if let Some(realtime_option) = option.realtime {
296 debug!(%track_id, "Adding RealtimeProcessor");
297 let realtime_processor = crate::media::realtime_processor::RealtimeProcessor::new(
298 track_id.clone(),
299 cancel_token.child_token(),
300 event_sender.clone(),
301 packet_sender.clone(),
302 realtime_option,
303 )?;
304 processors.push(Box::new(realtime_processor) as Box<dyn Processor>);
305 return Ok(processors);
308 }
309
310 match option.denoise {
311 Some(true) => {
312 debug!(%track_id, "Adding NoiseReducer processor");
313 let noise_reducer = NoiseReducer::new(INTERNAL_SAMPLERATE as usize);
314 processors.push(Box::new(noise_reducer) as Box<dyn Processor>);
315 }
316 _ => {}
317 }
318 match option.vad {
319 Some(mut option) => {
320 debug!(%track_id, "Adding VadProcessor processor type={:?}", option.r#type);
321 option.samplerate = INTERNAL_SAMPLERATE;
322 let vad_processor: Box<dyn Processor + 'static> = engine.create_vad_processor(
323 cancel_token.child_token(),
324 event_sender.clone(),
325 option.to_owned(),
326 )?;
327 processors.push(vad_processor);
328 }
329 None => {}
330 }
331 if let Some(agc_option) = option.agc.clone() {
332 debug!(%track_id, "Adding AutomaticGainControl processor");
333 let agc = AutomaticGainControl::new(INTERNAL_SAMPLERATE as u32, agc_option)?;
334 processors.push(Box::new(agc) as Box<dyn Processor>);
335 }
336 match option.asr {
337 Some(mut option) => {
338 debug!(%track_id, "Adding AsrProcessor processor provider={:?}", option.provider);
339 option.samplerate = Some(INTERNAL_SAMPLERATE);
340 let asr_processor = engine
341 .create_asr_processor(
342 track_id.clone(),
343 cancel_token.child_token(),
344 option.to_owned(),
345 event_sender.clone(),
346 )
347 .await?;
348 processors.push(asr_processor);
349 }
350 None => {}
351 }
352 match option.eou {
353 Some(ref option) => {
354 let eou_processor = engine.create_eou_processor(
355 cancel_token.child_token(),
356 event_sender.clone(),
357 option.to_owned(),
358 )?;
359 processors.push(eou_processor);
360 }
361 None => {}
362 }
363 match option.inactivity_timeout {
364 Some(timeout_secs) if timeout_secs > 0 => {
365 let inactivity_processor = crate::media::inactivity::InactivityProcessor::new(
366 track_id.clone(),
367 std::time::Duration::from_secs(timeout_secs),
368 event_sender.clone(),
369 cancel_token.child_token(),
370 );
371 processors.push(Box::new(inactivity_processor) as Box<dyn Processor>);
372 }
373 _ => {}
374 }
375
376 #[cfg(feature = "ringback-detection")]
377 if let Some(ref ringback_opt) = option.ringback_detection {
378 if ringback_opt.enabled.unwrap_or(false) {
379 debug!(%track_id, "Adding RingbackDetectionProcessor");
380 match
381 crate::media::ringback_detection::processor::RingbackDetectionProcessor::new(
382 track_id.clone(),
383 cancel_token.child_token(),
384 event_sender.clone(),
385 ringback_opt.clone(),
386 None,
387 ) {
388 Ok(p) => processors.push(Box::new(p) as Box<dyn Processor>),
389 Err(e) => warn!(%track_id, "Failed to create RingbackDetectionProcessor: {}", e),
390 }
391 }
392 }
393
394 Ok(processors)
395 })
396 }
397}