1use super::{
2 INTERNAL_SAMPLERATE,
3 agc::AutomaticGainControl,
4 asr_processor::AsrProcessor,
5 denoiser::{NoiseReducer, spawn_answer_watcher},
6 processor::Processor,
7 track::{
8 Track, TrackPacketSender,
9 tts::{SynthesisHandle, TtsTrack},
10 },
11 vad::{VADOption, VadProcessor, VadType},
12};
13use std::sync::atomic::AtomicBool;
14use crate::{
15 CallOption, EouOption,
16 event::EventSender,
17 media::TrackId,
18 synthesis::{
19 AliyunTtsClient, DeepegramTtsClient, SixtyDBTtsClient, SynthesisClient, SynthesisOption,
20 SynthesisType, TencentCloudTtsBasicClient, TencentCloudTtsClient,
21 },
22 transcription::{
23 AliyunAsrClientBuilder, DeepgramAsrClientBuilder, TencentCloudAsrClientBuilder,
24 TranscriptionClient, TranscriptionOption, TranscriptionType,
25 },
26};
27
28#[cfg(feature = "offline")]
29use crate::{synthesis::SupertonicTtsClient, transcription::SensevoiceAsrClientBuilder};
30
31use anyhow::Result;
32use std::{collections::HashMap, future::Future, pin::Pin, sync::Arc};
33use tokio::sync::mpsc;
34use tokio_util::sync::CancellationToken;
35use tracing::{debug, info};
36#[cfg(feature = "ringback-detection")]
37use tracing::warn;
38
39pub type FnCreateVadProcessor = fn(
40 token: CancellationToken,
41 event_sender: EventSender,
42 option: VADOption,
43) -> Result<Box<dyn Processor>>;
44
45pub type FnCreateEouProcessor = fn(
46 token: CancellationToken,
47 event_sender: EventSender,
48 option: EouOption,
49) -> Result<Box<dyn Processor>>;
50
51pub type FnCreateAsrClient = Box<
52 dyn Fn(
53 TrackId,
54 CancellationToken,
55 TranscriptionOption,
56 EventSender,
57 ) -> Pin<Box<dyn Future<Output = Result<Box<dyn TranscriptionClient>>> + Send>>
58 + Send
59 + Sync,
60>;
61pub type FnCreateTtsClient =
62 fn(streaming: bool, option: &SynthesisOption) -> Result<Box<dyn SynthesisClient>>;
63
64pub type CreateProcessorsHook = Box<
66 dyn Fn(
67 Arc<StreamEngine>,
68 TrackId,
69 CancellationToken,
70 EventSender,
71 TrackPacketSender,
72 CallOption,
73 ) -> Pin<Box<dyn Future<Output = Result<Vec<Box<dyn Processor>>>> + Send>>
74 + Send
75 + Sync,
76>;
77
78pub struct StreamEngine {
79 vad_creators: HashMap<VadType, FnCreateVadProcessor>,
80 eou_creators: HashMap<String, FnCreateEouProcessor>,
81 asr_creators: HashMap<TranscriptionType, FnCreateAsrClient>,
82 tts_creators: HashMap<SynthesisType, FnCreateTtsClient>,
83 create_processors_hook: Arc<CreateProcessorsHook>,
84}
85
86impl Default for StreamEngine {
87 fn default() -> Self {
88 let mut engine = Self::new();
89 engine.register_vad(VadType::Silero, VadProcessor::create);
90 engine.register_vad(VadType::Other("nop".to_string()), VadProcessor::create_nop);
91
92 engine.register_asr(
93 TranscriptionType::TencentCloud,
94 Box::new(TencentCloudAsrClientBuilder::create),
95 );
96 engine.register_asr(
97 TranscriptionType::Aliyun,
98 Box::new(AliyunAsrClientBuilder::create),
99 );
100 engine.register_asr(
101 TranscriptionType::Deepgram,
102 Box::new(DeepgramAsrClientBuilder::create),
103 );
104
105 #[cfg(feature = "offline")]
106 engine.register_asr(
107 TranscriptionType::Sensevoice,
108 Box::new(SensevoiceAsrClientBuilder::create),
109 );
110
111 engine.register_tts(SynthesisType::Aliyun, AliyunTtsClient::create);
112 engine.register_tts(SynthesisType::TencentCloud, TencentCloudTtsClient::create);
113 engine.register_tts(
114 SynthesisType::Other("tencent_basic".to_string()),
115 TencentCloudTtsBasicClient::create,
116 );
117 engine.register_tts(SynthesisType::Deepgram, DeepegramTtsClient::create);
118 engine.register_tts(SynthesisType::SixtyDB, SixtyDBTtsClient::create);
119
120 #[cfg(feature = "offline")]
121 engine.register_tts(SynthesisType::Supertonic, SupertonicTtsClient::create);
122
123 engine
124 }
125}
126
127impl StreamEngine {
128 pub fn new() -> Self {
129 Self {
130 vad_creators: HashMap::new(),
131 asr_creators: HashMap::new(),
132 tts_creators: HashMap::new(),
133 eou_creators: HashMap::new(),
134 create_processors_hook: Arc::new(Box::new(Self::default_create_procesors_hook)),
135 }
136 }
137
138 pub fn register_vad(&mut self, vad_type: VadType, creator: FnCreateVadProcessor) -> &mut Self {
139 self.vad_creators.insert(vad_type, creator);
140 self
141 }
142
143 pub fn register_eou(&mut self, name: String, creator: FnCreateEouProcessor) -> &mut Self {
144 self.eou_creators.insert(name, creator);
145 self
146 }
147
148 pub fn register_asr(
149 &mut self,
150 asr_type: TranscriptionType,
151 creator: FnCreateAsrClient,
152 ) -> &mut Self {
153 self.asr_creators.insert(asr_type, creator);
154 self
155 }
156
157 pub fn register_tts(
158 &mut self,
159 tts_type: SynthesisType,
160 creator: FnCreateTtsClient,
161 ) -> &mut Self {
162 self.tts_creators.insert(tts_type, creator);
163 self
164 }
165
166 pub fn create_vad_processor(
167 &self,
168 token: CancellationToken,
169 event_sender: EventSender,
170 option: VADOption,
171 ) -> Result<Box<dyn Processor>> {
172 let creator = self.vad_creators.get(&option.r#type);
173 if let Some(creator) = creator {
174 creator(token, event_sender, option)
175 } else {
176 Err(anyhow::anyhow!("VAD type not found: {}", option.r#type))
177 }
178 }
179 pub fn create_eou_processor(
180 &self,
181 token: CancellationToken,
182 event_sender: EventSender,
183 option: EouOption,
184 ) -> Result<Box<dyn Processor>> {
185 let creator = self
186 .eou_creators
187 .get(&option.r#type.clone().unwrap_or_default());
188 if let Some(creator) = creator {
189 creator(token, event_sender, option)
190 } else {
191 Err(anyhow::anyhow!("EOU type not found: {:?}", option.r#type))
192 }
193 }
194
195 pub async fn create_asr_processor(
196 &self,
197 track_id: TrackId,
198 cancel_token: CancellationToken,
199 option: TranscriptionOption,
200 event_sender: EventSender,
201 ) -> Result<Box<dyn Processor>> {
202 let asr_client = match option.provider {
203 Some(ref provider) => {
204 let creator = self.asr_creators.get(&provider);
205 if let Some(creator) = creator {
206 creator(track_id, cancel_token, option, event_sender).await?
207 } else {
208 return Err(anyhow::anyhow!("ASR type not found: {}", provider));
209 }
210 }
211 None => return Err(anyhow::anyhow!("ASR type not found: {:?}", option.provider)),
212 };
213 Ok(Box::new(AsrProcessor { asr_client }))
214 }
215
216 pub async fn create_tts_client(
217 &self,
218 streaming: bool,
219 tts_option: &SynthesisOption,
220 ) -> Result<Box<dyn SynthesisClient>> {
221 match tts_option.provider {
222 Some(ref provider) => {
223 let creator = self.tts_creators.get(&provider);
224 if let Some(creator) = creator {
225 creator(streaming, tts_option)
226 } else {
227 Err(anyhow::anyhow!("TTS type not found: {}", provider))
228 }
229 }
230 None => Err(anyhow::anyhow!(
231 "TTS type not found: {:?}",
232 tts_option.provider
233 )),
234 }
235 }
236
237 pub async fn create_processors(
238 engine: Arc<StreamEngine>,
239 track_id: TrackId,
240 cancel_token: CancellationToken,
241 event_sender: EventSender,
242 packet_sender: TrackPacketSender,
243 option: &CallOption,
244 ) -> Result<Vec<Box<dyn Processor>>> {
245 (engine.clone().create_processors_hook)(
246 engine,
247 track_id,
248 cancel_token,
249 event_sender,
250 packet_sender,
251 option.clone(),
252 )
253 .await
254 }
255
256 pub async fn create_tts_track(
257 engine: Arc<StreamEngine>,
258 cancel_token: CancellationToken,
259 session_id: String,
260 track_id: TrackId,
261 ssrc: u32,
262 play_id: Option<String>,
263 streaming: bool,
264 tts_option: &SynthesisOption,
265 auto_hangup: Option<bool>,
266 ) -> Result<(SynthesisHandle, Box<dyn Track>)> {
267 let (tx, rx) = mpsc::unbounded_channel();
268 let new_handle = SynthesisHandle::new(tx, play_id.clone(), ssrc);
269 let tts_client = engine.create_tts_client(streaming, tts_option).await?;
270 let sample_rate = tts_option.samplerate.unwrap_or(16000) as u32;
271 let tts_track = TtsTrack::new(track_id, session_id, streaming, play_id, rx, tts_client)
272 .with_ssrc(ssrc)
273 .with_auto_hangup(auto_hangup)
274 .with_sample_rate(sample_rate)
275 .with_cancel_token(cancel_token);
276 Ok((new_handle, Box::new(tts_track) as Box<dyn Track>))
277 }
278
279 pub fn with_processor_hook(&mut self, hook_fn: CreateProcessorsHook) -> &mut Self {
280 self.create_processors_hook = Arc::new(Box::new(hook_fn));
281 self
282 }
283
284 pub fn default_create_procesors_hook(
285 engine: Arc<StreamEngine>,
286 track_id: TrackId,
287 cancel_token: CancellationToken,
288 event_sender: EventSender,
289 packet_sender: TrackPacketSender,
290 option: CallOption,
291 ) -> Pin<Box<dyn Future<Output = Result<Vec<Box<dyn Processor>>>> + Send>> {
292 Box::pin(async move {
293 let mut processors = vec![];
294 debug!(%track_id, "Creating processors for track");
295
296 if let Some(realtime_option) = option.realtime {
297 debug!(%track_id, "Adding RealtimeProcessor");
298 let realtime_processor = crate::media::realtime_processor::RealtimeProcessor::new(
299 track_id.clone(),
300 cancel_token.child_token(),
301 event_sender.clone(),
302 packet_sender.clone(),
303 realtime_option,
304 )?;
305 processors.push(Box::new(realtime_processor) as Box<dyn Processor>);
306 return Ok(processors);
309 }
310
311 match option.denoise {
312 Some(true) if denoise_enabled() => {
313 debug!(%track_id, "Adding NoiseReducer processor");
314 let answered = Arc::new(AtomicBool::new(false));
315 spawn_answer_watcher(event_sender.subscribe(), answered.clone());
316 let noise_reducer =
317 NoiseReducer::new(INTERNAL_SAMPLERATE as usize, answered);
318 processors.push(Box::new(noise_reducer) as Box<dyn Processor>);
319 }
320 _ => {}
321 }
322 match option.vad {
323 Some(mut option) => {
324 debug!(%track_id, "Adding VadProcessor processor type={:?}", option.r#type);
325 option.samplerate = INTERNAL_SAMPLERATE;
326 let vad_processor: Box<dyn Processor + 'static> = engine.create_vad_processor(
327 cancel_token.child_token(),
328 event_sender.clone(),
329 option.to_owned(),
330 )?;
331 processors.push(vad_processor);
332 }
333 None => {}
334 }
335 if let Some(agc_option) = option.agc.clone() {
336 debug!(%track_id, "Adding AutomaticGainControl processor");
337 let agc = AutomaticGainControl::new(INTERNAL_SAMPLERATE as u32, agc_option)?;
338 processors.push(Box::new(agc) as Box<dyn Processor>);
339 }
340 match option.asr {
341 Some(mut option) => {
342 debug!(%track_id, "Adding AsrProcessor processor provider={:?}", option.provider);
343 option.samplerate = Some(INTERNAL_SAMPLERATE);
344 let asr_processor = engine
345 .create_asr_processor(
346 track_id.clone(),
347 cancel_token.child_token(),
348 option.to_owned(),
349 event_sender.clone(),
350 )
351 .await?;
352 processors.push(asr_processor);
353 }
354 None => {}
355 }
356 match option.eou {
357 Some(ref option) if eou_enabled() => {
358 let eou_processor = engine.create_eou_processor(
359 cancel_token.child_token(),
360 event_sender.clone(),
361 option.to_owned(),
362 )?;
363 processors.push(eou_processor);
364 }
365 Some(_) => {
366 info!(%track_id, "EOU processor skipped: disabled by EOU_ENABLED toggle");
370 }
371 None => {}
372 }
373 match option.inactivity_timeout {
374 Some(timeout_secs) if timeout_secs > 0 => {
375 let inactivity_processor = crate::media::inactivity::InactivityProcessor::new(
376 track_id.clone(),
377 std::time::Duration::from_secs(timeout_secs),
378 event_sender.clone(),
379 cancel_token.child_token(),
380 );
381 processors.push(Box::new(inactivity_processor) as Box<dyn Processor>);
382 }
383 _ => {}
384 }
385
386 #[cfg(feature = "ringback-detection")]
387 if let Some(ref ringback_opt) = option.ringback_detection {
388 if ringback_opt.enabled.unwrap_or(false) && ringback_detection_enabled() {
389 debug!(%track_id, "Adding RingbackDetectionProcessor");
390 match
391 crate::media::ringback_detection::processor::RingbackDetectionProcessor::new(
392 track_id.clone(),
393 cancel_token.child_token(),
394 event_sender.clone(),
395 ringback_opt.clone(),
396 None,
397 ) {
398 Ok(p) => processors.push(Box::new(p) as Box<dyn Processor>),
399 Err(e) => warn!(%track_id, "Failed to create RingbackDetectionProcessor: {}", e),
400 }
401 }
402 }
403
404 Ok(processors)
405 })
406 }
407}
408
409fn module_enabled(module: crate::config::RuntimeModule, configured: bool) -> bool {
413 match module.toggle_key() {
414 Some(key) => crate::runtime::module_override(key, configured),
415 None => configured,
416 }
417}
418
419fn denoise_enabled() -> bool {
420 let switches = crate::runtime::module_switches();
421 module_enabled(crate::config::RuntimeModule::Denoise, switches.denoise)
422}
423
424fn ringback_detection_enabled() -> bool {
425 let switches = crate::runtime::module_switches();
426 module_enabled(
427 crate::config::RuntimeModule::RingbackDetection,
428 switches.ringback_detection,
429 )
430}
431
432fn eou_enabled() -> bool {
433 let switches = crate::runtime::module_switches();
434 module_enabled(crate::config::RuntimeModule::Eou, switches.eou)
435}
436
437#[cfg(test)]
438mod module_toggle_tests {
439 use crate::config::{Config, ModulesConfig, ModuleToggle, RuntimeModule};
440 use crate::runtime::{ModuleSwitches, toggle_value_from};
441
442 #[test]
443 fn toggle_file_controls_switch() {
444 let dir = std::env::temp_dir().join(format!("zapbx_denoise_toggle_{}", std::process::id()));
445 std::fs::create_dir_all(&dir).unwrap();
446 let path = dir.join("app.env");
447
448 std::fs::remove_file(&path).ok();
450 assert_eq!(toggle_value_from(&path, "DENOISE_ENABLED"), None);
451 assert_eq!(toggle_value_from(&path, "EOU_ENABLED"), None);
452
453 std::fs::write(&path, "RUST_LOG=info\nOTHER=1\n").unwrap();
455 assert_eq!(toggle_value_from(&path, "DENOISE_ENABLED"), None);
456 std::fs::write(&path, "RUST_LOG=info\nDENOISE_ENABLED=false\n").unwrap();
458 assert_eq!(toggle_value_from(&path, "DENOISE_ENABLED"), Some(false));
459
460 std::fs::write(&path, "DENOISE_ENABLED=\"0\"\n").unwrap();
462 assert_eq!(toggle_value_from(&path, "DENOISE_ENABLED"), Some(false));
463 std::fs::write(&path, "DENOISE_ENABLED=true\n").unwrap();
464 assert_eq!(toggle_value_from(&path, "DENOISE_ENABLED"), Some(true));
465
466 std::fs::write(&path, "EOU_ENABLED=false\n").unwrap();
468 assert_eq!(toggle_value_from(&path, "EOU_ENABLED"), Some(false));
469 assert_eq!(toggle_value_from(&path, "DENOISE_ENABLED"), None);
470
471 std::fs::remove_dir_all(&dir).ok();
472 }
473
474 #[test]
477 fn config_module_switch_is_honoured() {
478 let mut config = Config::default();
479 config.modules = Some(ModulesConfig {
480 denoise: Some(ModuleToggle { enabled: false }),
481 eou: Some(ModuleToggle { enabled: false }),
482 ..Default::default()
483 });
484 let switches = ModuleSwitches::from_config(&config);
485 assert!(!switches.denoise);
486 assert!(!config.module_enabled(RuntimeModule::Eou));
487 assert!(config.module_enabled(RuntimeModule::RingbackDetection));
488 }
489}