Skip to main content

active_call/media/
engine.rs

1use super::{
2    INTERNAL_SAMPLERATE,
3    agc::AutomaticGainControl,
4    asr_processor::AsrProcessor,
5    denoiser::{NoiseReducer, spawn_answer_watcher},
6    processor::Processor,
7    track::{
8        Track, TrackPacketSender,
9        tts::{SynthesisHandle, TtsTrack},
10    },
11    vad::{VADOption, VadProcessor, VadType},
12};
13use std::sync::atomic::AtomicBool;
14use crate::{
15    CallOption, EouOption,
16    event::EventSender,
17    media::TrackId,
18    synthesis::{
19        AliyunTtsClient, DeepegramTtsClient, SixtyDBTtsClient, SynthesisClient, SynthesisOption,
20        SynthesisType, TencentCloudTtsBasicClient, TencentCloudTtsClient,
21    },
22    transcription::{
23        AliyunAsrClientBuilder, DeepgramAsrClientBuilder, TencentCloudAsrClientBuilder,
24        TranscriptionClient, TranscriptionOption, TranscriptionType,
25    },
26};
27
28#[cfg(feature = "offline")]
29use crate::{synthesis::SupertonicTtsClient, transcription::SensevoiceAsrClientBuilder};
30
31use anyhow::Result;
32use std::{collections::HashMap, future::Future, pin::Pin, sync::Arc};
33use tokio::sync::mpsc;
34use tokio_util::sync::CancellationToken;
35use tracing::{debug, info};
36#[cfg(feature = "ringback-detection")]
37use tracing::warn;
38
39pub type FnCreateVadProcessor = fn(
40    token: CancellationToken,
41    event_sender: EventSender,
42    option: VADOption,
43) -> Result<Box<dyn Processor>>;
44
45pub type FnCreateEouProcessor = fn(
46    token: CancellationToken,
47    event_sender: EventSender,
48    option: EouOption,
49) -> Result<Box<dyn Processor>>;
50
51pub type FnCreateAsrClient = Box<
52    dyn Fn(
53            TrackId,
54            CancellationToken,
55            TranscriptionOption,
56            EventSender,
57        ) -> Pin<Box<dyn Future<Output = Result<Box<dyn TranscriptionClient>>> + Send>>
58        + Send
59        + Sync,
60>;
61pub type FnCreateTtsClient =
62    fn(streaming: bool, option: &SynthesisOption) -> Result<Box<dyn SynthesisClient>>;
63
64// Define hook types
65pub type CreateProcessorsHook = Box<
66    dyn Fn(
67            Arc<StreamEngine>,
68            TrackId,
69            CancellationToken,
70            EventSender,
71            TrackPacketSender,
72            CallOption,
73        ) -> Pin<Box<dyn Future<Output = Result<Vec<Box<dyn Processor>>>> + Send>>
74        + Send
75        + Sync,
76>;
77
78pub struct StreamEngine {
79    vad_creators: HashMap<VadType, FnCreateVadProcessor>,
80    eou_creators: HashMap<String, FnCreateEouProcessor>,
81    asr_creators: HashMap<TranscriptionType, FnCreateAsrClient>,
82    tts_creators: HashMap<SynthesisType, FnCreateTtsClient>,
83    create_processors_hook: Arc<CreateProcessorsHook>,
84}
85
86impl Default for StreamEngine {
87    fn default() -> Self {
88        let mut engine = Self::new();
89        engine.register_vad(VadType::Silero, VadProcessor::create);
90        engine.register_vad(VadType::Other("nop".to_string()), VadProcessor::create_nop);
91
92        engine.register_asr(
93            TranscriptionType::TencentCloud,
94            Box::new(TencentCloudAsrClientBuilder::create),
95        );
96        engine.register_asr(
97            TranscriptionType::Aliyun,
98            Box::new(AliyunAsrClientBuilder::create),
99        );
100        engine.register_asr(
101            TranscriptionType::Deepgram,
102            Box::new(DeepgramAsrClientBuilder::create),
103        );
104
105        #[cfg(feature = "offline")]
106        engine.register_asr(
107            TranscriptionType::Sensevoice,
108            Box::new(SensevoiceAsrClientBuilder::create),
109        );
110
111        engine.register_tts(SynthesisType::Aliyun, AliyunTtsClient::create);
112        engine.register_tts(SynthesisType::TencentCloud, TencentCloudTtsClient::create);
113        engine.register_tts(
114            SynthesisType::Other("tencent_basic".to_string()),
115            TencentCloudTtsBasicClient::create,
116        );
117        engine.register_tts(SynthesisType::Deepgram, DeepegramTtsClient::create);
118        engine.register_tts(SynthesisType::SixtyDB, SixtyDBTtsClient::create);
119
120        #[cfg(feature = "offline")]
121        engine.register_tts(SynthesisType::Supertonic, SupertonicTtsClient::create);
122
123        engine
124    }
125}
126
127impl StreamEngine {
128    pub fn new() -> Self {
129        Self {
130            vad_creators: HashMap::new(),
131            asr_creators: HashMap::new(),
132            tts_creators: HashMap::new(),
133            eou_creators: HashMap::new(),
134            create_processors_hook: Arc::new(Box::new(Self::default_create_procesors_hook)),
135        }
136    }
137
138    pub fn register_vad(&mut self, vad_type: VadType, creator: FnCreateVadProcessor) -> &mut Self {
139        self.vad_creators.insert(vad_type, creator);
140        self
141    }
142
143    pub fn register_eou(&mut self, name: String, creator: FnCreateEouProcessor) -> &mut Self {
144        self.eou_creators.insert(name, creator);
145        self
146    }
147
148    pub fn register_asr(
149        &mut self,
150        asr_type: TranscriptionType,
151        creator: FnCreateAsrClient,
152    ) -> &mut Self {
153        self.asr_creators.insert(asr_type, creator);
154        self
155    }
156
157    pub fn register_tts(
158        &mut self,
159        tts_type: SynthesisType,
160        creator: FnCreateTtsClient,
161    ) -> &mut Self {
162        self.tts_creators.insert(tts_type, creator);
163        self
164    }
165
166    pub fn create_vad_processor(
167        &self,
168        token: CancellationToken,
169        event_sender: EventSender,
170        option: VADOption,
171    ) -> Result<Box<dyn Processor>> {
172        let creator = self.vad_creators.get(&option.r#type);
173        if let Some(creator) = creator {
174            creator(token, event_sender, option)
175        } else {
176            Err(anyhow::anyhow!("VAD type not found: {}", option.r#type))
177        }
178    }
179    pub fn create_eou_processor(
180        &self,
181        token: CancellationToken,
182        event_sender: EventSender,
183        option: EouOption,
184    ) -> Result<Box<dyn Processor>> {
185        let creator = self
186            .eou_creators
187            .get(&option.r#type.clone().unwrap_or_default());
188        if let Some(creator) = creator {
189            creator(token, event_sender, option)
190        } else {
191            Err(anyhow::anyhow!("EOU type not found: {:?}", option.r#type))
192        }
193    }
194
195    pub async fn create_asr_processor(
196        &self,
197        track_id: TrackId,
198        cancel_token: CancellationToken,
199        option: TranscriptionOption,
200        event_sender: EventSender,
201    ) -> Result<Box<dyn Processor>> {
202        let asr_client = match option.provider {
203            Some(ref provider) => {
204                let creator = self.asr_creators.get(&provider);
205                if let Some(creator) = creator {
206                    creator(track_id, cancel_token, option, event_sender).await?
207                } else {
208                    return Err(anyhow::anyhow!("ASR type not found: {}", provider));
209                }
210            }
211            None => return Err(anyhow::anyhow!("ASR type not found: {:?}", option.provider)),
212        };
213        Ok(Box::new(AsrProcessor { asr_client }))
214    }
215
216    pub async fn create_tts_client(
217        &self,
218        streaming: bool,
219        tts_option: &SynthesisOption,
220    ) -> Result<Box<dyn SynthesisClient>> {
221        match tts_option.provider {
222            Some(ref provider) => {
223                let creator = self.tts_creators.get(&provider);
224                if let Some(creator) = creator {
225                    creator(streaming, tts_option)
226                } else {
227                    Err(anyhow::anyhow!("TTS type not found: {}", provider))
228                }
229            }
230            None => Err(anyhow::anyhow!(
231                "TTS type not found: {:?}",
232                tts_option.provider
233            )),
234        }
235    }
236
237    pub async fn create_processors(
238        engine: Arc<StreamEngine>,
239        track_id: TrackId,
240        cancel_token: CancellationToken,
241        event_sender: EventSender,
242        packet_sender: TrackPacketSender,
243        option: &CallOption,
244    ) -> Result<Vec<Box<dyn Processor>>> {
245        (engine.clone().create_processors_hook)(
246            engine,
247            track_id,
248            cancel_token,
249            event_sender,
250            packet_sender,
251            option.clone(),
252        )
253        .await
254    }
255
256    pub async fn create_tts_track(
257        engine: Arc<StreamEngine>,
258        cancel_token: CancellationToken,
259        session_id: String,
260        track_id: TrackId,
261        ssrc: u32,
262        play_id: Option<String>,
263        streaming: bool,
264        tts_option: &SynthesisOption,
265        auto_hangup: Option<bool>,
266    ) -> Result<(SynthesisHandle, Box<dyn Track>)> {
267        let (tx, rx) = mpsc::unbounded_channel();
268        let new_handle = SynthesisHandle::new(tx, play_id.clone(), ssrc);
269        let tts_client = engine.create_tts_client(streaming, tts_option).await?;
270        let sample_rate = tts_option.samplerate.unwrap_or(16000) as u32;
271        let tts_track = TtsTrack::new(track_id, session_id, streaming, play_id, rx, tts_client)
272            .with_ssrc(ssrc)
273            .with_auto_hangup(auto_hangup)
274            .with_sample_rate(sample_rate)
275            .with_cancel_token(cancel_token);
276        Ok((new_handle, Box::new(tts_track) as Box<dyn Track>))
277    }
278
279    pub fn with_processor_hook(&mut self, hook_fn: CreateProcessorsHook) -> &mut Self {
280        self.create_processors_hook = Arc::new(Box::new(hook_fn));
281        self
282    }
283
284    pub fn default_create_procesors_hook(
285        engine: Arc<StreamEngine>,
286        track_id: TrackId,
287        cancel_token: CancellationToken,
288        event_sender: EventSender,
289        packet_sender: TrackPacketSender,
290        option: CallOption,
291    ) -> Pin<Box<dyn Future<Output = Result<Vec<Box<dyn Processor>>>> + Send>> {
292        Box::pin(async move {
293            let mut processors = vec![];
294            debug!(%track_id, "Creating processors for track");
295
296            if let Some(realtime_option) = option.realtime {
297                debug!(%track_id, "Adding RealtimeProcessor");
298                let realtime_processor = crate::media::realtime_processor::RealtimeProcessor::new(
299                    track_id.clone(),
300                    cancel_token.child_token(),
301                    event_sender.clone(),
302                    packet_sender.clone(),
303                    realtime_option,
304                )?;
305                processors.push(Box::new(realtime_processor) as Box<dyn Processor>);
306                // In realtime mode, we usually don't need separate VAD or ASR processors
307                // as they are handled by the realtime service (OpenAI/Azure)
308                return Ok(processors);
309            }
310
311            match option.denoise {
312                Some(true) if denoise_enabled() => {
313                    debug!(%track_id, "Adding NoiseReducer processor");
314                    let answered = Arc::new(AtomicBool::new(false));
315                    spawn_answer_watcher(event_sender.subscribe(), answered.clone());
316                    let noise_reducer =
317                        NoiseReducer::new(INTERNAL_SAMPLERATE as usize, answered);
318                    processors.push(Box::new(noise_reducer) as Box<dyn Processor>);
319                }
320                _ => {}
321            }
322            match option.vad {
323                Some(mut option) => {
324                    debug!(%track_id, "Adding VadProcessor processor type={:?}", option.r#type);
325                    option.samplerate = INTERNAL_SAMPLERATE;
326                    let vad_processor: Box<dyn Processor + 'static> = engine.create_vad_processor(
327                        cancel_token.child_token(),
328                        event_sender.clone(),
329                        option.to_owned(),
330                    )?;
331                    processors.push(vad_processor);
332                }
333                None => {}
334            }
335            if let Some(agc_option) = option.agc.clone() {
336                debug!(%track_id, "Adding AutomaticGainControl processor");
337                let agc = AutomaticGainControl::new(INTERNAL_SAMPLERATE as u32, agc_option)?;
338                processors.push(Box::new(agc) as Box<dyn Processor>);
339            }
340            match option.asr {
341                Some(mut option) => {
342                    debug!(%track_id, "Adding AsrProcessor processor provider={:?}", option.provider);
343                    option.samplerate = Some(INTERNAL_SAMPLERATE);
344                    let asr_processor = engine
345                        .create_asr_processor(
346                            track_id.clone(),
347                            cancel_token.child_token(),
348                            option.to_owned(),
349                            event_sender.clone(),
350                        )
351                        .await?;
352                    processors.push(asr_processor);
353                }
354                None => {}
355            }
356            match option.eou {
357                Some(ref option) if eou_enabled() => {
358                    let eou_processor = engine.create_eou_processor(
359                        cancel_token.child_token(),
360                        event_sender.clone(),
361                        option.to_owned(),
362                    )?;
363                    processors.push(eou_processor);
364                }
365                Some(_) => {
366                    // Defense in depth: even if the playbook still carries an
367                    // `eou` block, ops can disable EOU fleet-wide via the
368                    // toggle file without a restart (EOU_ENABLED=false).
369                    info!(%track_id, "EOU processor skipped: disabled by EOU_ENABLED toggle");
370                }
371                None => {}
372            }
373            match option.inactivity_timeout {
374                Some(timeout_secs) if timeout_secs > 0 => {
375                    let inactivity_processor = crate::media::inactivity::InactivityProcessor::new(
376                        track_id.clone(),
377                        std::time::Duration::from_secs(timeout_secs),
378                        event_sender.clone(),
379                        cancel_token.child_token(),
380                    );
381                    processors.push(Box::new(inactivity_processor) as Box<dyn Processor>);
382                }
383                _ => {}
384            }
385
386            #[cfg(feature = "ringback-detection")]
387            if let Some(ref ringback_opt) = option.ringback_detection {
388                if ringback_opt.enabled.unwrap_or(false) && ringback_detection_enabled() {
389                    debug!(%track_id, "Adding RingbackDetectionProcessor");
390                    match
391                        crate::media::ringback_detection::processor::RingbackDetectionProcessor::new(
392                            track_id.clone(),
393                            cancel_token.child_token(),
394                            event_sender.clone(),
395                            ringback_opt.clone(),
396                            None,
397                        ) {
398                        Ok(p) => processors.push(Box::new(p) as Box<dyn Processor>),
399                        Err(e) => warn!(%track_id, "Failed to create RingbackDetectionProcessor: {}", e),
400                    }
401                }
402            }
403
404            Ok(processors)
405        })
406    }
407}
408
409/// Module switches for the current config snapshot, with the ops toggle file
410/// (`PBX_TOGGLE_FILE`) taking precedence per key. Both are consulted at call
411/// setup, so a config reload or a toggle-file edit applies to the next call.
412fn module_enabled(module: crate::config::RuntimeModule, configured: bool) -> bool {
413    match module.toggle_key() {
414        Some(key) => crate::runtime::module_override(key, configured),
415        None => configured,
416    }
417}
418
419fn denoise_enabled() -> bool {
420    let switches = crate::runtime::module_switches();
421    module_enabled(crate::config::RuntimeModule::Denoise, switches.denoise)
422}
423
424fn ringback_detection_enabled() -> bool {
425    let switches = crate::runtime::module_switches();
426    module_enabled(
427        crate::config::RuntimeModule::RingbackDetection,
428        switches.ringback_detection,
429    )
430}
431
432fn eou_enabled() -> bool {
433    let switches = crate::runtime::module_switches();
434    module_enabled(crate::config::RuntimeModule::Eou, switches.eou)
435}
436
437#[cfg(test)]
438mod module_toggle_tests {
439    use crate::config::{Config, ModulesConfig, ModuleToggle, RuntimeModule};
440    use crate::runtime::{ModuleSwitches, toggle_value_from};
441
442    #[test]
443    fn toggle_file_controls_switch() {
444        let dir = std::env::temp_dir().join(format!("zapbx_denoise_toggle_{}", std::process::id()));
445        std::fs::create_dir_all(&dir).unwrap();
446        let path = dir.join("app.env");
447
448        // No file => unspecified, so the configured value applies.
449        std::fs::remove_file(&path).ok();
450        assert_eq!(toggle_value_from(&path, "DENOISE_ENABLED"), None);
451        assert_eq!(toggle_value_from(&path, "EOU_ENABLED"), None);
452
453        // Key missing => unspecified.
454        std::fs::write(&path, "RUST_LOG=info\nOTHER=1\n").unwrap();
455        assert_eq!(toggle_value_from(&path, "DENOISE_ENABLED"), None);
456        // Explicitly disabled.
457        std::fs::write(&path, "RUST_LOG=info\nDENOISE_ENABLED=false\n").unwrap();
458        assert_eq!(toggle_value_from(&path, "DENOISE_ENABLED"), Some(false));
459
460        // Quoted and numeric forms.
461        std::fs::write(&path, "DENOISE_ENABLED=\"0\"\n").unwrap();
462        assert_eq!(toggle_value_from(&path, "DENOISE_ENABLED"), Some(false));
463        std::fs::write(&path, "DENOISE_ENABLED=true\n").unwrap();
464        assert_eq!(toggle_value_from(&path, "DENOISE_ENABLED"), Some(true));
465
466        // Keys are independent: disabling EOU does not touch denoise.
467        std::fs::write(&path, "EOU_ENABLED=false\n").unwrap();
468        assert_eq!(toggle_value_from(&path, "EOU_ENABLED"), Some(false));
469        assert_eq!(toggle_value_from(&path, "DENOISE_ENABLED"), None);
470
471        std::fs::remove_dir_all(&dir).ok();
472    }
473
474    /// A config-file switch alone is enough to disable a module (this is the
475    /// part that hot reload drives).
476    #[test]
477    fn config_module_switch_is_honoured() {
478        let mut config = Config::default();
479        config.modules = Some(ModulesConfig {
480            denoise: Some(ModuleToggle { enabled: false }),
481            eou: Some(ModuleToggle { enabled: false }),
482            ..Default::default()
483        });
484        let switches = ModuleSwitches::from_config(&config);
485        assert!(!switches.denoise);
486        assert!(!config.module_enabled(RuntimeModule::Eou));
487        assert!(config.module_enabled(RuntimeModule::RingbackDetection));
488    }
489}