Skip to main content

active_call/media/
engine.rs

1use super::{
2    INTERNAL_SAMPLERATE,
3    agc::AutomaticGainControl,
4    asr_processor::AsrProcessor,
5    denoiser::NoiseReducer,
6    processor::Processor,
7    track::{
8        Track, TrackPacketSender,
9        tts::{SynthesisHandle, TtsTrack},
10    },
11    vad::{VADOption, VadProcessor, VadType},
12};
13use crate::{
14    CallOption, EouOption,
15    event::EventSender,
16    media::TrackId,
17    synthesis::{
18        AliyunTtsClient, DeepegramTtsClient, SixtyDBTtsClient, SynthesisClient, SynthesisOption,
19        SynthesisType, TencentCloudTtsBasicClient, TencentCloudTtsClient,
20    },
21    transcription::{
22        AliyunAsrClientBuilder, DeepgramAsrClientBuilder, TencentCloudAsrClientBuilder,
23        TranscriptionClient, TranscriptionOption, TranscriptionType,
24    },
25};
26
27#[cfg(feature = "offline")]
28use crate::{synthesis::SupertonicTtsClient, transcription::SensevoiceAsrClientBuilder};
29
30use anyhow::Result;
31use std::{collections::HashMap, future::Future, pin::Pin, sync::Arc};
32use tokio::sync::mpsc;
33use tokio_util::sync::CancellationToken;
34use tracing::debug;
35#[cfg(feature = "ringback-detection")]
36use tracing::warn;
37
38pub type FnCreateVadProcessor = fn(
39    token: CancellationToken,
40    event_sender: EventSender,
41    option: VADOption,
42) -> Result<Box<dyn Processor>>;
43
44pub type FnCreateEouProcessor = fn(
45    token: CancellationToken,
46    event_sender: EventSender,
47    option: EouOption,
48) -> Result<Box<dyn Processor>>;
49
50pub type FnCreateAsrClient = Box<
51    dyn Fn(
52            TrackId,
53            CancellationToken,
54            TranscriptionOption,
55            EventSender,
56        ) -> Pin<Box<dyn Future<Output = Result<Box<dyn TranscriptionClient>>> + Send>>
57        + Send
58        + Sync,
59>;
60pub type FnCreateTtsClient =
61    fn(streaming: bool, option: &SynthesisOption) -> Result<Box<dyn SynthesisClient>>;
62
63// Define hook types
64pub type CreateProcessorsHook = Box<
65    dyn Fn(
66            Arc<StreamEngine>,
67            TrackId,
68            CancellationToken,
69            EventSender,
70            TrackPacketSender,
71            CallOption,
72        ) -> Pin<Box<dyn Future<Output = Result<Vec<Box<dyn Processor>>>> + Send>>
73        + Send
74        + Sync,
75>;
76
77pub struct StreamEngine {
78    vad_creators: HashMap<VadType, FnCreateVadProcessor>,
79    eou_creators: HashMap<String, FnCreateEouProcessor>,
80    asr_creators: HashMap<TranscriptionType, FnCreateAsrClient>,
81    tts_creators: HashMap<SynthesisType, FnCreateTtsClient>,
82    create_processors_hook: Arc<CreateProcessorsHook>,
83}
84
85impl Default for StreamEngine {
86    fn default() -> Self {
87        let mut engine = Self::new();
88        engine.register_vad(VadType::Silero, VadProcessor::create);
89        engine.register_vad(VadType::Other("nop".to_string()), VadProcessor::create_nop);
90
91        engine.register_asr(
92            TranscriptionType::TencentCloud,
93            Box::new(TencentCloudAsrClientBuilder::create),
94        );
95        engine.register_asr(
96            TranscriptionType::Aliyun,
97            Box::new(AliyunAsrClientBuilder::create),
98        );
99        engine.register_asr(
100            TranscriptionType::Deepgram,
101            Box::new(DeepgramAsrClientBuilder::create),
102        );
103
104        #[cfg(feature = "offline")]
105        engine.register_asr(
106            TranscriptionType::Sensevoice,
107            Box::new(SensevoiceAsrClientBuilder::create),
108        );
109
110        engine.register_tts(SynthesisType::Aliyun, AliyunTtsClient::create);
111        engine.register_tts(SynthesisType::TencentCloud, TencentCloudTtsClient::create);
112        engine.register_tts(
113            SynthesisType::Other("tencent_basic".to_string()),
114            TencentCloudTtsBasicClient::create,
115        );
116        engine.register_tts(SynthesisType::Deepgram, DeepegramTtsClient::create);
117        engine.register_tts(SynthesisType::SixtyDB, SixtyDBTtsClient::create);
118
119        #[cfg(feature = "offline")]
120        engine.register_tts(SynthesisType::Supertonic, SupertonicTtsClient::create);
121
122        engine
123    }
124}
125
126impl StreamEngine {
127    pub fn new() -> Self {
128        Self {
129            vad_creators: HashMap::new(),
130            asr_creators: HashMap::new(),
131            tts_creators: HashMap::new(),
132            eou_creators: HashMap::new(),
133            create_processors_hook: Arc::new(Box::new(Self::default_create_procesors_hook)),
134        }
135    }
136
137    pub fn register_vad(&mut self, vad_type: VadType, creator: FnCreateVadProcessor) -> &mut Self {
138        self.vad_creators.insert(vad_type, creator);
139        self
140    }
141
142    pub fn register_eou(&mut self, name: String, creator: FnCreateEouProcessor) -> &mut Self {
143        self.eou_creators.insert(name, creator);
144        self
145    }
146
147    pub fn register_asr(
148        &mut self,
149        asr_type: TranscriptionType,
150        creator: FnCreateAsrClient,
151    ) -> &mut Self {
152        self.asr_creators.insert(asr_type, creator);
153        self
154    }
155
156    pub fn register_tts(
157        &mut self,
158        tts_type: SynthesisType,
159        creator: FnCreateTtsClient,
160    ) -> &mut Self {
161        self.tts_creators.insert(tts_type, creator);
162        self
163    }
164
165    pub fn create_vad_processor(
166        &self,
167        token: CancellationToken,
168        event_sender: EventSender,
169        option: VADOption,
170    ) -> Result<Box<dyn Processor>> {
171        let creator = self.vad_creators.get(&option.r#type);
172        if let Some(creator) = creator {
173            creator(token, event_sender, option)
174        } else {
175            Err(anyhow::anyhow!("VAD type not found: {}", option.r#type))
176        }
177    }
178    pub fn create_eou_processor(
179        &self,
180        token: CancellationToken,
181        event_sender: EventSender,
182        option: EouOption,
183    ) -> Result<Box<dyn Processor>> {
184        let creator = self
185            .eou_creators
186            .get(&option.r#type.clone().unwrap_or_default());
187        if let Some(creator) = creator {
188            creator(token, event_sender, option)
189        } else {
190            Err(anyhow::anyhow!("EOU type not found: {:?}", option.r#type))
191        }
192    }
193
194    pub async fn create_asr_processor(
195        &self,
196        track_id: TrackId,
197        cancel_token: CancellationToken,
198        option: TranscriptionOption,
199        event_sender: EventSender,
200    ) -> Result<Box<dyn Processor>> {
201        let asr_client = match option.provider {
202            Some(ref provider) => {
203                let creator = self.asr_creators.get(&provider);
204                if let Some(creator) = creator {
205                    creator(track_id, cancel_token, option, event_sender).await?
206                } else {
207                    return Err(anyhow::anyhow!("ASR type not found: {}", provider));
208                }
209            }
210            None => return Err(anyhow::anyhow!("ASR type not found: {:?}", option.provider)),
211        };
212        Ok(Box::new(AsrProcessor { asr_client }))
213    }
214
215    pub async fn create_tts_client(
216        &self,
217        streaming: bool,
218        tts_option: &SynthesisOption,
219    ) -> Result<Box<dyn SynthesisClient>> {
220        match tts_option.provider {
221            Some(ref provider) => {
222                let creator = self.tts_creators.get(&provider);
223                if let Some(creator) = creator {
224                    creator(streaming, tts_option)
225                } else {
226                    Err(anyhow::anyhow!("TTS type not found: {}", provider))
227                }
228            }
229            None => Err(anyhow::anyhow!(
230                "TTS type not found: {:?}",
231                tts_option.provider
232            )),
233        }
234    }
235
236    pub async fn create_processors(
237        engine: Arc<StreamEngine>,
238        track_id: TrackId,
239        cancel_token: CancellationToken,
240        event_sender: EventSender,
241        packet_sender: TrackPacketSender,
242        option: &CallOption,
243    ) -> Result<Vec<Box<dyn Processor>>> {
244        (engine.clone().create_processors_hook)(
245            engine,
246            track_id,
247            cancel_token,
248            event_sender,
249            packet_sender,
250            option.clone(),
251        )
252        .await
253    }
254
255    pub async fn create_tts_track(
256        engine: Arc<StreamEngine>,
257        cancel_token: CancellationToken,
258        session_id: String,
259        track_id: TrackId,
260        ssrc: u32,
261        play_id: Option<String>,
262        streaming: bool,
263        tts_option: &SynthesisOption,
264        auto_hangup: Option<bool>,
265    ) -> Result<(SynthesisHandle, Box<dyn Track>)> {
266        let (tx, rx) = mpsc::unbounded_channel();
267        let new_handle = SynthesisHandle::new(tx, play_id.clone(), ssrc);
268        let tts_client = engine.create_tts_client(streaming, tts_option).await?;
269        let sample_rate = tts_option.samplerate.unwrap_or(16000) as u32;
270        let tts_track = TtsTrack::new(track_id, session_id, streaming, play_id, rx, tts_client)
271            .with_ssrc(ssrc)
272            .with_auto_hangup(auto_hangup)
273            .with_sample_rate(sample_rate)
274            .with_cancel_token(cancel_token);
275        Ok((new_handle, Box::new(tts_track) as Box<dyn Track>))
276    }
277
278    pub fn with_processor_hook(&mut self, hook_fn: CreateProcessorsHook) -> &mut Self {
279        self.create_processors_hook = Arc::new(Box::new(hook_fn));
280        self
281    }
282
283    pub fn default_create_procesors_hook(
284        engine: Arc<StreamEngine>,
285        track_id: TrackId,
286        cancel_token: CancellationToken,
287        event_sender: EventSender,
288        packet_sender: TrackPacketSender,
289        option: CallOption,
290    ) -> Pin<Box<dyn Future<Output = Result<Vec<Box<dyn Processor>>>> + Send>> {
291        Box::pin(async move {
292            let mut processors = vec![];
293            debug!(%track_id, "Creating processors for track");
294
295            if let Some(realtime_option) = option.realtime {
296                debug!(%track_id, "Adding RealtimeProcessor");
297                let realtime_processor = crate::media::realtime_processor::RealtimeProcessor::new(
298                    track_id.clone(),
299                    cancel_token.child_token(),
300                    event_sender.clone(),
301                    packet_sender.clone(),
302                    realtime_option,
303                )?;
304                processors.push(Box::new(realtime_processor) as Box<dyn Processor>);
305                // In realtime mode, we usually don't need separate VAD or ASR processors
306                // as they are handled by the realtime service (OpenAI/Azure)
307                return Ok(processors);
308            }
309
310            match option.denoise {
311                Some(true) => {
312                    debug!(%track_id, "Adding NoiseReducer processor");
313                    let noise_reducer = NoiseReducer::new(INTERNAL_SAMPLERATE as usize);
314                    processors.push(Box::new(noise_reducer) as Box<dyn Processor>);
315                }
316                _ => {}
317            }
318            match option.vad {
319                Some(mut option) => {
320                    debug!(%track_id, "Adding VadProcessor processor type={:?}", option.r#type);
321                    option.samplerate = INTERNAL_SAMPLERATE;
322                    let vad_processor: Box<dyn Processor + 'static> = engine.create_vad_processor(
323                        cancel_token.child_token(),
324                        event_sender.clone(),
325                        option.to_owned(),
326                    )?;
327                    processors.push(vad_processor);
328                }
329                None => {}
330            }
331            if let Some(agc_option) = option.agc.clone() {
332                debug!(%track_id, "Adding AutomaticGainControl processor");
333                let agc = AutomaticGainControl::new(INTERNAL_SAMPLERATE as u32, agc_option)?;
334                processors.push(Box::new(agc) as Box<dyn Processor>);
335            }
336            match option.asr {
337                Some(mut option) => {
338                    debug!(%track_id, "Adding AsrProcessor processor provider={:?}", option.provider);
339                    option.samplerate = Some(INTERNAL_SAMPLERATE);
340                    let asr_processor = engine
341                        .create_asr_processor(
342                            track_id.clone(),
343                            cancel_token.child_token(),
344                            option.to_owned(),
345                            event_sender.clone(),
346                        )
347                        .await?;
348                    processors.push(asr_processor);
349                }
350                None => {}
351            }
352            match option.eou {
353                Some(ref option) => {
354                    let eou_processor = engine.create_eou_processor(
355                        cancel_token.child_token(),
356                        event_sender.clone(),
357                        option.to_owned(),
358                    )?;
359                    processors.push(eou_processor);
360                }
361                None => {}
362            }
363            match option.inactivity_timeout {
364                Some(timeout_secs) if timeout_secs > 0 => {
365                    let inactivity_processor = crate::media::inactivity::InactivityProcessor::new(
366                        track_id.clone(),
367                        std::time::Duration::from_secs(timeout_secs),
368                        event_sender.clone(),
369                        cancel_token.child_token(),
370                    );
371                    processors.push(Box::new(inactivity_processor) as Box<dyn Processor>);
372                }
373                _ => {}
374            }
375
376            #[cfg(feature = "ringback-detection")]
377            if let Some(ref ringback_opt) = option.ringback_detection {
378                if ringback_opt.enabled.unwrap_or(false) {
379                    debug!(%track_id, "Adding RingbackDetectionProcessor");
380                    match
381                        crate::media::ringback_detection::processor::RingbackDetectionProcessor::new(
382                            track_id.clone(),
383                            cancel_token.child_token(),
384                            event_sender.clone(),
385                            ringback_opt.clone(),
386                            None,
387                        ) {
388                        Ok(p) => processors.push(Box::new(p) as Box<dyn Processor>),
389                        Err(e) => warn!(%track_id, "Failed to create RingbackDetectionProcessor: {}", e),
390                    }
391                }
392            }
393
394            Ok(processors)
395        })
396    }
397}