1use super::INTERNAL_SAMPLERATE;
2use super::track::track_codec::TrackCodec;
3use crate::event::{EventSender, SessionEvent};
4use crate::media::{AudioFrame, Samples, SourcePacket, TrackId};
5use anyhow::Result;
6use std::any::Any;
7use std::sync::{Arc, Mutex, RwLock};
8use tokio::sync::mpsc;
9
10pub trait Processor: Send + Sync + Any {
11 fn process_frame(&mut self, frame: &mut AudioFrame) -> Result<()>;
12}
13
14pub fn convert_to_mono(samples: &mut Vec<i16>, channels: u16) {
15 if channels != 2 {
16 return;
17 }
18 let mut i = 0;
19 let mut j = 0;
20 while i < samples.len() {
21 let l = samples[i] as i32;
22 let r = samples[i + 1] as i32;
23 samples[j] = ((l + r) / 2) as i16;
24 i += 2;
25 j += 1;
26 }
27 samples.truncate(j);
28}
29
30impl Default for AudioFrame {
31 fn default() -> Self {
32 Self {
33 track_id: "".to_string(),
34 samples: Samples::Empty,
35 timestamp: 0,
36 sample_rate: 16000,
37 channels: 1,
38 src_packet: None,
39 speech_probability: None,
40 }
41 }
42}
43
44impl Samples {
45 pub fn is_empty(&self) -> bool {
46 match self {
47 Samples::PCM { samples } => samples.is_empty(),
48 Samples::RTP { payload, .. } => payload.is_empty(),
49 Samples::Dtmf { .. } => false,
50 Samples::Empty => true,
51 }
52 }
53}
54
55#[derive(Clone)]
56pub struct ProcessorChain {
57 processors: Arc<Mutex<Vec<Box<dyn Processor>>>>,
58 pub codec: TrackCodec,
59 sample_rate: u32,
60 pub force_decode: bool,
61 pub raw_tap: Arc<RwLock<Option<mpsc::UnboundedSender<AudioFrame>>>>,
70 pub tap_owner: Arc<RwLock<Option<TrackId>>>,
76}
77
78fn lock_processors(
89 processors: &Mutex<Vec<Box<dyn Processor>>>,
90) -> std::sync::MutexGuard<'_, Vec<Box<dyn Processor>>> {
91 processors.lock().unwrap_or_else(|e| e.into_inner())
92}
93
94impl ProcessorChain {
95 pub fn new(_sample_rate: u32) -> Self {
96 Self {
97 processors: Arc::new(Mutex::new(Vec::new())),
98 codec: TrackCodec::new(),
99 sample_rate: INTERNAL_SAMPLERATE,
100 force_decode: true,
101 raw_tap: Arc::new(RwLock::new(None)),
102 tap_owner: Arc::new(RwLock::new(None)),
103 }
104 }
105 pub fn set_raw_tap(&mut self, tap: Option<mpsc::UnboundedSender<AudioFrame>>) {
106 *self.raw_tap.write().unwrap() = tap;
107 }
108 pub fn set_tap_owner(&mut self, owner: Option<TrackId>) {
110 *self.tap_owner.write().unwrap() = owner;
111 }
112 fn raw_tap(&self) -> Option<mpsc::UnboundedSender<AudioFrame>> {
113 self.raw_tap.read().unwrap().clone()
114 }
115 fn tap_owns(&self, track_id: &str) -> bool {
116 match self.tap_owner.read().unwrap().as_deref() {
117 Some(owner) => owner == track_id,
118 None => true,
119 }
120 }
121 pub fn insert_processor(&mut self, processor: Box<dyn Processor>) {
122 lock_processors(&self.processors).insert(0, processor);
123 }
124 pub fn append_processor(&mut self, processor: Box<dyn Processor>) {
125 lock_processors(&self.processors).push(processor);
126 }
127
128 pub fn has_processor<T: 'static>(&self) -> bool {
129 let processors = lock_processors(&self.processors);
130 processors
131 .iter()
132 .any(|processor| (processor.as_ref() as &dyn Any).is::<T>())
133 }
134
135 pub fn remove_processor<T: 'static>(&self) {
136 let mut processors = lock_processors(&self.processors);
137 processors.retain(|processor| !(processor.as_ref() as &dyn Any).is::<T>());
138 }
139
140 pub fn process_frame(&mut self, frame: &mut AudioFrame) -> Result<()> {
141 let mut processors = lock_processors(&self.processors);
142 if !self.force_decode && processors.is_empty() && self.raw_tap().is_none() {
143 return Ok(());
144 }
145 match &mut frame.samples {
146 Samples::RTP {
147 payload_type,
148 payload,
149 sequence_number,
150 } => {
151 if self
154 .codec
155 .get_codec_for_pt(*payload_type)
156 .is_some_and(|codec| codec.is_audio())
157 {
158 let (decoded_sample_rate, channels, samples) =
159 self.codec.decode(*payload_type, &payload);
160 let src_packet = SourcePacket {
161 sequence_number: *sequence_number,
162 payload_type: *payload_type,
163 payload: std::mem::take(payload),
164 };
165 frame.src_packet = Some(src_packet);
166 frame.channels = channels;
167 frame.samples = Samples::PCM { samples };
168 frame.sample_rate = decoded_sample_rate;
169 }
170 }
171 _ => {}
172 }
173
174 if let Some(tap) = self.raw_tap()
179 && self.tap_owns(&frame.track_id)
180 && let Samples::PCM { samples } = &frame.samples
181 && !samples.is_empty()
182 && frame.sample_rate > 0
183 {
184 let mut raw = frame.clone();
185 raw.src_packet = None;
186 let mono = match &mut raw.samples {
187 Samples::PCM { samples } => samples,
188 _ => unreachable!("checked PCM above"),
189 };
190 if raw.channels == 2 {
191 convert_to_mono(mono, 2);
192 raw.channels = 1;
193 }
194 let _ = tap.send(raw);
195 }
196
197 if let Samples::PCM { samples } = &mut frame.samples {
198 if frame.sample_rate != self.sample_rate {
199 let new_samples = self.codec.resample(
200 std::mem::take(samples),
201 frame.sample_rate,
202 self.sample_rate,
203 );
204 *samples = new_samples;
205 frame.sample_rate = self.sample_rate;
206 }
207 if frame.channels == 2 {
208 convert_to_mono(samples, 2);
209 frame.channels = 1;
210 }
211 }
212 for processor in processors.iter_mut() {
214 processor.process_frame(frame)?;
215 }
216 Ok(())
217 }
218}
219
220pub struct SubscribeProcessor {
221 event_sender: EventSender,
222 track_id: String,
223 track_index: u8, }
225
226impl SubscribeProcessor {
227 pub fn new(event_sender: EventSender, track_id: String, track_index: u8) -> Self {
228 Self {
229 event_sender,
230 track_id,
231 track_index,
232 }
233 }
234}
235
236impl Processor for SubscribeProcessor {
237 fn process_frame(&mut self, frame: &mut AudioFrame) -> Result<()> {
238 if let Samples::PCM { samples } = &frame.samples {
239 if !samples.is_empty() {
240 let pcm_data = audio_codec::samples_to_bytes(samples);
241 let mut data = Vec::with_capacity(pcm_data.len() + 1);
242 data.push(self.track_index);
243 data.extend_from_slice(&pcm_data);
244
245 let event = SessionEvent::Binary {
246 track_id: self.track_id.clone(),
247 timestamp: frame.timestamp,
248 data,
249 };
250 self.event_sender.send(event).ok();
251 }
252 }
253 Ok(())
254 }
255}
256
257#[cfg(test)]
258mod tests {
259 use super::*;
260
261 #[test]
267 fn raw_tap_visible_to_clones_taken_before_attach() {
268 let mut chain = ProcessorChain::new(INTERNAL_SAMPLERATE);
269 let mut worker = chain.clone();
270
271 let (tx, mut rx) = mpsc::unbounded_channel();
272 chain.set_raw_tap(Some(tx));
273
274 let mut frame = AudioFrame {
275 track_id: "track".to_string(),
276 samples: Samples::PCM {
277 samples: vec![100i16; 160],
278 },
279 sample_rate: 8000,
280 channels: 1,
281 ..Default::default()
282 };
283 worker.process_frame(&mut frame).unwrap();
284
285 let raw = rx
286 .try_recv()
287 .expect("raw tap should receive a frame from a pre-attach clone");
288 assert_eq!(raw.sample_rate, 8000);
289 assert_eq!(raw.channels, 1);
290 match raw.samples {
291 Samples::PCM { samples } => assert_eq!(samples.len(), 160),
292 _ => panic!("expected PCM samples on the raw tap"),
293 }
294 assert_eq!(frame.sample_rate, INTERNAL_SAMPLERATE);
296 }
297
298 #[test]
301 fn raw_tap_detach_visible_to_clones() {
302 let mut chain = ProcessorChain::new(INTERNAL_SAMPLERATE);
303 let (tx, _rx) = mpsc::unbounded_channel();
304 chain.set_raw_tap(Some(tx));
305 let mut worker = chain.clone();
306 chain.set_raw_tap(None);
307
308 let mut frame = AudioFrame {
309 track_id: "track".to_string(),
310 samples: Samples::PCM {
311 samples: vec![100i16; 160],
312 },
313 sample_rate: 8000,
314 channels: 1,
315 ..Default::default()
316 };
317 worker.process_frame(&mut frame).unwrap();
318 assert_eq!(frame.sample_rate, INTERNAL_SAMPLERATE);
319 }
320
321 #[test]
327 fn raw_tap_only_mirrors_owned_frames() {
328 let mut chain = ProcessorChain::new(INTERNAL_SAMPLERATE);
329 let (tx, mut rx) = mpsc::unbounded_channel();
330 chain.set_raw_tap(Some(tx));
331 chain.set_tap_owner(Some("server-side-track".to_string()));
332
333 let mut relayed = AudioFrame {
336 track_id: "caller".to_string(),
337 samples: Samples::PCM {
338 samples: vec![1i16; 160],
339 },
340 sample_rate: 8000,
341 channels: 1,
342 ..Default::default()
343 };
344 chain.process_frame(&mut relayed).unwrap();
345 assert!(
346 rx.try_recv().is_err(),
347 "relayed frames must not be mirrored to the raw tap"
348 );
349
350 let mut owned = AudioFrame {
352 track_id: "server-side-track".to_string(),
353 samples: Samples::PCM {
354 samples: vec![1i16; 160],
355 },
356 sample_rate: 16000,
357 channels: 1,
358 ..Default::default()
359 };
360 chain.process_frame(&mut owned).unwrap();
361 let raw = rx
362 .try_recv()
363 .expect("owned frames must be mirrored to the raw tap");
364 assert_eq!(raw.track_id, "server-side-track");
365 assert_eq!(raw.sample_rate, 16000);
366 }
367
368 #[test]
372 fn process_frame_tolerates_zero_sample_rate() {
373 let mut chain = ProcessorChain::new(INTERNAL_SAMPLERATE);
374 let mut frame = AudioFrame {
375 track_id: "track".to_string(),
376 samples: Samples::PCM {
377 samples: vec![100i16; 160],
378 },
379 sample_rate: 0,
380 channels: 1,
381 ..Default::default()
382 };
383
384 chain.process_frame(&mut frame).unwrap();
385 assert_eq!(frame.sample_rate, INTERNAL_SAMPLERATE);
386 match frame.samples {
387 Samples::PCM { samples } => assert_eq!(samples.len(), 160),
388 _ => panic!("expected PCM samples"),
389 }
390 }
391}