1use super::INTERNAL_SAMPLERATE;
2use super::track::track_codec::TrackCodec;
3use crate::event::{EventSender, SessionEvent};
4use crate::media::{AudioFrame, Samples, SourcePacket, TrackId};
5use anyhow::Result;
6use std::any::Any;
7use std::sync::{Arc, Mutex, RwLock};
8use tokio::sync::mpsc;
9
10pub trait Processor: Send + Sync + Any {
11 fn process_frame(&mut self, frame: &mut AudioFrame) -> Result<()>;
12}
13
14pub fn convert_to_mono(samples: &mut Vec<i16>, channels: u16) {
15 if channels != 2 {
16 return;
17 }
18 let mut i = 0;
19 let mut j = 0;
20 while i < samples.len() {
21 let l = samples[i] as i32;
22 let r = samples[i + 1] as i32;
23 samples[j] = ((l + r) / 2) as i16;
24 i += 2;
25 j += 1;
26 }
27 samples.truncate(j);
28}
29
30impl Default for AudioFrame {
31 fn default() -> Self {
32 Self {
33 track_id: "".to_string(),
34 samples: Samples::Empty,
35 timestamp: 0,
36 sample_rate: 16000,
37 channels: 1,
38 src_packet: None,
39 speech_probability: None,
40 }
41 }
42}
43
44impl Samples {
45 pub fn is_empty(&self) -> bool {
46 match self {
47 Samples::PCM { samples } => samples.is_empty(),
48 Samples::RTP { payload, .. } => payload.is_empty(),
49 Samples::Empty => true,
50 }
51 }
52}
53
54#[derive(Clone)]
55pub struct ProcessorChain {
56 processors: Arc<Mutex<Vec<Box<dyn Processor>>>>,
57 pub codec: TrackCodec,
58 sample_rate: u32,
59 pub force_decode: bool,
60 pub raw_tap: Arc<RwLock<Option<mpsc::UnboundedSender<AudioFrame>>>>,
69 pub tap_owner: Arc<RwLock<Option<TrackId>>>,
75}
76
77fn lock_processors(
88 processors: &Mutex<Vec<Box<dyn Processor>>>,
89) -> std::sync::MutexGuard<'_, Vec<Box<dyn Processor>>> {
90 processors.lock().unwrap_or_else(|e| e.into_inner())
91}
92
93impl ProcessorChain {
94 pub fn new(_sample_rate: u32) -> Self {
95 Self {
96 processors: Arc::new(Mutex::new(Vec::new())),
97 codec: TrackCodec::new(),
98 sample_rate: INTERNAL_SAMPLERATE,
99 force_decode: true,
100 raw_tap: Arc::new(RwLock::new(None)),
101 tap_owner: Arc::new(RwLock::new(None)),
102 }
103 }
104 pub fn set_raw_tap(&mut self, tap: Option<mpsc::UnboundedSender<AudioFrame>>) {
105 *self.raw_tap.write().unwrap() = tap;
106 }
107 pub fn set_tap_owner(&mut self, owner: Option<TrackId>) {
109 *self.tap_owner.write().unwrap() = owner;
110 }
111 fn raw_tap(&self) -> Option<mpsc::UnboundedSender<AudioFrame>> {
112 self.raw_tap.read().unwrap().clone()
113 }
114 fn tap_owns(&self, track_id: &str) -> bool {
115 match self.tap_owner.read().unwrap().as_deref() {
116 Some(owner) => owner == track_id,
117 None => true,
118 }
119 }
120 pub fn insert_processor(&mut self, processor: Box<dyn Processor>) {
121 lock_processors(&self.processors).insert(0, processor);
122 }
123 pub fn append_processor(&mut self, processor: Box<dyn Processor>) {
124 lock_processors(&self.processors).push(processor);
125 }
126
127 pub fn has_processor<T: 'static>(&self) -> bool {
128 let processors = lock_processors(&self.processors);
129 processors
130 .iter()
131 .any(|processor| (processor.as_ref() as &dyn Any).is::<T>())
132 }
133
134 pub fn remove_processor<T: 'static>(&self) {
135 let mut processors = lock_processors(&self.processors);
136 processors.retain(|processor| !(processor.as_ref() as &dyn Any).is::<T>());
137 }
138
139 pub fn process_frame(&mut self, frame: &mut AudioFrame) -> Result<()> {
140 let mut processors = lock_processors(&self.processors);
141 if !self.force_decode && processors.is_empty() && self.raw_tap().is_none() {
142 return Ok(());
143 }
144 match &mut frame.samples {
145 Samples::RTP {
146 payload_type,
147 payload,
148 sequence_number,
149 } => {
150 if TrackCodec::is_audio(*payload_type) {
151 let (decoded_sample_rate, channels, samples) =
152 self.codec.decode(*payload_type, &payload);
153 let src_packet = SourcePacket {
154 sequence_number: *sequence_number,
155 payload_type: *payload_type,
156 payload: std::mem::take(payload),
157 };
158 frame.src_packet = Some(src_packet);
159 frame.channels = channels;
160 frame.samples = Samples::PCM { samples };
161 frame.sample_rate = decoded_sample_rate;
162 }
163 }
164 _ => {}
165 }
166
167 if let Some(tap) = self.raw_tap()
172 && self.tap_owns(&frame.track_id)
173 && let Samples::PCM { samples } = &frame.samples
174 && !samples.is_empty()
175 && frame.sample_rate > 0
176 {
177 let mut raw = frame.clone();
178 raw.src_packet = None;
179 let mono = match &mut raw.samples {
180 Samples::PCM { samples } => samples,
181 _ => unreachable!("checked PCM above"),
182 };
183 if raw.channels == 2 {
184 convert_to_mono(mono, 2);
185 raw.channels = 1;
186 }
187 let _ = tap.send(raw);
188 }
189
190 if let Samples::PCM { samples } = &mut frame.samples {
191 if frame.sample_rate != self.sample_rate {
192 let new_samples = self.codec.resample(
193 std::mem::take(samples),
194 frame.sample_rate,
195 self.sample_rate,
196 );
197 *samples = new_samples;
198 frame.sample_rate = self.sample_rate;
199 }
200 if frame.channels == 2 {
201 convert_to_mono(samples, 2);
202 frame.channels = 1;
203 }
204 }
205 for processor in processors.iter_mut() {
207 processor.process_frame(frame)?;
208 }
209 Ok(())
210 }
211}
212
213pub struct SubscribeProcessor {
214 event_sender: EventSender,
215 track_id: String,
216 track_index: u8, }
218
219impl SubscribeProcessor {
220 pub fn new(event_sender: EventSender, track_id: String, track_index: u8) -> Self {
221 Self {
222 event_sender,
223 track_id,
224 track_index,
225 }
226 }
227}
228
229impl Processor for SubscribeProcessor {
230 fn process_frame(&mut self, frame: &mut AudioFrame) -> Result<()> {
231 if let Samples::PCM { samples } = &frame.samples {
232 if !samples.is_empty() {
233 let pcm_data = audio_codec::samples_to_bytes(samples);
234 let mut data = Vec::with_capacity(pcm_data.len() + 1);
235 data.push(self.track_index);
236 data.extend_from_slice(&pcm_data);
237
238 let event = SessionEvent::Binary {
239 track_id: self.track_id.clone(),
240 timestamp: frame.timestamp,
241 data,
242 };
243 self.event_sender.send(event).ok();
244 }
245 }
246 Ok(())
247 }
248}
249
250#[cfg(test)]
251mod tests {
252 use super::*;
253
254 #[test]
260 fn raw_tap_visible_to_clones_taken_before_attach() {
261 let mut chain = ProcessorChain::new(INTERNAL_SAMPLERATE);
262 let mut worker = chain.clone();
263
264 let (tx, mut rx) = mpsc::unbounded_channel();
265 chain.set_raw_tap(Some(tx));
266
267 let mut frame = AudioFrame {
268 track_id: "track".to_string(),
269 samples: Samples::PCM {
270 samples: vec![100i16; 160],
271 },
272 sample_rate: 8000,
273 channels: 1,
274 ..Default::default()
275 };
276 worker.process_frame(&mut frame).unwrap();
277
278 let raw = rx
279 .try_recv()
280 .expect("raw tap should receive a frame from a pre-attach clone");
281 assert_eq!(raw.sample_rate, 8000);
282 assert_eq!(raw.channels, 1);
283 match raw.samples {
284 Samples::PCM { samples } => assert_eq!(samples.len(), 160),
285 _ => panic!("expected PCM samples on the raw tap"),
286 }
287 assert_eq!(frame.sample_rate, INTERNAL_SAMPLERATE);
289 }
290
291 #[test]
294 fn raw_tap_detach_visible_to_clones() {
295 let mut chain = ProcessorChain::new(INTERNAL_SAMPLERATE);
296 let (tx, _rx) = mpsc::unbounded_channel();
297 chain.set_raw_tap(Some(tx));
298 let mut worker = chain.clone();
299 chain.set_raw_tap(None);
300
301 let mut frame = AudioFrame {
302 track_id: "track".to_string(),
303 samples: Samples::PCM {
304 samples: vec![100i16; 160],
305 },
306 sample_rate: 8000,
307 channels: 1,
308 ..Default::default()
309 };
310 worker.process_frame(&mut frame).unwrap();
311 assert_eq!(frame.sample_rate, INTERNAL_SAMPLERATE);
312 }
313
314 #[test]
320 fn raw_tap_only_mirrors_owned_frames() {
321 let mut chain = ProcessorChain::new(INTERNAL_SAMPLERATE);
322 let (tx, mut rx) = mpsc::unbounded_channel();
323 chain.set_raw_tap(Some(tx));
324 chain.set_tap_owner(Some("server-side-track".to_string()));
325
326 let mut relayed = AudioFrame {
329 track_id: "caller".to_string(),
330 samples: Samples::PCM {
331 samples: vec![1i16; 160],
332 },
333 sample_rate: 8000,
334 channels: 1,
335 ..Default::default()
336 };
337 chain.process_frame(&mut relayed).unwrap();
338 assert!(
339 rx.try_recv().is_err(),
340 "relayed frames must not be mirrored to the raw tap"
341 );
342
343 let mut owned = AudioFrame {
345 track_id: "server-side-track".to_string(),
346 samples: Samples::PCM {
347 samples: vec![1i16; 160],
348 },
349 sample_rate: 16000,
350 channels: 1,
351 ..Default::default()
352 };
353 chain.process_frame(&mut owned).unwrap();
354 let raw = rx
355 .try_recv()
356 .expect("owned frames must be mirrored to the raw tap");
357 assert_eq!(raw.track_id, "server-side-track");
358 assert_eq!(raw.sample_rate, 16000);
359 }
360
361 #[test]
365 fn process_frame_tolerates_zero_sample_rate() {
366 let mut chain = ProcessorChain::new(INTERNAL_SAMPLERATE);
367 let mut frame = AudioFrame {
368 track_id: "track".to_string(),
369 samples: Samples::PCM {
370 samples: vec![100i16; 160],
371 },
372 sample_rate: 0,
373 channels: 1,
374 ..Default::default()
375 };
376
377 chain.process_frame(&mut frame).unwrap();
378 assert_eq!(frame.sample_rate, INTERNAL_SAMPLERATE);
379 match frame.samples {
380 Samples::PCM { samples } => assert_eq!(samples.len(), 160),
381 _ => panic!("expected PCM samples"),
382 }
383 }
384}