1use super::INTERNAL_SAMPLERATE;
2use super::track::track_codec::TrackCodec;
3use crate::event::{EventSender, SessionEvent};
4use crate::media::{AudioFrame, Samples, SourcePacket, TrackId};
5use anyhow::Result;
6use std::any::Any;
7use std::sync::{Arc, Mutex, RwLock};
8use tokio::sync::mpsc;
9
10pub trait Processor: Send + Sync + Any {
11 fn process_frame(&mut self, frame: &mut AudioFrame) -> Result<()>;
12}
13
14pub fn convert_to_mono(samples: &mut Vec<i16>, channels: u16) {
15 if channels != 2 {
16 return;
17 }
18 let mut i = 0;
19 let mut j = 0;
20 while i < samples.len() {
21 let l = samples[i] as i32;
22 let r = samples[i + 1] as i32;
23 samples[j] = ((l + r) / 2) as i16;
24 i += 2;
25 j += 1;
26 }
27 samples.truncate(j);
28}
29
30impl Default for AudioFrame {
31 fn default() -> Self {
32 Self {
33 track_id: "".to_string(),
34 samples: Samples::Empty,
35 timestamp: 0,
36 sample_rate: 16000,
37 channels: 1,
38 src_packet: None,
39 speech_probability: None,
40 }
41 }
42}
43
44impl Samples {
45 pub fn is_empty(&self) -> bool {
46 match self {
47 Samples::PCM { samples } => samples.is_empty(),
48 Samples::RTP { payload, .. } => payload.is_empty(),
49 Samples::Dtmf { .. } => false,
50 Samples::Empty => true,
51 }
52 }
53}
54
55#[derive(Clone)]
56pub struct ProcessorChain {
57 processors: Arc<Mutex<Vec<Box<dyn Processor>>>>,
58 pub codec: TrackCodec,
59 sample_rate: u32,
60 pub force_decode: bool,
61 pub raw_tap: Arc<RwLock<Option<mpsc::UnboundedSender<AudioFrame>>>>,
70 pub tap_owner: Arc<RwLock<Option<TrackId>>>,
76}
77
78fn lock_processors(
89 processors: &Mutex<Vec<Box<dyn Processor>>>,
90) -> std::sync::MutexGuard<'_, Vec<Box<dyn Processor>>> {
91 processors.lock().unwrap_or_else(|e| e.into_inner())
92}
93
94impl ProcessorChain {
95 pub fn new(_sample_rate: u32) -> Self {
96 Self {
97 processors: Arc::new(Mutex::new(Vec::new())),
98 codec: TrackCodec::new(),
99 sample_rate: INTERNAL_SAMPLERATE,
100 force_decode: true,
101 raw_tap: Arc::new(RwLock::new(None)),
102 tap_owner: Arc::new(RwLock::new(None)),
103 }
104 }
105 pub fn set_raw_tap(&mut self, tap: Option<mpsc::UnboundedSender<AudioFrame>>) {
106 *self.raw_tap.write().unwrap() = tap;
107 }
108 pub fn set_tap_owner(&mut self, owner: Option<TrackId>) {
110 *self.tap_owner.write().unwrap() = owner;
111 }
112 fn raw_tap(&self) -> Option<mpsc::UnboundedSender<AudioFrame>> {
113 self.raw_tap.read().unwrap().clone()
114 }
115 fn tap_owns(&self, track_id: &str) -> bool {
116 match self.tap_owner.read().unwrap().as_deref() {
117 Some(owner) => owner == track_id,
118 None => true,
119 }
120 }
121 pub fn insert_processor(&mut self, processor: Box<dyn Processor>) {
122 lock_processors(&self.processors).insert(0, processor);
123 }
124 pub fn append_processor(&mut self, processor: Box<dyn Processor>) {
125 lock_processors(&self.processors).push(processor);
126 }
127
128 pub fn has_processor<T: 'static>(&self) -> bool {
129 let processors = lock_processors(&self.processors);
130 processors
131 .iter()
132 .any(|processor| (processor.as_ref() as &dyn Any).is::<T>())
133 }
134
135 pub fn remove_processor<T: 'static>(&self) {
136 let mut processors = lock_processors(&self.processors);
137 processors.retain(|processor| !(processor.as_ref() as &dyn Any).is::<T>());
138 }
139
140 pub fn process_frame(&mut self, frame: &mut AudioFrame) -> Result<()> {
141 let mut processors = lock_processors(&self.processors);
142 if !self.force_decode && processors.is_empty() && self.raw_tap().is_none() {
143 return Ok(());
144 }
145 match &mut frame.samples {
146 Samples::RTP {
147 payload_type,
148 payload,
149 sequence_number,
150 } => {
151 if self
154 .codec
155 .get_codec_for_pt(*payload_type)
156 .is_some_and(|codec| codec.is_audio())
157 {
158 let (decoded_sample_rate, channels, samples) =
159 self.codec.decode(*payload_type, &payload);
160 let src_packet = SourcePacket {
161 sequence_number: *sequence_number,
162 payload_type: *payload_type,
163 payload: std::mem::take(payload),
164 };
165 frame.src_packet = Some(src_packet);
166 frame.channels = channels;
167 frame.samples = Samples::PCM { samples };
168 frame.sample_rate = decoded_sample_rate;
169 }
170 }
171 _ => {}
172 }
173
174 if let Some(tap) = self.raw_tap()
179 && self.tap_owns(&frame.track_id)
180 && let Samples::PCM { samples } = &frame.samples
181 && !samples.is_empty()
182 && frame.sample_rate > 0
183 {
184 let mut raw = frame.clone();
185 raw.src_packet = None;
186 let mono = match &mut raw.samples {
187 Samples::PCM { samples } => samples,
188 _ => unreachable!("checked PCM above"),
189 };
190 if raw.channels == 2 {
191 convert_to_mono(mono, 2);
192 raw.channels = 1;
193 }
194 let _ = tap.send(raw);
195 }
196
197 if let Samples::PCM { samples } = &mut frame.samples {
198 if frame.sample_rate != self.sample_rate {
199 let new_samples = self.codec.resample(
200 std::mem::take(samples),
201 frame.sample_rate,
202 self.sample_rate,
203 );
204 *samples = new_samples;
205 frame.sample_rate = self.sample_rate;
206 }
207 if frame.channels == 2 {
208 convert_to_mono(samples, 2);
209 frame.channels = 1;
210 }
211 }
212 for processor in processors.iter_mut() {
214 processor.process_frame(frame)?;
215 }
216 Ok(())
217 }
218}
219
220pub struct SubscribeProcessor {
221 event_sender: EventSender,
222 track_id: String,
223 owner: TrackId,
229 track_index: u8, }
231
232impl SubscribeProcessor {
233 pub fn new(event_sender: EventSender, track_id: String, owner: TrackId, track_index: u8) -> Self {
234 Self {
235 event_sender,
236 track_id,
237 owner,
238 track_index,
239 }
240 }
241}
242
243impl Processor for SubscribeProcessor {
244 fn process_frame(&mut self, frame: &mut AudioFrame) -> Result<()> {
245 if frame.track_id != self.owner {
246 return Ok(());
247 }
248 if let Samples::PCM { samples } = &frame.samples {
249 if !samples.is_empty() {
250 let pcm_data = audio_codec::samples_to_bytes(samples);
251 let mut data = Vec::with_capacity(pcm_data.len() + 1);
252 data.push(self.track_index);
253 data.extend_from_slice(&pcm_data);
254
255 let event = SessionEvent::Binary {
256 track_id: self.track_id.clone(),
257 timestamp: frame.timestamp,
258 data,
259 };
260 self.event_sender.send(event).ok();
261 }
262 }
263 Ok(())
264 }
265}
266
267#[cfg(test)]
268mod tests {
269 use super::*;
270
271 #[test]
277 fn raw_tap_visible_to_clones_taken_before_attach() {
278 let mut chain = ProcessorChain::new(INTERNAL_SAMPLERATE);
279 let mut worker = chain.clone();
280
281 let (tx, mut rx) = mpsc::unbounded_channel();
282 chain.set_raw_tap(Some(tx));
283
284 let mut frame = AudioFrame {
285 track_id: "track".to_string(),
286 samples: Samples::PCM {
287 samples: vec![100i16; 160],
288 },
289 sample_rate: 8000,
290 channels: 1,
291 ..Default::default()
292 };
293 worker.process_frame(&mut frame).unwrap();
294
295 let raw = rx
296 .try_recv()
297 .expect("raw tap should receive a frame from a pre-attach clone");
298 assert_eq!(raw.sample_rate, 8000);
299 assert_eq!(raw.channels, 1);
300 match raw.samples {
301 Samples::PCM { samples } => assert_eq!(samples.len(), 160),
302 _ => panic!("expected PCM samples on the raw tap"),
303 }
304 assert_eq!(frame.sample_rate, INTERNAL_SAMPLERATE);
306 }
307
308 #[test]
311 fn raw_tap_detach_visible_to_clones() {
312 let mut chain = ProcessorChain::new(INTERNAL_SAMPLERATE);
313 let (tx, _rx) = mpsc::unbounded_channel();
314 chain.set_raw_tap(Some(tx));
315 let mut worker = chain.clone();
316 chain.set_raw_tap(None);
317
318 let mut frame = AudioFrame {
319 track_id: "track".to_string(),
320 samples: Samples::PCM {
321 samples: vec![100i16; 160],
322 },
323 sample_rate: 8000,
324 channels: 1,
325 ..Default::default()
326 };
327 worker.process_frame(&mut frame).unwrap();
328 assert_eq!(frame.sample_rate, INTERNAL_SAMPLERATE);
329 }
330
331 #[test]
337 fn raw_tap_only_mirrors_owned_frames() {
338 let mut chain = ProcessorChain::new(INTERNAL_SAMPLERATE);
339 let (tx, mut rx) = mpsc::unbounded_channel();
340 chain.set_raw_tap(Some(tx));
341 chain.set_tap_owner(Some("server-side-track".to_string()));
342
343 let mut relayed = AudioFrame {
346 track_id: "caller".to_string(),
347 samples: Samples::PCM {
348 samples: vec![1i16; 160],
349 },
350 sample_rate: 8000,
351 channels: 1,
352 ..Default::default()
353 };
354 chain.process_frame(&mut relayed).unwrap();
355 assert!(
356 rx.try_recv().is_err(),
357 "relayed frames must not be mirrored to the raw tap"
358 );
359
360 let mut owned = AudioFrame {
362 track_id: "server-side-track".to_string(),
363 samples: Samples::PCM {
364 samples: vec![1i16; 160],
365 },
366 sample_rate: 16000,
367 channels: 1,
368 ..Default::default()
369 };
370 chain.process_frame(&mut owned).unwrap();
371 let raw = rx
372 .try_recv()
373 .expect("owned frames must be mirrored to the raw tap");
374 assert_eq!(raw.track_id, "server-side-track");
375 assert_eq!(raw.sample_rate, 16000);
376 }
377
378 #[test]
382 fn process_frame_tolerates_zero_sample_rate() {
383 let mut chain = ProcessorChain::new(INTERNAL_SAMPLERATE);
384 let mut frame = AudioFrame {
385 track_id: "track".to_string(),
386 samples: Samples::PCM {
387 samples: vec![100i16; 160],
388 },
389 sample_rate: 0,
390 channels: 1,
391 ..Default::default()
392 };
393
394 chain.process_frame(&mut frame).unwrap();
395 assert_eq!(frame.sample_rate, INTERNAL_SAMPLERATE);
396 match frame.samples {
397 Samples::PCM { samples } => assert_eq!(samples.len(), 160),
398 _ => panic!("expected PCM samples"),
399 }
400 }
401}