1use super::INTERNAL_SAMPLERATE;
2use super::track::track_codec::TrackCodec;
3use crate::event::{EventSender, SessionEvent};
4use crate::media::{AudioFrame, Samples, SourcePacket};
5use anyhow::Result;
6use std::any::Any;
7use std::sync::{Arc, Mutex, RwLock};
8use tokio::sync::mpsc;
9
10pub trait Processor: Send + Sync + Any {
11 fn process_frame(&mut self, frame: &mut AudioFrame) -> Result<()>;
12}
13
14pub fn convert_to_mono(samples: &mut Vec<i16>, channels: u16) {
15 if channels != 2 {
16 return;
17 }
18 let mut i = 0;
19 let mut j = 0;
20 while i < samples.len() {
21 let l = samples[i] as i32;
22 let r = samples[i + 1] as i32;
23 samples[j] = ((l + r) / 2) as i16;
24 i += 2;
25 j += 1;
26 }
27 samples.truncate(j);
28}
29
30impl Default for AudioFrame {
31 fn default() -> Self {
32 Self {
33 track_id: "".to_string(),
34 samples: Samples::Empty,
35 timestamp: 0,
36 sample_rate: 16000,
37 channels: 1,
38 src_packet: None,
39 speech_probability: None,
40 }
41 }
42}
43
44impl Samples {
45 pub fn is_empty(&self) -> bool {
46 match self {
47 Samples::PCM { samples } => samples.is_empty(),
48 Samples::RTP { payload, .. } => payload.is_empty(),
49 Samples::Dtmf { .. } => false,
50 Samples::Empty => true,
51 }
52 }
53}
54
55#[derive(Clone)]
56pub struct ProcessorChain {
57 processors: Arc<Mutex<Vec<Box<dyn Processor>>>>,
58 pub codec: TrackCodec,
59 sample_rate: u32,
60 pub force_decode: bool,
61 pub raw_tap: Arc<RwLock<Option<mpsc::UnboundedSender<AudioFrame>>>>,
70}
71
72fn lock_processors(
83 processors: &Mutex<Vec<Box<dyn Processor>>>,
84) -> std::sync::MutexGuard<'_, Vec<Box<dyn Processor>>> {
85 processors.lock().unwrap_or_else(|e| e.into_inner())
86}
87
88impl ProcessorChain {
89 pub fn new(_sample_rate: u32) -> Self {
90 Self {
91 processors: Arc::new(Mutex::new(Vec::new())),
92 codec: TrackCodec::new(),
93 sample_rate: INTERNAL_SAMPLERATE,
94 force_decode: true,
95 raw_tap: Arc::new(RwLock::new(None)),
96 }
97 }
98 pub fn set_raw_tap(&mut self, tap: Option<mpsc::UnboundedSender<AudioFrame>>) {
99 *self.raw_tap.write().unwrap() = tap;
100 }
101 fn raw_tap(&self) -> Option<mpsc::UnboundedSender<AudioFrame>> {
102 self.raw_tap.read().unwrap().clone()
103 }
104 pub fn insert_processor(&mut self, processor: Box<dyn Processor>) {
105 lock_processors(&self.processors).insert(0, processor);
106 }
107 pub fn append_processor(&mut self, processor: Box<dyn Processor>) {
108 lock_processors(&self.processors).push(processor);
109 }
110
111 pub fn has_processor<T: 'static>(&self) -> bool {
112 let processors = lock_processors(&self.processors);
113 processors
114 .iter()
115 .any(|processor| (processor.as_ref() as &dyn Any).is::<T>())
116 }
117
118 pub fn remove_processor<T: 'static>(&self) {
119 let mut processors = lock_processors(&self.processors);
120 processors.retain(|processor| !(processor.as_ref() as &dyn Any).is::<T>());
121 }
122
123 pub fn process_frame(&mut self, frame: &mut AudioFrame) -> Result<()> {
124 let mut processors = lock_processors(&self.processors);
125 if !self.force_decode && processors.is_empty() && self.raw_tap().is_none() {
126 return Ok(());
127 }
128 match &mut frame.samples {
129 Samples::RTP {
130 payload_type,
131 payload,
132 sequence_number,
133 } => {
134 if self
137 .codec
138 .get_codec_for_pt(*payload_type)
139 .is_some_and(|codec| codec.is_audio())
140 {
141 let (decoded_sample_rate, channels, samples) =
142 self.codec.decode(*payload_type, &payload);
143 let src_packet = SourcePacket {
144 sequence_number: *sequence_number,
145 payload_type: *payload_type,
146 payload: std::mem::take(payload),
147 };
148 frame.src_packet = Some(src_packet);
149 frame.channels = channels;
150 frame.samples = Samples::PCM { samples };
151 frame.sample_rate = decoded_sample_rate;
152 }
153 }
154 _ => {}
155 }
156
157 if let Some(tap) = self.raw_tap()
160 && let Samples::PCM { samples } = &frame.samples
161 && !samples.is_empty()
162 && frame.sample_rate > 0
163 {
164 let mut raw = frame.clone();
165 raw.src_packet = None;
166 let mono = match &mut raw.samples {
167 Samples::PCM { samples } => samples,
168 _ => unreachable!("checked PCM above"),
169 };
170 if raw.channels == 2 {
171 convert_to_mono(mono, 2);
172 raw.channels = 1;
173 }
174 let _ = tap.send(raw);
175 }
176
177 if let Samples::PCM { samples } = &mut frame.samples {
178 if frame.sample_rate != self.sample_rate {
179 let new_samples = self.codec.resample(
180 std::mem::take(samples),
181 frame.sample_rate,
182 self.sample_rate,
183 );
184 *samples = new_samples;
185 frame.sample_rate = self.sample_rate;
186 }
187 if frame.channels == 2 {
188 convert_to_mono(samples, 2);
189 frame.channels = 1;
190 }
191 }
192 for processor in processors.iter_mut() {
194 processor.process_frame(frame)?;
195 }
196 Ok(())
197 }
198}
199
200pub struct SubscribeProcessor {
201 event_sender: EventSender,
202 track_id: String,
203 track_index: u8, }
205
206impl SubscribeProcessor {
207 pub fn new(event_sender: EventSender, track_id: String, track_index: u8) -> Self {
208 Self {
209 event_sender,
210 track_id,
211 track_index,
212 }
213 }
214}
215
216impl Processor for SubscribeProcessor {
217 fn process_frame(&mut self, frame: &mut AudioFrame) -> Result<()> {
218 if let Samples::PCM { samples } = &frame.samples {
219 if !samples.is_empty() {
220 let pcm_data = audio_codec::samples_to_bytes(samples);
221 let mut data = Vec::with_capacity(pcm_data.len() + 1);
222 data.push(self.track_index);
223 data.extend_from_slice(&pcm_data);
224
225 let event = SessionEvent::Binary {
226 track_id: self.track_id.clone(),
227 timestamp: frame.timestamp,
228 data,
229 };
230 self.event_sender.send(event).ok();
231 }
232 }
233 Ok(())
234 }
235}
236
237#[cfg(test)]
238mod tests {
239 use super::*;
240
241 #[test]
247 fn raw_tap_visible_to_clones_taken_before_attach() {
248 let mut chain = ProcessorChain::new(INTERNAL_SAMPLERATE);
249 let mut worker = chain.clone();
250
251 let (tx, mut rx) = mpsc::unbounded_channel();
252 chain.set_raw_tap(Some(tx));
253
254 let mut frame = AudioFrame {
255 track_id: "track".to_string(),
256 samples: Samples::PCM {
257 samples: vec![100i16; 160],
258 },
259 sample_rate: 8000,
260 channels: 1,
261 ..Default::default()
262 };
263 worker.process_frame(&mut frame).unwrap();
264
265 let raw = rx
266 .try_recv()
267 .expect("raw tap should receive a frame from a pre-attach clone");
268 assert_eq!(raw.sample_rate, 8000);
269 assert_eq!(raw.channels, 1);
270 match raw.samples {
271 Samples::PCM { samples } => assert_eq!(samples.len(), 160),
272 _ => panic!("expected PCM samples on the raw tap"),
273 }
274 assert_eq!(frame.sample_rate, INTERNAL_SAMPLERATE);
276 }
277
278 #[test]
281 fn raw_tap_detach_visible_to_clones() {
282 let mut chain = ProcessorChain::new(INTERNAL_SAMPLERATE);
283 let (tx, _rx) = mpsc::unbounded_channel();
284 chain.set_raw_tap(Some(tx));
285 let mut worker = chain.clone();
286 chain.set_raw_tap(None);
287
288 let mut frame = AudioFrame {
289 track_id: "track".to_string(),
290 samples: Samples::PCM {
291 samples: vec![100i16; 160],
292 },
293 sample_rate: 8000,
294 channels: 1,
295 ..Default::default()
296 };
297 worker.process_frame(&mut frame).unwrap();
298 assert_eq!(frame.sample_rate, INTERNAL_SAMPLERATE);
299 }
300
301 #[test]
305 fn process_frame_tolerates_zero_sample_rate() {
306 let mut chain = ProcessorChain::new(INTERNAL_SAMPLERATE);
307 let mut frame = AudioFrame {
308 track_id: "track".to_string(),
309 samples: Samples::PCM {
310 samples: vec![100i16; 160],
311 },
312 sample_rate: 0,
313 channels: 1,
314 ..Default::default()
315 };
316
317 chain.process_frame(&mut frame).unwrap();
318 assert_eq!(frame.sample_rate, INTERNAL_SAMPLERATE);
319 match frame.samples {
320 Samples::PCM { samples } => assert_eq!(samples.len(), 160),
321 _ => panic!("expected PCM samples"),
322 }
323 }
324}