Skip to main content

active_call/media/
processor.rs

1use super::INTERNAL_SAMPLERATE;
2use super::track::track_codec::TrackCodec;
3use crate::event::{EventSender, SessionEvent};
4use crate::media::{AudioFrame, Samples, SourcePacket, TrackId};
5use anyhow::Result;
6use std::any::Any;
7use std::sync::{Arc, Mutex, RwLock};
8use tokio::sync::mpsc;
9
10pub trait Processor: Send + Sync + Any {
11    fn process_frame(&mut self, frame: &mut AudioFrame) -> Result<()>;
12}
13
14pub fn convert_to_mono(samples: &mut Vec<i16>, channels: u16) {
15    if channels != 2 {
16        return;
17    }
18    let mut i = 0;
19    let mut j = 0;
20    while i < samples.len() {
21        let l = samples[i] as i32;
22        let r = samples[i + 1] as i32;
23        samples[j] = ((l + r) / 2) as i16;
24        i += 2;
25        j += 1;
26    }
27    samples.truncate(j);
28}
29
30impl Default for AudioFrame {
31    fn default() -> Self {
32        Self {
33            track_id: "".to_string(),
34            samples: Samples::Empty,
35            timestamp: 0,
36            sample_rate: 16000,
37            channels: 1,
38            src_packet: None,
39            speech_probability: None,
40        }
41    }
42}
43
44impl Samples {
45    pub fn is_empty(&self) -> bool {
46        match self {
47            Samples::PCM { samples } => samples.is_empty(),
48            Samples::RTP { payload, .. } => payload.is_empty(),
49            Samples::Empty => true,
50        }
51    }
52}
53
54#[derive(Clone)]
55pub struct ProcessorChain {
56    processors: Arc<Mutex<Vec<Box<dyn Processor>>>>,
57    pub codec: TrackCodec,
58    sample_rate: u32,
59    pub force_decode: bool,
60    /// Optional raw tap: when set, frames are mirrored to this channel at
61    /// their native (pre-resample) sample rate right after decoding, before
62    /// the pipeline normalizes them to `INTERNAL_SAMPLERATE`. Used by the
63    /// native-samplerate recorder.
64    ///
65    /// Shared via `Arc<RwLock<..>>`: `RtcTrack::create()` clones the chain
66    /// into its long-lived worker tasks *before* the recorder attaches the
67    /// tap, so a plain field would leave those workers with a stale `None`.
68    pub raw_tap: Arc<RwLock<Option<mpsc::UnboundedSender<AudioFrame>>>>,
69    /// Track id that owns this chain. When set, the raw tap only mirrors
70    /// frames tagged with this id. A track often relays another leg's audio
71    /// (e.g. the media-pass track forwards the caller's audio to the
72    /// websocket), and that relayed copy must not be recorded a second time:
73    /// the originating leg already recorded it.
74    pub tap_owner: Arc<RwLock<Option<TrackId>>>,
75}
76
77/// Lock the processor list, recovering from a poisoned mutex.
78///
79/// A panic inside a processor (or in the decode/resample step) must not
80/// permanently take down the track: the `Vec` itself is never left in an
81/// inconsistent state by unwinding, so reusing the guard is safe and far
82/// better than propagating a `PoisonError` on every subsequent frame.
83///
84/// Takes the mutex directly (rather than `&self`) so the returned guard only
85/// borrows the `processors` field, leaving `self.codec` free to be borrowed
86/// mutably for the decode/resample steps in `process_frame`.
87fn lock_processors(
88    processors: &Mutex<Vec<Box<dyn Processor>>>,
89) -> std::sync::MutexGuard<'_, Vec<Box<dyn Processor>>> {
90    processors.lock().unwrap_or_else(|e| e.into_inner())
91}
92
93impl ProcessorChain {
94    pub fn new(_sample_rate: u32) -> Self {
95        Self {
96            processors: Arc::new(Mutex::new(Vec::new())),
97            codec: TrackCodec::new(),
98            sample_rate: INTERNAL_SAMPLERATE,
99            force_decode: true,
100            raw_tap: Arc::new(RwLock::new(None)),
101            tap_owner: Arc::new(RwLock::new(None)),
102        }
103    }
104    pub fn set_raw_tap(&mut self, tap: Option<mpsc::UnboundedSender<AudioFrame>>) {
105        *self.raw_tap.write().unwrap() = tap;
106    }
107    /// Restrict the raw tap to frames owned by `owner` (see `tap_owner`).
108    pub fn set_tap_owner(&mut self, owner: Option<TrackId>) {
109        *self.tap_owner.write().unwrap() = owner;
110    }
111    fn raw_tap(&self) -> Option<mpsc::UnboundedSender<AudioFrame>> {
112        self.raw_tap.read().unwrap().clone()
113    }
114    fn tap_owns(&self, track_id: &str) -> bool {
115        match self.tap_owner.read().unwrap().as_deref() {
116            Some(owner) => owner == track_id,
117            None => true,
118        }
119    }
120    pub fn insert_processor(&mut self, processor: Box<dyn Processor>) {
121        lock_processors(&self.processors).insert(0, processor);
122    }
123    pub fn append_processor(&mut self, processor: Box<dyn Processor>) {
124        lock_processors(&self.processors).push(processor);
125    }
126
127    pub fn has_processor<T: 'static>(&self) -> bool {
128        let processors = lock_processors(&self.processors);
129        processors
130            .iter()
131            .any(|processor| (processor.as_ref() as &dyn Any).is::<T>())
132    }
133
134    pub fn remove_processor<T: 'static>(&self) {
135        let mut processors = lock_processors(&self.processors);
136        processors.retain(|processor| !(processor.as_ref() as &dyn Any).is::<T>());
137    }
138
139    pub fn process_frame(&mut self, frame: &mut AudioFrame) -> Result<()> {
140        let mut processors = lock_processors(&self.processors);
141        if !self.force_decode && processors.is_empty() && self.raw_tap().is_none() {
142            return Ok(());
143        }
144        match &mut frame.samples {
145            Samples::RTP {
146                payload_type,
147                payload,
148                sequence_number,
149            } => {
150                if TrackCodec::is_audio(*payload_type) {
151                    let (decoded_sample_rate, channels, samples) =
152                        self.codec.decode(*payload_type, &payload);
153                    let src_packet = SourcePacket {
154                        sequence_number: *sequence_number,
155                        payload_type: *payload_type,
156                        payload: std::mem::take(payload),
157                    };
158                    frame.src_packet = Some(src_packet);
159                    frame.channels = channels;
160                    frame.samples = Samples::PCM { samples };
161                    frame.sample_rate = decoded_sample_rate;
162                }
163            }
164            _ => {}
165        }
166
167        // Mirror the frame to the raw tap at its native sample rate, before
168        // the pipeline resamples it to INTERNAL_SAMPLERATE. Only frames owned
169        // by this chain's track are mirrored: audio the track merely relays
170        // for another leg is already recorded at its origin.
171        if let Some(tap) = self.raw_tap()
172            && self.tap_owns(&frame.track_id)
173            && let Samples::PCM { samples } = &frame.samples
174            && !samples.is_empty()
175            && frame.sample_rate > 0
176        {
177            let mut raw = frame.clone();
178            raw.src_packet = None;
179            let mono = match &mut raw.samples {
180                Samples::PCM { samples } => samples,
181                _ => unreachable!("checked PCM above"),
182            };
183            if raw.channels == 2 {
184                convert_to_mono(mono, 2);
185                raw.channels = 1;
186            }
187            let _ = tap.send(raw);
188        }
189
190        if let Samples::PCM { samples } = &mut frame.samples {
191            if frame.sample_rate != self.sample_rate {
192                let new_samples = self.codec.resample(
193                    std::mem::take(samples),
194                    frame.sample_rate,
195                    self.sample_rate,
196                );
197                *samples = new_samples;
198                frame.sample_rate = self.sample_rate;
199            }
200            if frame.channels == 2 {
201                convert_to_mono(samples, 2);
202                frame.channels = 1;
203            }
204        }
205        // Process the frame with all processors
206        for processor in processors.iter_mut() {
207            processor.process_frame(frame)?;
208        }
209        Ok(())
210    }
211}
212
213pub struct SubscribeProcessor {
214    event_sender: EventSender,
215    track_id: String,
216    track_index: u8, // 0 for caller, 1 for callee
217}
218
219impl SubscribeProcessor {
220    pub fn new(event_sender: EventSender, track_id: String, track_index: u8) -> Self {
221        Self {
222            event_sender,
223            track_id,
224            track_index,
225        }
226    }
227}
228
229impl Processor for SubscribeProcessor {
230    fn process_frame(&mut self, frame: &mut AudioFrame) -> Result<()> {
231        if let Samples::PCM { samples } = &frame.samples {
232            if !samples.is_empty() {
233                let pcm_data = audio_codec::samples_to_bytes(samples);
234                let mut data = Vec::with_capacity(pcm_data.len() + 1);
235                data.push(self.track_index);
236                data.extend_from_slice(&pcm_data);
237
238                let event = SessionEvent::Binary {
239                    track_id: self.track_id.clone(),
240                    timestamp: frame.timestamp,
241                    data,
242                };
243                self.event_sender.send(event).ok();
244            }
245        }
246        Ok(())
247    }
248}
249
250#[cfg(test)]
251mod tests {
252    use super::*;
253
254    /// Regression: `RtcTrack::create()` clones the chain into its worker
255    /// tasks *before* the recorder attaches the raw tap (native-samplerate
256    /// recording). The tap must be shared state so clones taken before the
257    /// attach observe it; with a plain `Option` field this test fails and
258    /// SIP/RTP calls silently fall back to the 16 kHz recorder.
259    #[test]
260    fn raw_tap_visible_to_clones_taken_before_attach() {
261        let mut chain = ProcessorChain::new(INTERNAL_SAMPLERATE);
262        let mut worker = chain.clone();
263
264        let (tx, mut rx) = mpsc::unbounded_channel();
265        chain.set_raw_tap(Some(tx));
266
267        let mut frame = AudioFrame {
268            track_id: "track".to_string(),
269            samples: Samples::PCM {
270                samples: vec![100i16; 160],
271            },
272            sample_rate: 8000,
273            channels: 1,
274            ..Default::default()
275        };
276        worker.process_frame(&mut frame).unwrap();
277
278        let raw = rx
279            .try_recv()
280            .expect("raw tap should receive a frame from a pre-attach clone");
281        assert_eq!(raw.sample_rate, 8000);
282        assert_eq!(raw.channels, 1);
283        match raw.samples {
284            Samples::PCM { samples } => assert_eq!(samples.len(), 160),
285            _ => panic!("expected PCM samples on the raw tap"),
286        }
287        // The pipeline output is still normalized to the internal rate.
288        assert_eq!(frame.sample_rate, INTERNAL_SAMPLERATE);
289    }
290
291    /// Detaching the tap on the original chain must be observed by clones as
292    /// well (e.g. recorder restart swapping the sender).
293    #[test]
294    fn raw_tap_detach_visible_to_clones() {
295        let mut chain = ProcessorChain::new(INTERNAL_SAMPLERATE);
296        let (tx, _rx) = mpsc::unbounded_channel();
297        chain.set_raw_tap(Some(tx));
298        let mut worker = chain.clone();
299        chain.set_raw_tap(None);
300
301        let mut frame = AudioFrame {
302            track_id: "track".to_string(),
303            samples: Samples::PCM {
304                samples: vec![100i16; 160],
305            },
306            sample_rate: 8000,
307            channels: 1,
308            ..Default::default()
309        };
310        worker.process_frame(&mut frame).unwrap();
311        assert_eq!(frame.sample_rate, INTERNAL_SAMPLERATE);
312    }
313
314    /// Regression: a track often relays another leg's audio through its own
315    /// processor chain (the media-pass track forwards the caller's audio to the
316    /// websocket). The native-samplerate raw tap must mirror only frames the
317    /// chain owns, otherwise a media-pass call records the caller's audio from
318    /// both legs and the recording comes out twice as long.
319    #[test]
320    fn raw_tap_only_mirrors_owned_frames() {
321        let mut chain = ProcessorChain::new(INTERNAL_SAMPLERATE);
322        let (tx, mut rx) = mpsc::unbounded_channel();
323        chain.set_raw_tap(Some(tx));
324        chain.set_tap_owner(Some("server-side-track".to_string()));
325
326        // A frame forwarded from another leg (caller -> websocket) must not be
327        // mirrored by this chain.
328        let mut relayed = AudioFrame {
329            track_id: "caller".to_string(),
330            samples: Samples::PCM {
331                samples: vec![1i16; 160],
332            },
333            sample_rate: 8000,
334            channels: 1,
335            ..Default::default()
336        };
337        chain.process_frame(&mut relayed).unwrap();
338        assert!(
339            rx.try_recv().is_err(),
340            "relayed frames must not be mirrored to the raw tap"
341        );
342
343        // The chain's own frame must be mirrored.
344        let mut owned = AudioFrame {
345            track_id: "server-side-track".to_string(),
346            samples: Samples::PCM {
347                samples: vec![1i16; 160],
348            },
349            sample_rate: 16000,
350            channels: 1,
351            ..Default::default()
352        };
353        chain.process_frame(&mut owned).unwrap();
354        let raw = rx
355            .try_recv()
356            .expect("owned frames must be mirrored to the raw tap");
357        assert_eq!(raw.track_id, "server-side-track");
358        assert_eq!(raw.sample_rate, 16000);
359    }
360
361    /// Regression: a track report with no valid source rate (e.g. a media-pass
362    /// track whose `input_sample_rate` is 0) must not panic by trying to build
363    /// a resampler from 0 Hz, which previously poisoned the processor mutex.
364    #[test]
365    fn process_frame_tolerates_zero_sample_rate() {
366        let mut chain = ProcessorChain::new(INTERNAL_SAMPLERATE);
367        let mut frame = AudioFrame {
368            track_id: "track".to_string(),
369            samples: Samples::PCM {
370                samples: vec![100i16; 160],
371            },
372            sample_rate: 0,
373            channels: 1,
374            ..Default::default()
375        };
376
377        chain.process_frame(&mut frame).unwrap();
378        assert_eq!(frame.sample_rate, INTERNAL_SAMPLERATE);
379        match frame.samples {
380            Samples::PCM { samples } => assert_eq!(samples.len(), 160),
381            _ => panic!("expected PCM samples"),
382        }
383    }
384}