Skip to main content

active_call/media/
processor.rs

1use super::INTERNAL_SAMPLERATE;
2use super::track::track_codec::TrackCodec;
3use crate::event::{EventSender, SessionEvent};
4use crate::media::{AudioFrame, Samples, SourcePacket};
5use anyhow::Result;
6use std::any::Any;
7use std::sync::{Arc, Mutex, RwLock};
8use tokio::sync::mpsc;
9
10pub trait Processor: Send + Sync + Any {
11    fn process_frame(&mut self, frame: &mut AudioFrame) -> Result<()>;
12}
13
14pub fn convert_to_mono(samples: &mut Vec<i16>, channels: u16) {
15    if channels != 2 {
16        return;
17    }
18    let mut i = 0;
19    let mut j = 0;
20    while i < samples.len() {
21        let l = samples[i] as i32;
22        let r = samples[i + 1] as i32;
23        samples[j] = ((l + r) / 2) as i16;
24        i += 2;
25        j += 1;
26    }
27    samples.truncate(j);
28}
29
30impl Default for AudioFrame {
31    fn default() -> Self {
32        Self {
33            track_id: "".to_string(),
34            samples: Samples::Empty,
35            timestamp: 0,
36            sample_rate: 16000,
37            channels: 1,
38            src_packet: None,
39            speech_probability: None,
40        }
41    }
42}
43
44impl Samples {
45    pub fn is_empty(&self) -> bool {
46        match self {
47            Samples::PCM { samples } => samples.is_empty(),
48            Samples::RTP { payload, .. } => payload.is_empty(),
49            Samples::Dtmf { .. } => false,
50            Samples::Empty => true,
51        }
52    }
53}
54
55#[derive(Clone)]
56pub struct ProcessorChain {
57    processors: Arc<Mutex<Vec<Box<dyn Processor>>>>,
58    pub codec: TrackCodec,
59    sample_rate: u32,
60    pub force_decode: bool,
61    /// Optional raw tap: when set, frames are mirrored to this channel at
62    /// their native (pre-resample) sample rate right after decoding, before
63    /// the pipeline normalizes them to `INTERNAL_SAMPLERATE`. Used by the
64    /// native-samplerate recorder.
65    ///
66    /// Shared via `Arc<RwLock<..>>`: `RtcTrack::create()` clones the chain
67    /// into its long-lived worker tasks *before* the recorder attaches the
68    /// tap, so a plain field would leave those workers with a stale `None`.
69    pub raw_tap: Arc<RwLock<Option<mpsc::UnboundedSender<AudioFrame>>>>,
70}
71
72/// Lock the processor list, recovering from a poisoned mutex.
73///
74/// A panic inside a processor (or in the decode/resample step) must not
75/// permanently take down the track: the `Vec` itself is never left in an
76/// inconsistent state by unwinding, so reusing the guard is safe and far
77/// better than propagating a `PoisonError` on every subsequent frame.
78///
79/// Takes the mutex directly (rather than `&self`) so the returned guard only
80/// borrows the `processors` field, leaving `self.codec` free to be borrowed
81/// mutably for the decode/resample steps in `process_frame`.
82fn lock_processors(
83    processors: &Mutex<Vec<Box<dyn Processor>>>,
84) -> std::sync::MutexGuard<'_, Vec<Box<dyn Processor>>> {
85    processors.lock().unwrap_or_else(|e| e.into_inner())
86}
87
88impl ProcessorChain {
89    pub fn new(_sample_rate: u32) -> Self {
90        Self {
91            processors: Arc::new(Mutex::new(Vec::new())),
92            codec: TrackCodec::new(),
93            sample_rate: INTERNAL_SAMPLERATE,
94            force_decode: true,
95            raw_tap: Arc::new(RwLock::new(None)),
96        }
97    }
98    pub fn set_raw_tap(&mut self, tap: Option<mpsc::UnboundedSender<AudioFrame>>) {
99        *self.raw_tap.write().unwrap() = tap;
100    }
101    fn raw_tap(&self) -> Option<mpsc::UnboundedSender<AudioFrame>> {
102        self.raw_tap.read().unwrap().clone()
103    }
104    pub fn insert_processor(&mut self, processor: Box<dyn Processor>) {
105        lock_processors(&self.processors).insert(0, processor);
106    }
107    pub fn append_processor(&mut self, processor: Box<dyn Processor>) {
108        lock_processors(&self.processors).push(processor);
109    }
110
111    pub fn has_processor<T: 'static>(&self) -> bool {
112        let processors = lock_processors(&self.processors);
113        processors
114            .iter()
115            .any(|processor| (processor.as_ref() as &dyn Any).is::<T>())
116    }
117
118    pub fn remove_processor<T: 'static>(&self) {
119        let mut processors = lock_processors(&self.processors);
120        processors.retain(|processor| !(processor.as_ref() as &dyn Any).is::<T>());
121    }
122
123    pub fn process_frame(&mut self, frame: &mut AudioFrame) -> Result<()> {
124        let mut processors = lock_processors(&self.processors);
125        if !self.force_decode && processors.is_empty() && self.raw_tap().is_none() {
126            return Ok(());
127        }
128        match &mut frame.samples {
129            Samples::RTP {
130                payload_type,
131                payload,
132                sequence_number,
133            } => {
134                // Decode only payload types negotiated as an audio codec;
135                // telephone-event (on any payload type) stays RTP.
136                if self
137                    .codec
138                    .get_codec_for_pt(*payload_type)
139                    .is_some_and(|codec| codec.is_audio())
140                {
141                    let (decoded_sample_rate, channels, samples) =
142                        self.codec.decode(*payload_type, &payload);
143                    let src_packet = SourcePacket {
144                        sequence_number: *sequence_number,
145                        payload_type: *payload_type,
146                        payload: std::mem::take(payload),
147                    };
148                    frame.src_packet = Some(src_packet);
149                    frame.channels = channels;
150                    frame.samples = Samples::PCM { samples };
151                    frame.sample_rate = decoded_sample_rate;
152                }
153            }
154            _ => {}
155        }
156
157        // Mirror the frame to the raw tap at its native sample rate, before
158        // the pipeline resamples it to INTERNAL_SAMPLERATE.
159        if let Some(tap) = self.raw_tap()
160            && let Samples::PCM { samples } = &frame.samples
161            && !samples.is_empty()
162            && frame.sample_rate > 0
163        {
164            let mut raw = frame.clone();
165            raw.src_packet = None;
166            let mono = match &mut raw.samples {
167                Samples::PCM { samples } => samples,
168                _ => unreachable!("checked PCM above"),
169            };
170            if raw.channels == 2 {
171                convert_to_mono(mono, 2);
172                raw.channels = 1;
173            }
174            let _ = tap.send(raw);
175        }
176
177        if let Samples::PCM { samples } = &mut frame.samples {
178            if frame.sample_rate != self.sample_rate {
179                let new_samples = self.codec.resample(
180                    std::mem::take(samples),
181                    frame.sample_rate,
182                    self.sample_rate,
183                );
184                *samples = new_samples;
185                frame.sample_rate = self.sample_rate;
186            }
187            if frame.channels == 2 {
188                convert_to_mono(samples, 2);
189                frame.channels = 1;
190            }
191        }
192        // Process the frame with all processors
193        for processor in processors.iter_mut() {
194            processor.process_frame(frame)?;
195        }
196        Ok(())
197    }
198}
199
200pub struct SubscribeProcessor {
201    event_sender: EventSender,
202    track_id: String,
203    track_index: u8, // 0 for caller, 1 for callee
204}
205
206impl SubscribeProcessor {
207    pub fn new(event_sender: EventSender, track_id: String, track_index: u8) -> Self {
208        Self {
209            event_sender,
210            track_id,
211            track_index,
212        }
213    }
214}
215
216impl Processor for SubscribeProcessor {
217    fn process_frame(&mut self, frame: &mut AudioFrame) -> Result<()> {
218        if let Samples::PCM { samples } = &frame.samples {
219            if !samples.is_empty() {
220                let pcm_data = audio_codec::samples_to_bytes(samples);
221                let mut data = Vec::with_capacity(pcm_data.len() + 1);
222                data.push(self.track_index);
223                data.extend_from_slice(&pcm_data);
224
225                let event = SessionEvent::Binary {
226                    track_id: self.track_id.clone(),
227                    timestamp: frame.timestamp,
228                    data,
229                };
230                self.event_sender.send(event).ok();
231            }
232        }
233        Ok(())
234    }
235}
236
237#[cfg(test)]
238mod tests {
239    use super::*;
240
241    /// Regression: `RtcTrack::create()` clones the chain into its worker
242    /// tasks *before* the recorder attaches the raw tap (native-samplerate
243    /// recording). The tap must be shared state so clones taken before the
244    /// attach observe it; with a plain `Option` field this test fails and
245    /// SIP/RTP calls silently fall back to the 16 kHz recorder.
246    #[test]
247    fn raw_tap_visible_to_clones_taken_before_attach() {
248        let mut chain = ProcessorChain::new(INTERNAL_SAMPLERATE);
249        let mut worker = chain.clone();
250
251        let (tx, mut rx) = mpsc::unbounded_channel();
252        chain.set_raw_tap(Some(tx));
253
254        let mut frame = AudioFrame {
255            track_id: "track".to_string(),
256            samples: Samples::PCM {
257                samples: vec![100i16; 160],
258            },
259            sample_rate: 8000,
260            channels: 1,
261            ..Default::default()
262        };
263        worker.process_frame(&mut frame).unwrap();
264
265        let raw = rx
266            .try_recv()
267            .expect("raw tap should receive a frame from a pre-attach clone");
268        assert_eq!(raw.sample_rate, 8000);
269        assert_eq!(raw.channels, 1);
270        match raw.samples {
271            Samples::PCM { samples } => assert_eq!(samples.len(), 160),
272            _ => panic!("expected PCM samples on the raw tap"),
273        }
274        // The pipeline output is still normalized to the internal rate.
275        assert_eq!(frame.sample_rate, INTERNAL_SAMPLERATE);
276    }
277
278    /// Detaching the tap on the original chain must be observed by clones as
279    /// well (e.g. recorder restart swapping the sender).
280    #[test]
281    fn raw_tap_detach_visible_to_clones() {
282        let mut chain = ProcessorChain::new(INTERNAL_SAMPLERATE);
283        let (tx, _rx) = mpsc::unbounded_channel();
284        chain.set_raw_tap(Some(tx));
285        let mut worker = chain.clone();
286        chain.set_raw_tap(None);
287
288        let mut frame = AudioFrame {
289            track_id: "track".to_string(),
290            samples: Samples::PCM {
291                samples: vec![100i16; 160],
292            },
293            sample_rate: 8000,
294            channels: 1,
295            ..Default::default()
296        };
297        worker.process_frame(&mut frame).unwrap();
298        assert_eq!(frame.sample_rate, INTERNAL_SAMPLERATE);
299    }
300
301    /// Regression: a track report with no valid source rate (e.g. a media-pass
302    /// track whose `input_sample_rate` is 0) must not panic by trying to build
303    /// a resampler from 0 Hz, which previously poisoned the processor mutex.
304    #[test]
305    fn process_frame_tolerates_zero_sample_rate() {
306        let mut chain = ProcessorChain::new(INTERNAL_SAMPLERATE);
307        let mut frame = AudioFrame {
308            track_id: "track".to_string(),
309            samples: Samples::PCM {
310                samples: vec![100i16; 160],
311            },
312            sample_rate: 0,
313            channels: 1,
314            ..Default::default()
315        };
316
317        chain.process_frame(&mut frame).unwrap();
318        assert_eq!(frame.sample_rate, INTERNAL_SAMPLERATE);
319        match frame.samples {
320            Samples::PCM { samples } => assert_eq!(samples.len(), 160),
321            _ => panic!("expected PCM samples"),
322        }
323    }
324}