Skip to main content

active_call/media/
processor.rs

1use super::INTERNAL_SAMPLERATE;
2use super::track::track_codec::TrackCodec;
3use crate::event::{EventSender, SessionEvent};
4use crate::media::{AudioFrame, Samples, SourcePacket, TrackId};
5use anyhow::Result;
6use std::any::Any;
7use std::sync::{Arc, Mutex, RwLock};
8use tokio::sync::mpsc;
9
10pub trait Processor: Send + Sync + Any {
11    fn process_frame(&mut self, frame: &mut AudioFrame) -> Result<()>;
12}
13
14pub fn convert_to_mono(samples: &mut Vec<i16>, channels: u16) {
15    if channels != 2 {
16        return;
17    }
18    let mut i = 0;
19    let mut j = 0;
20    while i < samples.len() {
21        let l = samples[i] as i32;
22        let r = samples[i + 1] as i32;
23        samples[j] = ((l + r) / 2) as i16;
24        i += 2;
25        j += 1;
26    }
27    samples.truncate(j);
28}
29
30impl Default for AudioFrame {
31    fn default() -> Self {
32        Self {
33            track_id: "".to_string(),
34            samples: Samples::Empty,
35            timestamp: 0,
36            sample_rate: 16000,
37            channels: 1,
38            src_packet: None,
39            speech_probability: None,
40        }
41    }
42}
43
44impl Samples {
45    pub fn is_empty(&self) -> bool {
46        match self {
47            Samples::PCM { samples } => samples.is_empty(),
48            Samples::RTP { payload, .. } => payload.is_empty(),
49            Samples::Dtmf { .. } => false,
50            Samples::Empty => true,
51        }
52    }
53}
54
55#[derive(Clone)]
56pub struct ProcessorChain {
57    processors: Arc<Mutex<Vec<Box<dyn Processor>>>>,
58    pub codec: TrackCodec,
59    sample_rate: u32,
60    pub force_decode: bool,
61    /// Optional raw tap: when set, frames are mirrored to this channel at
62    /// their native (pre-resample) sample rate right after decoding, before
63    /// the pipeline normalizes them to `INTERNAL_SAMPLERATE`. Used by the
64    /// native-samplerate recorder.
65    ///
66    /// Shared via `Arc<RwLock<..>>`: `RtcTrack::create()` clones the chain
67    /// into its long-lived worker tasks *before* the recorder attaches the
68    /// tap, so a plain field would leave those workers with a stale `None`.
69    pub raw_tap: Arc<RwLock<Option<mpsc::UnboundedSender<AudioFrame>>>>,
70    /// Track id that owns this chain. When set, the raw tap only mirrors
71    /// frames tagged with this id. A track often relays another leg's audio
72    /// (e.g. the media-pass track forwards the caller's audio to the
73    /// websocket), and that relayed copy must not be recorded a second time:
74    /// the originating leg already recorded it.
75    pub tap_owner: Arc<RwLock<Option<TrackId>>>,
76}
77
78/// Lock the processor list, recovering from a poisoned mutex.
79///
80/// A panic inside a processor (or in the decode/resample step) must not
81/// permanently take down the track: the `Vec` itself is never left in an
82/// inconsistent state by unwinding, so reusing the guard is safe and far
83/// better than propagating a `PoisonError` on every subsequent frame.
84///
85/// Takes the mutex directly (rather than `&self`) so the returned guard only
86/// borrows the `processors` field, leaving `self.codec` free to be borrowed
87/// mutably for the decode/resample steps in `process_frame`.
88fn lock_processors(
89    processors: &Mutex<Vec<Box<dyn Processor>>>,
90) -> std::sync::MutexGuard<'_, Vec<Box<dyn Processor>>> {
91    processors.lock().unwrap_or_else(|e| e.into_inner())
92}
93
94impl ProcessorChain {
95    pub fn new(_sample_rate: u32) -> Self {
96        Self {
97            processors: Arc::new(Mutex::new(Vec::new())),
98            codec: TrackCodec::new(),
99            sample_rate: INTERNAL_SAMPLERATE,
100            force_decode: true,
101            raw_tap: Arc::new(RwLock::new(None)),
102            tap_owner: Arc::new(RwLock::new(None)),
103        }
104    }
105    pub fn set_raw_tap(&mut self, tap: Option<mpsc::UnboundedSender<AudioFrame>>) {
106        *self.raw_tap.write().unwrap() = tap;
107    }
108    /// Restrict the raw tap to frames owned by `owner` (see `tap_owner`).
109    pub fn set_tap_owner(&mut self, owner: Option<TrackId>) {
110        *self.tap_owner.write().unwrap() = owner;
111    }
112    fn raw_tap(&self) -> Option<mpsc::UnboundedSender<AudioFrame>> {
113        self.raw_tap.read().unwrap().clone()
114    }
115    fn tap_owns(&self, track_id: &str) -> bool {
116        match self.tap_owner.read().unwrap().as_deref() {
117            Some(owner) => owner == track_id,
118            None => true,
119        }
120    }
121    pub fn insert_processor(&mut self, processor: Box<dyn Processor>) {
122        lock_processors(&self.processors).insert(0, processor);
123    }
124    pub fn append_processor(&mut self, processor: Box<dyn Processor>) {
125        lock_processors(&self.processors).push(processor);
126    }
127
128    pub fn has_processor<T: 'static>(&self) -> bool {
129        let processors = lock_processors(&self.processors);
130        processors
131            .iter()
132            .any(|processor| (processor.as_ref() as &dyn Any).is::<T>())
133    }
134
135    pub fn remove_processor<T: 'static>(&self) {
136        let mut processors = lock_processors(&self.processors);
137        processors.retain(|processor| !(processor.as_ref() as &dyn Any).is::<T>());
138    }
139
140    pub fn process_frame(&mut self, frame: &mut AudioFrame) -> Result<()> {
141        let mut processors = lock_processors(&self.processors);
142        if !self.force_decode && processors.is_empty() && self.raw_tap().is_none() {
143            return Ok(());
144        }
145        match &mut frame.samples {
146            Samples::RTP {
147                payload_type,
148                payload,
149                sequence_number,
150            } => {
151                // Decode only payload types negotiated as an audio codec;
152                // telephone-event (on any payload type) stays RTP.
153                if self
154                    .codec
155                    .get_codec_for_pt(*payload_type)
156                    .is_some_and(|codec| codec.is_audio())
157                {
158                    let (decoded_sample_rate, channels, samples) =
159                        self.codec.decode(*payload_type, &payload);
160                    let src_packet = SourcePacket {
161                        sequence_number: *sequence_number,
162                        payload_type: *payload_type,
163                        payload: std::mem::take(payload),
164                    };
165                    frame.src_packet = Some(src_packet);
166                    frame.channels = channels;
167                    frame.samples = Samples::PCM { samples };
168                    frame.sample_rate = decoded_sample_rate;
169                }
170            }
171            _ => {}
172        }
173
174        // Mirror the frame to the raw tap at its native sample rate, before
175        // the pipeline resamples it to INTERNAL_SAMPLERATE. Only frames owned
176        // by this chain's track are mirrored: audio the track merely relays
177        // for another leg is already recorded at its origin.
178        if let Some(tap) = self.raw_tap()
179            && self.tap_owns(&frame.track_id)
180            && let Samples::PCM { samples } = &frame.samples
181            && !samples.is_empty()
182            && frame.sample_rate > 0
183        {
184            let mut raw = frame.clone();
185            raw.src_packet = None;
186            let mono = match &mut raw.samples {
187                Samples::PCM { samples } => samples,
188                _ => unreachable!("checked PCM above"),
189            };
190            if raw.channels == 2 {
191                convert_to_mono(mono, 2);
192                raw.channels = 1;
193            }
194            let _ = tap.send(raw);
195        }
196
197        if let Samples::PCM { samples } = &mut frame.samples {
198            if frame.sample_rate != self.sample_rate {
199                let new_samples = self.codec.resample(
200                    std::mem::take(samples),
201                    frame.sample_rate,
202                    self.sample_rate,
203                );
204                *samples = new_samples;
205                frame.sample_rate = self.sample_rate;
206            }
207            if frame.channels == 2 {
208                convert_to_mono(samples, 2);
209                frame.channels = 1;
210            }
211        }
212        // Process the frame with all processors
213        for processor in processors.iter_mut() {
214            processor.process_frame(frame)?;
215        }
216        Ok(())
217    }
218}
219
220pub struct SubscribeProcessor {
221    event_sender: EventSender,
222    track_id: String,
223    /// Only frames tagged with this track id are mirrored. A track may relay
224    /// another leg's audio (e.g. the media-pass track forwards the caller's
225    /// audio to the websocket and runs it through its own chain); that
226    /// relayed copy is already published by its origin track and must not be
227    /// subscribed a second time.
228    owner: TrackId,
229    track_index: u8, // 0 for caller, 1 for callee
230}
231
232impl SubscribeProcessor {
233    pub fn new(event_sender: EventSender, track_id: String, owner: TrackId, track_index: u8) -> Self {
234        Self {
235            event_sender,
236            track_id,
237            owner,
238            track_index,
239        }
240    }
241}
242
243impl Processor for SubscribeProcessor {
244    fn process_frame(&mut self, frame: &mut AudioFrame) -> Result<()> {
245        if frame.track_id != self.owner {
246            return Ok(());
247        }
248        if let Samples::PCM { samples } = &frame.samples {
249            if !samples.is_empty() {
250                let pcm_data = audio_codec::samples_to_bytes(samples);
251                let mut data = Vec::with_capacity(pcm_data.len() + 1);
252                data.push(self.track_index);
253                data.extend_from_slice(&pcm_data);
254
255                let event = SessionEvent::Binary {
256                    track_id: self.track_id.clone(),
257                    timestamp: frame.timestamp,
258                    data,
259                };
260                self.event_sender.send(event).ok();
261            }
262        }
263        Ok(())
264    }
265}
266
267#[cfg(test)]
268mod tests {
269    use super::*;
270
271    /// Regression: `RtcTrack::create()` clones the chain into its worker
272    /// tasks *before* the recorder attaches the raw tap (native-samplerate
273    /// recording). The tap must be shared state so clones taken before the
274    /// attach observe it; with a plain `Option` field this test fails and
275    /// SIP/RTP calls silently fall back to the 16 kHz recorder.
276    #[test]
277    fn raw_tap_visible_to_clones_taken_before_attach() {
278        let mut chain = ProcessorChain::new(INTERNAL_SAMPLERATE);
279        let mut worker = chain.clone();
280
281        let (tx, mut rx) = mpsc::unbounded_channel();
282        chain.set_raw_tap(Some(tx));
283
284        let mut frame = AudioFrame {
285            track_id: "track".to_string(),
286            samples: Samples::PCM {
287                samples: vec![100i16; 160],
288            },
289            sample_rate: 8000,
290            channels: 1,
291            ..Default::default()
292        };
293        worker.process_frame(&mut frame).unwrap();
294
295        let raw = rx
296            .try_recv()
297            .expect("raw tap should receive a frame from a pre-attach clone");
298        assert_eq!(raw.sample_rate, 8000);
299        assert_eq!(raw.channels, 1);
300        match raw.samples {
301            Samples::PCM { samples } => assert_eq!(samples.len(), 160),
302            _ => panic!("expected PCM samples on the raw tap"),
303        }
304        // The pipeline output is still normalized to the internal rate.
305        assert_eq!(frame.sample_rate, INTERNAL_SAMPLERATE);
306    }
307
308    /// Detaching the tap on the original chain must be observed by clones as
309    /// well (e.g. recorder restart swapping the sender).
310    #[test]
311    fn raw_tap_detach_visible_to_clones() {
312        let mut chain = ProcessorChain::new(INTERNAL_SAMPLERATE);
313        let (tx, _rx) = mpsc::unbounded_channel();
314        chain.set_raw_tap(Some(tx));
315        let mut worker = chain.clone();
316        chain.set_raw_tap(None);
317
318        let mut frame = AudioFrame {
319            track_id: "track".to_string(),
320            samples: Samples::PCM {
321                samples: vec![100i16; 160],
322            },
323            sample_rate: 8000,
324            channels: 1,
325            ..Default::default()
326        };
327        worker.process_frame(&mut frame).unwrap();
328        assert_eq!(frame.sample_rate, INTERNAL_SAMPLERATE);
329    }
330
331    /// Regression: a track often relays another leg's audio through its own
332    /// processor chain (the media-pass track forwards the caller's audio to the
333    /// websocket). The native-samplerate raw tap must mirror only frames the
334    /// chain owns, otherwise a media-pass call records the caller's audio from
335    /// both legs and the recording comes out twice as long.
336    #[test]
337    fn raw_tap_only_mirrors_owned_frames() {
338        let mut chain = ProcessorChain::new(INTERNAL_SAMPLERATE);
339        let (tx, mut rx) = mpsc::unbounded_channel();
340        chain.set_raw_tap(Some(tx));
341        chain.set_tap_owner(Some("server-side-track".to_string()));
342
343        // A frame forwarded from another leg (caller -> websocket) must not be
344        // mirrored by this chain.
345        let mut relayed = AudioFrame {
346            track_id: "caller".to_string(),
347            samples: Samples::PCM {
348                samples: vec![1i16; 160],
349            },
350            sample_rate: 8000,
351            channels: 1,
352            ..Default::default()
353        };
354        chain.process_frame(&mut relayed).unwrap();
355        assert!(
356            rx.try_recv().is_err(),
357            "relayed frames must not be mirrored to the raw tap"
358        );
359
360        // The chain's own frame must be mirrored.
361        let mut owned = AudioFrame {
362            track_id: "server-side-track".to_string(),
363            samples: Samples::PCM {
364                samples: vec![1i16; 160],
365            },
366            sample_rate: 16000,
367            channels: 1,
368            ..Default::default()
369        };
370        chain.process_frame(&mut owned).unwrap();
371        let raw = rx
372            .try_recv()
373            .expect("owned frames must be mirrored to the raw tap");
374        assert_eq!(raw.track_id, "server-side-track");
375        assert_eq!(raw.sample_rate, 16000);
376    }
377
378    /// Regression: a track report with no valid source rate (e.g. a media-pass
379    /// track whose `input_sample_rate` is 0) must not panic by trying to build
380    /// a resampler from 0 Hz, which previously poisoned the processor mutex.
381    #[test]
382    fn process_frame_tolerates_zero_sample_rate() {
383        let mut chain = ProcessorChain::new(INTERNAL_SAMPLERATE);
384        let mut frame = AudioFrame {
385            track_id: "track".to_string(),
386            samples: Samples::PCM {
387                samples: vec![100i16; 160],
388            },
389            sample_rate: 0,
390            channels: 1,
391            ..Default::default()
392        };
393
394        chain.process_frame(&mut frame).unwrap();
395        assert_eq!(frame.sample_rate, INTERNAL_SAMPLERATE);
396        match frame.samples {
397            Samples::PCM { samples } => assert_eq!(samples.len(), 160),
398            _ => panic!("expected PCM samples"),
399        }
400    }
401}