iris_chat_core/
call_audio.rs

1//! Shared, bounded 48 kHz Opus voice codec and playout buffer. Every platform
2//! supplies 20 ms mono PCM after its platform echo/noise processing.
3use std::{
4    collections::BTreeMap,
5    sync::{Arc, Mutex},
6};
7const SAMPLES: usize = 960;
8const MAX_PACKET: usize = 1275;
9#[derive(Debug, uniffi::Error)]
10pub enum CallAudioError {
11    CodecUnavailable,
12    InvalidSamples,
13}
14impl std::fmt::Display for CallAudioError {
15    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
16        f.write_str(match self {
17            Self::CodecUnavailable => "Call audio is unavailable",
18            Self::InvalidSamples => "Expected 20 ms of mono call audio",
19        })
20    }
21}
22impl std::error::Error for CallAudioError {}
23struct Playout {
24    decoder: opus::Decoder,
25    packets: BTreeMap<u32, Vec<u8>>,
26    next: Option<u32>,
27    priming: u8,
28    lost: u8,
29}
30#[derive(uniffi::Object)]
31pub struct CallAudioCodec {
32    encoder: Mutex<opus::Encoder>,
33    playout: Mutex<Playout>,
34}
35#[uniffi::export]
36impl CallAudioCodec {
37    #[uniffi::constructor]
38    pub fn new() -> Result<Arc<Self>, CallAudioError> {
39        let build = || -> opus::Result<Self> {
40            let mut encoder =
41                opus::Encoder::new(48000, opus::Channels::Mono, opus::Application::Voip)?;
42            encoder.set_bitrate(opus::Bitrate::Bits(32000))?;
43            encoder.set_vbr(false)?;
44            encoder.set_inband_fec(true)?;
45            encoder.set_packet_loss_perc(10)?;
46            encoder.set_dtx(true)?;
47            encoder.set_complexity(5)?;
48            Ok(Self {
49                encoder: Mutex::new(encoder),
50                playout: Mutex::new(Playout {
51                    decoder: opus::Decoder::new(48000, opus::Channels::Mono)?,
52                    packets: BTreeMap::new(),
53                    next: None,
54                    priming: 3,
55                    lost: 0,
56                }),
57            })
58        };
59        build()
60            .map(Arc::new)
61            .map_err(|_| CallAudioError::CodecUnavailable)
62    }
63    pub fn encode(&self, samples: Vec<i16>) -> Result<Vec<u8>, CallAudioError> {
64        if samples.len() != SAMPLES {
65            return Err(CallAudioError::InvalidSamples);
66        }
67        self.encoder
68            .lock()
69            .map_err(|_| CallAudioError::CodecUnavailable)?
70            .encode_vec(&samples, MAX_PACKET)
71            .map_err(|_| CallAudioError::CodecUnavailable)
72    }
73    pub fn queue(&self, sequence: u32, data: Vec<u8>) {
74        if data.is_empty() || data.len() > MAX_PACKET {
75            return;
76        }
77        let Ok(mut state) = self.playout.lock() else {
78            return;
79        };
80        if state.decoder.get_nb_samples(&data).ok() != Some(SAMPLES) {
81            return;
82        }
83        if let Some(next) = state.next {
84            let delta = sequence.wrapping_sub(next);
85            if delta >= 1 << 31 {
86                if state.priming > 0 && next.wrapping_sub(sequence) <= 3 {
87                    state.next = Some(sequence);
88                } else {
89                    return;
90                }
91            } else if delta > 12 {
92                // Rebase after a pause instead of replaying an old audio backlog.
93                state.packets.clear();
94                state.next = Some(sequence);
95                state.priming = 3;
96                state.lost = 0;
97                let _ = state.decoder.reset_state();
98            }
99        } else {
100            state.next = Some(sequence);
101        }
102        if state.packets.len() < 12 {
103            state.packets.entry(sequence).or_insert(data);
104        }
105    }
106    /// Call once per 20 ms of device output. Missing packets use in-band FEC
107    /// from the next packet, then Opus PLC, followed by silence after 120 ms.
108    pub fn playout(&self) -> Vec<i16> {
109        let mut output = vec![0; SAMPLES];
110        let Ok(mut state) = self.playout.lock() else {
111            return output;
112        };
113        let Some(next) = state.next else {
114            return output;
115        };
116        if state.priming > 0 {
117            state.priming -= 1;
118            return output;
119        }
120        let packet = state.packets.remove(&next);
121        let decoded = if let Some(packet) = packet {
122            state.lost = 0;
123            state.decoder.decode(&packet, &mut output, false)
124        } else {
125            state.lost = state.lost.saturating_add(1);
126            let fec = state.packets.get(&next.wrapping_add(1)).cloned();
127            if let Some(packet) = fec {
128                state.decoder.decode(&packet, &mut output, true)
129            } else if state.lost <= 6 {
130                state.decoder.decode(&[], &mut output, false)
131            } else {
132                Ok(SAMPLES)
133            }
134        };
135        if decoded.ok() != Some(SAMPLES) {
136            output.fill(0);
137        }
138        if state.lost > 6 && state.packets.is_empty() {
139            state.next = None;
140            state.priming = 3;
141            state.lost = 0;
142            let _ = state.decoder.reset_state();
143        } else {
144            state.next = Some(next.wrapping_add(1));
145        }
146        output
147    }
148}
149#[cfg(test)]
150mod tests {
151    use super::*;
152    #[test]
153    fn opus_compresses_voice_and_jitter_playout_recovers_reordering_and_loss() {
154        let codec = CallAudioCodec::new().unwrap();
155        let samples: Vec<_> = (0..960)
156            .map(|i| ((i as f32 * 0.05).sin() * 10000.0) as i16)
157            .collect();
158        let packets: Vec<_> = (0..8)
159            .map(|_| codec.encode(samples.clone()).unwrap())
160            .collect();
161        assert!(packets.iter().all(|p| p.len() <= 100));
162        for i in [0, 2, 1, 4, 5, 6, 7] {
163            codec.queue(i as u32, packets[i].clone());
164        }
165        for _ in 0..3 {
166            assert!(codec.playout().iter().all(|v| *v == 0));
167        }
168        for _ in 0..8 {
169            let decoded = codec.playout();
170            assert_eq!(decoded.len(), 960);
171            assert!(decoded.iter().any(|v| v.abs() > 20));
172        }
173        for _ in 0..7 {
174            codec.playout();
175        }
176        assert!(codec.playout().iter().all(|v| *v == 0));
177    }
178    #[test]
179    fn audio_resumes_after_mute_without_waiting_for_old_sequence_numbers() {
180        let codec = CallAudioCodec::new().unwrap();
181        let data = codec
182            .encode(
183                (0..960)
184                    .map(|i| ((i as f32 * 0.05).sin() * 10000.0) as i16)
185                    .collect(),
186            )
187            .unwrap();
188        codec.queue(0, data.clone());
189        for _ in 0..100 {
190            codec.playout();
191        }
192        codec.queue(1, data);
193        for _ in 0..3 {
194            codec.playout();
195        }
196        assert!(codec.playout().iter().any(|v| v.abs() > 20));
197    }
198
199    #[test]
200    fn malformed_input_and_far_future_packets_cannot_grow_audio_queues() {
201        let codec = CallAudioCodec::new().unwrap();
202        assert!(codec.encode(vec![0; 640]).is_err());
203        codec.queue(0, vec![0; 2000]);
204        assert_eq!(codec.playout.lock().unwrap().packets.len(), 0);
205        let data = codec.encode(vec![0; 960]).unwrap();
206        for i in 0..10000 {
207            codec.queue(i, data.clone());
208        }
209        assert!(codec.playout.lock().unwrap().packets.len() <= 12);
210    }
211}
212
213#[cfg(test)]
214#[path = "call_audio/sustained_tests.rs"]
215mod sustained_tests;
iris_chat_core/call_audio.rs

iris_chat_core/
call_audio.rs
