use crate::{media::AudioFrame, media::PcmBuf, media::Samples};
use audio_codec::{
BoxedResampler, CodecType, Decoder, Encoder, bytes_to_samples,
g722::{G722Decoder, G722Encoder},
pcma::{PcmaDecoder, PcmaEncoder},
pcmu::{PcmuDecoder, PcmuEncoder},
samples_to_bytes,
};
use std::collections::HashMap;
use std::sync::{Arc, RwLock};
use std::time::Duration;
use tracing::warn;
use audio_codec::g729::{G729Decoder, G729Encoder};
use audio_codec::opus::{OpusDecoder, OpusEncoder};
const OPUS_BITRATE: u32 = 64000;
#[derive(Debug, Default)]
pub struct OpusLossEstimate {
smoothed: Option<f32>,
level: f32,
}
impl OpusLossEstimate {
const ALPHA_PER_MS: f32 = 0.9999;
pub fn update(&mut self, fraction_lost: f32, elapsed: Duration) -> i32 {
let fraction_lost = fraction_lost.clamp(0.0, 1.0);
let smoothed = match self.smoothed {
Some(smoothed) => {
let alpha = Self::ALPHA_PER_MS.powf(elapsed.as_millis() as f32);
alpha * smoothed + (1.0 - alpha) * fraction_lost
}
None => fraction_lost,
};
self.smoothed = Some(smoothed);
self.level = Self::level(smoothed, self.level);
(self.level * 100.0).round() as i32
}
fn level(loss: f32, level: f32) -> f32 {
let past =
|rate: f32, margin: f32| loss >= rate + if rate > level { margin } else { -margin };
if past(0.20, 0.02) {
0.20
} else if past(0.10, 0.01) {
0.10
} else if past(0.05, 0.01) {
0.05
} else if loss >= 0.01 {
0.01
} else {
0.0
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct OpusParams {
pub max_playback_rate: u32,
pub max_average_bitrate: Option<u32>,
pub inband_fec: bool,
}
impl Default for OpusParams {
fn default() -> Self {
Self {
max_playback_rate: 48000,
max_average_bitrate: None,
inband_fec: false,
}
}
}
impl OpusParams {
pub fn from_fmtp(fmtp: &str) -> Self {
let mut params = Self::default();
for (key, value) in fmtp
.split(';')
.filter_map(|p| p.split_once('='))
.map(|(k, v)| (k.trim().to_ascii_lowercase(), v.trim()))
{
match key.as_str() {
"maxplaybackrate" => {
if let Ok(rate) = value.parse() {
params.max_playback_rate = rate;
}
}
"maxaveragebitrate" => params.max_average_bitrate = value.parse().ok(),
"useinbandfec" => params.inband_fec = value == "1",
_ => {}
}
}
params
}
pub fn sample_rate(&self) -> u32 {
[48000, 24000, 16000, 12000]
.into_iter()
.find(|r| *r <= self.max_playback_rate)
.unwrap_or(8000)
}
pub fn bitrate(&self) -> u32 {
self.max_average_bitrate
.map_or(OPUS_BITRATE, |max| max.min(OPUS_BITRATE))
.clamp(6000, 510_000)
}
fn encoder(&self, packet_loss_perc: i32) -> OpusEncoder {
let mut encoder = OpusEncoder::new(self.sample_rate(), 1);
encoder.set_bitrate(self.bitrate() as i32);
encoder.set_inband_fec(self.inband_fec);
encoder.set_packet_loss_perc(packet_loss_perc);
encoder
}
}
pub fn duration_to_rtp_ticks(duration: Duration, clock_rate: u32) -> u32 {
(duration.as_nanos() * clock_rate as u128 / 1_000_000_000) as u32
}
pub struct TrackCodec {
pcmu_encoder: PcmuEncoder,
pcmu_decoder: PcmuDecoder,
pcma_encoder: PcmaEncoder,
pcma_decoder: PcmaDecoder,
g722_encoder: Option<Box<G722Encoder>>,
g722_decoder: Option<Box<G722Decoder>>,
g729_encoder: Option<Box<G729Encoder>>,
g729_decoder: Option<Box<G729Decoder>>,
opus_encoder: Option<OpusEncoder>,
opus_params: OpusParams,
opus_packet_loss: i32,
opus_decoder: Option<OpusDecoder>,
resampler: Option<BoxedResampler>,
resampler_in_rate: u32,
resampler_out_rate: u32,
pub payload_type_map: Arc<RwLock<HashMap<u8, CodecType>>>,
concealment: bool,
last_payload_type: Option<u8>,
}
pub fn codec_for_pt(map: &RwLock<HashMap<u8, CodecType>>, pt: u8) -> Option<CodecType> {
map.read()
.unwrap()
.get(&pt)
.cloned()
.or_else(|| CodecType::try_from(pt).ok())
}
impl Clone for TrackCodec {
fn clone(&self) -> Self {
let mut new = Self::new();
new.payload_type_map = Arc::clone(&self.payload_type_map);
new.opus_params = self.opus_params;
if self.concealment {
new.enable_concealment();
}
new
}
}
impl TrackCodec {
pub fn new() -> Self {
let mut map = HashMap::new();
map.insert(0, CodecType::PCMU);
map.insert(8, CodecType::PCMA);
map.insert(9, CodecType::G722);
map.insert(18, CodecType::G729);
map.insert(101, CodecType::TelephoneEvent);
map.insert(111, CodecType::Opus);
let payload_type_map = Arc::new(RwLock::new(map));
Self {
pcmu_encoder: PcmuEncoder::new(),
pcmu_decoder: PcmuDecoder::new(),
pcma_encoder: PcmaEncoder::new(),
pcma_decoder: PcmaDecoder::new(),
g722_encoder: None,
g722_decoder: None,
g729_encoder: None,
g729_decoder: None,
opus_encoder: None,
opus_params: OpusParams::default(),
opus_packet_loss: 0,
opus_decoder: None,
resampler: None,
resampler_in_rate: 0,
resampler_out_rate: 0,
payload_type_map,
concealment: false,
last_payload_type: None,
}
}
pub fn enable_concealment(&mut self) {
self.concealment = true;
self.pcmu_decoder.enable_concealment();
self.pcma_decoder.enable_concealment();
for decoder in [
self.g722_decoder
.as_deref_mut()
.map(|d| d as &mut dyn Decoder),
self.g729_decoder
.as_deref_mut()
.map(|d| d as &mut dyn Decoder),
self.opus_decoder.as_mut().map(|d| d as &mut dyn Decoder),
]
.into_iter()
.flatten()
{
decoder.enable_concealment();
}
}
fn codec_for_pt(&self, payload_type: u8) -> CodecType {
self.payload_type_map
.read()
.unwrap()
.get(&payload_type)
.cloned()
.unwrap_or_else(|| match payload_type {
0 => CodecType::PCMU,
8 => CodecType::PCMA,
9 => CodecType::G722,
18 => CodecType::G729,
111 => CodecType::Opus,
_ => CodecType::PCMU,
})
}
fn decoder(&mut self, codec: CodecType) -> Option<&mut dyn Decoder> {
let concealment = self.concealment;
let decoder: &mut dyn Decoder = match codec {
CodecType::PCMU => &mut self.pcmu_decoder,
CodecType::PCMA => &mut self.pcma_decoder,
CodecType::G722 => self
.g722_decoder
.get_or_insert_with(|| {
let mut d = Box::new(G722Decoder::new());
if concealment {
d.enable_concealment();
}
d
})
.as_mut(),
CodecType::G729 => self
.g729_decoder
.get_or_insert_with(|| {
let mut d = Box::new(G729Decoder::new());
if concealment {
d.enable_concealment();
}
d
})
.as_mut(),
CodecType::Opus => self.opus_decoder.get_or_insert_with(|| {
let mut d = OpusDecoder::new_default();
if concealment {
d.enable_concealment();
}
d
}),
_ => return None,
};
Some(decoder)
}
fn native_format(codec: CodecType, samples: usize) -> (u32, u16) {
match codec {
CodecType::PCMU | CodecType::PCMA | CodecType::G729 => (8000, 1),
CodecType::G722 => (16000, 1),
CodecType::Opus if samples >= 1920 => (48000, 2),
CodecType::Opus => (48000, 1),
_ => (8000, 1),
}
}
pub fn conceal(
&mut self,
ptime: Duration,
next: Option<(u8, &[u8])>,
) -> Option<(u32, u16, PcmBuf, bool)> {
if !self.concealment {
return None;
}
let payload_type = self.last_payload_type?;
let codec = self.codec_for_pt(payload_type);
let (rate, _) = Self::native_format(codec, 0);
let samples = (rate as u128 * ptime.as_nanos() / 1_000_000_000) as usize;
let next = next
.filter(|(pt, _)| *pt == payload_type && codec == CodecType::Opus)
.map(|(_, p)| p);
let used_next = next.is_some();
let decoder = self.decoder(codec)?;
let pcm = match next {
Some(next) => decoder.recover(next, samples),
None => decoder.conceal(samples),
};
if pcm.is_empty() {
return None;
}
let (rate, channels) = Self::native_format(codec, pcm.len());
Some((rate, channels, pcm, used_next))
}
pub fn set_opus_params(&mut self, params: OpusParams) {
if params != self.opus_params {
self.opus_params = params;
self.opus_encoder = None;
}
}
pub fn set_opus_packet_loss(&mut self, percent: i32) {
if percent != self.opus_packet_loss {
self.opus_packet_loss = percent;
if let Some(encoder) = self.opus_encoder.as_mut() {
encoder.set_packet_loss_perc(percent);
}
}
}
pub fn opus_params(&self) -> OpusParams {
self.opus_params
}
pub fn set_payload_type(&mut self, pt: u8, codec: CodecType) {
self.payload_type_map.write().unwrap().insert(pt, codec);
}
pub fn get_codec_for_pt(&self, pt: u8) -> Option<CodecType> {
codec_for_pt(&self.payload_type_map, pt)
}
pub fn decode(&mut self, payload_type: u8, payload: &[u8]) -> (u32, u16, PcmBuf) {
let codec = self.codec_for_pt(payload_type);
let pcm = match self.decoder(codec) {
Some(decoder) => {
let pcm = decoder.decode(payload);
self.last_payload_type = Some(payload_type);
pcm
}
None => bytes_to_samples(payload),
};
let (in_rate, channels) = Self::native_format(codec, pcm.len());
(in_rate, channels, pcm)
}
pub fn resample(&mut self, pcm: PcmBuf, in_rate: u32, out_rate: u32) -> PcmBuf {
if in_rate == 0 || out_rate == 0 || in_rate == out_rate {
return pcm;
}
if self.resampler.is_none()
|| self.resampler_in_rate != in_rate
|| self.resampler_out_rate != out_rate
{
match BoxedResampler::new(in_rate as usize, out_rate as usize) {
Ok(resampler) => {
self.resampler = Some(resampler);
self.resampler_in_rate = in_rate;
self.resampler_out_rate = out_rate;
}
Err(e) => {
warn!(in_rate, out_rate, "failed to build resampler: {e}");
self.resampler = None;
self.resampler_in_rate = 0;
self.resampler_out_rate = 0;
return pcm;
}
}
}
self.resampler.as_mut().unwrap().resample(&pcm)
}
pub fn encode(&mut self, payload_type: u8, frame: AudioFrame) -> (u8, Vec<u8>) {
match frame.samples {
Samples::PCM { samples: mut pcm } => {
let codec = self
.payload_type_map
.read()
.unwrap()
.get(&payload_type)
.cloned()
.or_else(|| CodecType::try_from(payload_type).ok());
let target_samplerate = match codec {
Some(CodecType::Opus) => self.opus_params.sample_rate(),
Some(c) => c.samplerate(),
None => 8000,
};
if frame.sample_rate != target_samplerate {
pcm = self.resample(pcm, frame.sample_rate, target_samplerate);
}
let payload = match codec {
Some(CodecType::PCMU) => self.pcmu_encoder.encode(&pcm),
Some(CodecType::PCMA) => self.pcma_encoder.encode(&pcm),
Some(CodecType::G722) => self
.g722_encoder
.get_or_insert_with(|| Box::new(G722Encoder::new()))
.encode(&pcm),
Some(CodecType::G729) => self
.g729_encoder
.get_or_insert_with(|| Box::new(G729Encoder::new()))
.encode(&pcm),
Some(CodecType::Opus) => {
let (params, loss) = (self.opus_params, self.opus_packet_loss);
self.opus_encoder
.get_or_insert_with(|| params.encoder(loss))
.encode(&pcm)
}
_ => samples_to_bytes(&pcm),
};
(payload_type, payload)
}
Samples::RTP {
payload_type,
payload,
..
} => (payload_type, payload),
_ => (payload_type, vec![]),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_encode_dynamic_opus_payload_type_uses_opus_encoder() {
let mut codec = TrackCodec::new();
codec.set_payload_type(96, CodecType::Opus);
let frame = AudioFrame {
samples: Samples::PCM {
samples: vec![0; 320],
},
sample_rate: 16000,
channels: 1,
..Default::default()
};
let (pt, payload) = codec.encode(96, frame);
assert_eq!(pt, 96);
assert!(!payload.is_empty());
assert!(payload.len() < 640);
}
#[test]
fn test_resample_zero_rate_passes_through_without_panic() {
let mut codec = TrackCodec::new();
let samples = vec![1000i16; 160];
assert_eq!(codec.resample(samples.clone(), 0, 16000), samples);
assert_eq!(codec.resample(samples.clone(), 16000, 0), samples);
assert_eq!(codec.resample(samples.clone(), 0, 0), samples);
}
}
#[cfg(test)]
mod conceal_tests {
use super::*;
use audio_codec::Encoder;
const PTIME: Duration = Duration::from_millis(20);
fn voice(rate: u32, frames: usize) -> Vec<Vec<i16>> {
let frame = (rate / 50) as usize;
let mut phase = 0f32;
let pcm: Vec<i16> = (0..frames * frame)
.map(|i| {
let t = i as f32 / rate as f32;
let f0 = 120.0 + 60.0 * (t / 0.6).min(1.0);
phase += 2.0 * std::f32::consts::PI * f0 / rate as f32;
let env = 0.6 + 0.4 * (2.0 * std::f32::consts::PI * 2.0 * t).sin();
let v: f32 = (1..=8).map(|k| (k as f32 * phase).sin() / k as f32).sum();
(6000.0 * env * v) as i16
})
.collect();
pcm.chunks(frame).map(<[i16]>::to_vec).collect()
}
#[test]
fn nothing_to_conceal_with_until_enabled_and_decoded() {
let mut codec = TrackCodec::new();
let payload = PcmaEncoder::new().encode(&voice(8000, 1)[0]);
codec.decode(8, &payload);
assert!(codec.conceal(PTIME, None).is_none(), "concealment is off");
let mut codec = TrackCodec::new();
codec.enable_concealment();
assert!(codec.conceal(PTIME, None).is_none(), "nothing decoded yet");
}
#[test]
fn pcma_conceals_in_its_native_format() {
let mut codec = TrackCodec::new();
codec.enable_concealment();
let mut enc = PcmaEncoder::new();
for frame in voice(8000, 6) {
codec.decode(8, &enc.encode(&frame));
}
let (rate, channels, pcm, used_next) = codec.conceal(PTIME, None).unwrap();
assert_eq!(
(rate, channels, pcm.len(), used_next),
(8000, 1, 160, false)
);
assert!(pcm.iter().any(|s| *s != 0));
}
#[test]
fn g729_conceals_in_its_native_format() {
let mut codec = TrackCodec::new();
codec.enable_concealment();
let mut enc = G729Encoder::new();
for frame in voice(8000, 6) {
codec.decode(18, &enc.encode(&frame));
}
let (rate, channels, pcm, used_next) = codec.conceal(PTIME, None).unwrap();
assert_eq!(
(rate, channels, pcm.len(), used_next),
(8000, 1, 160, false)
);
assert!(pcm.iter().any(|s| *s != 0));
}
#[test]
fn opus_rebuilds_a_lost_packet_from_the_next_ones_fec() {
let mut enc = OpusEncoder::new(48000, 1);
enc.set_bitrate(32000);
enc.set_inband_fec(true);
enc.set_packet_loss_perc(20);
let packets: Vec<Vec<u8>> = voice(48000, 30).iter().map(|f| enc.encode(f)).collect();
let mut reference = TrackCodec::new();
let clean: Vec<PcmBuf> = packets.iter().map(|p| reference.decode(111, p).2).collect();
let error_db = |out: &[i16], clean: &[i16]| {
let err: f64 = out
.iter()
.zip(clean)
.map(|(a, b)| (*a as f64 - *b as f64).powi(2))
.sum();
let sig: f64 = clean.iter().map(|b| (*b as f64).powi(2)).sum();
10.0 * (err / sig).log10()
};
let losses = [6, 9, 12, 24];
let (mut fec_db, mut plc_db) = (0f64, 0f64);
for lost in losses {
let mut recovering = TrackCodec::new();
let mut concealing = TrackCodec::new();
for codec in [&mut recovering, &mut concealing] {
codec.enable_concealment();
for p in &packets[..lost] {
codec.decode(111, p);
}
}
let (rate, _, recovered, used_next) = recovering
.conceal(PTIME, Some((111, &packets[lost + 1])))
.unwrap();
assert_eq!((rate, recovered.len(), used_next), (48000, 960, true));
let (_, _, concealed, used_next) = concealing.conceal(PTIME, None).unwrap();
assert!(!used_next);
fec_db += error_db(&recovered, &clean[lost]);
plc_db += error_db(&concealed, &clean[lost]);
}
let (fec_db, plc_db) = (fec_db / losses.len() as f64, plc_db / losses.len() as f64);
assert!(
fec_db < plc_db - 10.0,
"FEC {fec_db:.1} dB vs concealment {plc_db:.1} dB"
);
let mut recovering = TrackCodec::new();
recovering.enable_concealment();
recovering.decode(111, &packets[0]);
let (_, _, _, used_next) = recovering.conceal(PTIME, Some((0, &packets[1]))).unwrap();
assert!(!used_next);
}
#[test]
fn opus_params_from_fmtp() {
let firefox = OpusParams::from_fmtp("maxplaybackrate=48000;stereo=1;useinbandfec=1");
assert_eq!(
firefox,
OpusParams {
max_playback_rate: 48000,
max_average_bitrate: None,
inband_fec: true,
}
);
assert_eq!((firefox.sample_rate(), firefox.bitrate()), (48000, 64000));
let capped = OpusParams::from_fmtp("maxplaybackrate=16000; maxaveragebitrate=20000");
assert_eq!(
(capped.sample_rate(), capped.bitrate(), capped.inband_fec),
(16000, 20000, false)
);
assert_eq!(
OpusParams::from_fmtp("maxplaybackrate=22050").sample_rate(),
16000
);
assert_eq!(
OpusParams::from_fmtp("maxaveragebitrate=128000").bitrate(),
64000
);
assert_eq!(
OpusParams::from_fmtp("useinbandfec=0"),
OpusParams::default()
);
}
#[test]
fn opus_loss_estimate_follows_reports_like_libwebrtc() {
let every = Duration::from_secs(5);
let mut estimate = OpusLossEstimate::default();
assert_eq!(estimate.update(0.0, every), 0);
let rising: Vec<i32> = (0..6).map(|_| estimate.update(0.12, every)).collect();
assert_eq!(rising, [1, 5, 5, 5, 10, 10]);
assert_eq!(estimate.update(0.095, every), 10);
let falling: Vec<i32> = (0..6).map(|_| estimate.update(0.0, every)).collect();
assert_eq!(falling, [5, 1, 1, 1, 0, 0]);
}
#[test]
fn opus_encoded_for_a_fec_receiver_carries_fec() {
let mut sender = TrackCodec::new();
sender.set_opus_params(OpusParams::from_fmtp(
"maxplaybackrate=48000;stereo=1;useinbandfec=1",
));
sender.set_opus_packet_loss(10);
let packets: Vec<Vec<u8>> = voice(16000, 30)
.into_iter()
.map(|samples| {
let frame = AudioFrame {
samples: Samples::PCM { samples },
sample_rate: 16000,
channels: 1,
..Default::default()
};
sender.encode(111, frame).1
})
.collect();
assert!(packets.iter().all(|p| p[0] & 0x04 == 0), "stereo packets");
let bitrate = packets.iter().map(Vec::len).sum::<usize>() * 8 * 50 / packets.len();
assert!((58000..=66000).contains(&bitrate), "{bitrate} bit/s");
let mut reference = TrackCodec::new();
let clean: Vec<PcmBuf> = packets.iter().map(|p| reference.decode(111, p).2).collect();
let error_db = |out: &[i16], clean: &[i16]| {
let err: f64 = out
.iter()
.zip(clean)
.map(|(a, b)| (*a as f64 - *b as f64).powi(2))
.sum();
let sig: f64 = clean.iter().map(|b| (*b as f64).powi(2)).sum();
10.0 * (err / sig).log10()
};
let losses = [6, 9, 12, 24];
let (mut fec_db, mut plc_db) = (0f64, 0f64);
for lost in losses {
let mut recovering = TrackCodec::new();
let mut concealing = TrackCodec::new();
for codec in [&mut recovering, &mut concealing] {
codec.enable_concealment();
for p in &packets[..lost] {
codec.decode(111, p);
}
}
let (_, _, recovered, used_next) = recovering
.conceal(PTIME, Some((111, &packets[lost + 1])))
.unwrap();
assert!(used_next);
let (_, _, concealed, _) = concealing.conceal(PTIME, None).unwrap();
fec_db += error_db(&recovered, &clean[lost]);
plc_db += error_db(&concealed, &clean[lost]);
}
let (fec_db, plc_db) = (fec_db / losses.len() as f64, plc_db / losses.len() as f64);
assert!(
fec_db < plc_db - 6.0,
"FEC {fec_db:.1} dB vs concealment {plc_db:.1} dB"
);
}
fn rms(pcm: &[i16]) -> f64 {
(pcm.iter().map(|s| (*s as f64).powi(2)).sum::<f64>() / pcm.len() as f64).sqrt()
}
#[test]
fn opus_concealment_fades_to_silence() {
let mut enc = OpusEncoder::new(48000, 1);
let packets: Vec<Vec<u8>> = voice(48000, 20).iter().map(|f| enc.encode(f)).collect();
let mut codec = TrackCodec::new();
codec.enable_concealment();
for p in &packets[..10] {
codec.decode(111, p);
}
let levels: Vec<f64> = (0..5)
.map(|_| rms(&codec.conceal(PTIME, None).unwrap().2))
.collect();
assert!(
levels[0] > 300.0,
"first frame of a loss is audible: {levels:?}"
);
assert!(levels[1] < levels[0] * 0.8, "not fading: {levels:?}");
assert!(
levels[3..].iter().all(|l| *l == 0.0),
"not silent at 60 ms: {levels:?}"
);
}
}
#[cfg(test)]
mod duration_tests {
use super::*;
#[test]
fn test_duration_to_rtp_ticks() {
assert_eq!(duration_to_rtp_ticks(Duration::from_millis(20), 8000), 160);
assert_eq!(duration_to_rtp_ticks(Duration::from_millis(20), 48000), 960);
assert_eq!(
duration_to_rtp_ticks(Duration::from_micros(2_500), 48000),
120
);
}
}