use crate::error::{Error, Result};
use crate::pipeline::{CodecConfig, Sample, TrackSpec};
use crate::rtcp::SenderReport;
use crate::rtp::{RtpMediaKind, parse_rtp_header, reassemble_audio, reassemble_video};
use alloc::collections::VecDeque;
use alloc::vec::Vec;
use broadcast_common::{Demand, Parse, Stage, Timestamp};
const NTP_FRACTION_SCALE: f64 = 4_294_967_296.0;
const MAX_AU_BUFFER_BYTES: usize = 4 * 1024 * 1024;
pub const DEFAULT_REORDER_DEPTH: usize = 16;
#[non_exhaustive]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RtpLossEvent {
SequenceGap {
track_id: u32,
ssrc: u32,
expected: u16,
got: u16,
},
DamagedAccessUnit {
track_id: u32,
},
}
#[non_exhaustive]
pub struct RtpStreamTrack {
pub track_id: u32,
pub kind: RtpMediaKind,
pub config: CodecConfig,
pub clock_rate: u32,
reorder_depth: usize,
}
impl RtpStreamTrack {
pub fn new(track_id: u32, kind: RtpMediaKind, config: CodecConfig, clock_rate: u32) -> Self {
Self {
track_id,
kind,
config,
clock_rate,
reorder_depth: DEFAULT_REORDER_DEPTH,
}
}
pub fn with_reorder_depth(mut self, reorder_depth: usize) -> Self {
self.reorder_depth = reorder_depth;
self
}
}
#[derive(Default)]
struct SeqState {
ssrc: Option<u32>,
expected: Option<u16>,
held: Vec<(u16, Vec<u8>)>,
}
struct Admission {
gap: Option<(u16, u16)>,
released: Vec<Vec<u8>>,
}
impl SeqState {
fn admit(&mut self, reorder_depth: usize, ssrc: u32, seq: u16, packet: &[u8]) -> Admission {
if self.ssrc != Some(ssrc) || self.expected.is_none() {
self.ssrc = Some(ssrc);
self.expected = Some(seq.wrapping_add(1));
self.held.clear();
return Admission {
gap: None,
released: alloc::vec![packet.to_vec()],
};
}
let expected = self.expected.expect("checked above");
let delta = seq.wrapping_sub(expected) as i16;
if delta == 0 {
self.expected = Some(seq.wrapping_add(1));
let mut released = alloc::vec![packet.to_vec()];
self.drain_contiguous(&mut released);
return Admission {
gap: None,
released,
};
}
if delta < 0 {
return Admission {
gap: None,
released: Vec::new(),
};
}
if self.held.iter().any(|(s, _)| *s == seq) {
return Admission {
gap: None,
released: Vec::new(),
};
}
self.held.push((seq, packet.to_vec()));
if self.held.len() <= reorder_depth {
return Admission {
gap: None,
released: Vec::new(),
};
}
let (expected, got, released) = self
.force_resolve()
.expect("held is non-empty: just pushed to it");
Admission {
gap: Some((expected, got)),
released,
}
}
fn force_resolve(&mut self) -> Option<(u16, u16, Vec<Vec<u8>>)> {
let expected = self.expected?;
if self.held.is_empty() {
return None;
}
let idx = self
.held
.iter()
.enumerate()
.min_by_key(|(_, (s, _))| s.wrapping_sub(expected))
.map(|(i, _)| i)?;
let (got, bytes) = self.held.remove(idx);
self.expected = Some(got.wrapping_add(1));
let mut released = alloc::vec![bytes];
self.drain_contiguous(&mut released);
Some((expected, got, released))
}
fn drain_contiguous(&mut self, out: &mut Vec<Vec<u8>>) {
while let Some(expected) = self.expected {
let Some(pos) = self.held.iter().position(|(s, _)| *s == expected) else {
break;
};
let (_, bytes) = self.held.remove(pos);
out.push(bytes);
self.expected = Some(expected.wrapping_add(1));
}
}
}
struct TrackState {
kind: RtpMediaKind,
config: CodecConfig,
clock_rate: u32,
cur_ts: Option<u32>,
cur_pkts: Vec<Vec<u8>>,
cur_bytes: usize,
last_unwrapped: Option<u64>,
first_unwrapped: Option<u64>,
pending: Option<PendingAu>,
last_duration: u32,
sr_anchor: Option<SrAnchor>,
reorder_depth: usize,
seq: SeqState,
}
struct SrAnchor {
ntp_seconds: f64,
raw_rtp_ts: u32,
}
fn wall_seconds(anchor: &SrAnchor, clock_rate: u32, au_unwrapped_ts: u64) -> f64 {
let anchor_unwrapped = unwrap_ts(Some(au_unwrapped_ts), anchor.raw_rtp_ts);
let delta_ticks = au_unwrapped_ts as i128 - i128::from(anchor_unwrapped);
anchor.ntp_seconds + (delta_ticks as f64) / f64::from(clock_rate)
}
struct PendingAu {
unwrapped_ts: u64,
is_sync: bool,
data: Vec<u8>,
}
pub struct RtpStreamDepacketiser {
tracks: Vec<(u32, TrackState)>,
stage_ready: VecDeque<Sample>,
loss_events: VecDeque<RtpLossEvent>,
}
fn unwrap_ts(prev: Option<u64>, ts: u32) -> u64 {
let Some(prev) = prev else {
return u64::from(ts);
};
let prev_low = prev as u32;
let delta = ts.wrapping_sub(prev_low) as i32;
if delta >= 0 {
prev + u64::from(delta as u32)
} else {
prev.saturating_sub(u64::from(delta.unsigned_abs()))
}
}
fn to_ticks(uw: u64) -> i64 {
uw.min(i64::MAX as u64) as i64
}
impl RtpStreamDepacketiser {
pub fn new(tracks: Vec<RtpStreamTrack>) -> Self {
let tracks = tracks
.into_iter()
.map(|t| {
(
t.track_id,
TrackState {
kind: t.kind,
config: t.config,
clock_rate: t.clock_rate,
cur_ts: None,
cur_pkts: Vec::new(),
cur_bytes: 0,
last_unwrapped: None,
first_unwrapped: None,
pending: None,
last_duration: 0,
sr_anchor: None,
reorder_depth: t.reorder_depth,
seq: SeqState::default(),
},
)
})
.collect();
Self {
tracks,
stage_ready: VecDeque::new(),
loss_events: VecDeque::new(),
}
}
pub fn track_specs(&self) -> Vec<TrackSpec> {
self.tracks
.iter()
.map(|(id, st)| TrackSpec::new(*id, st.clock_rate, st.config.clone()))
.collect()
}
fn state(&mut self, track_id: u32) -> Option<&mut TrackState> {
self.tracks
.iter_mut()
.find(|(id, _)| *id == track_id)
.map(|(_, st)| st)
}
pub fn poll_loss_event(&mut self) -> Option<RtpLossEvent> {
self.loss_events.pop_front()
}
pub fn push_sender_report(&mut self, track_id: u32, sr: SenderReport) {
if let Some(st) = self.state(track_id) {
let ntp_seconds = f64::from(sr.ntp_msw) + f64::from(sr.ntp_lsw) / NTP_FRACTION_SCALE;
st.sr_anchor = Some(SrAnchor {
ntp_seconds,
raw_rtp_ts: sr.rtp_timestamp,
});
}
}
pub fn push_rtcp(&mut self, track_id: u32, bytes: &[u8]) {
if let Ok(sr) = SenderReport::parse(bytes) {
self.push_sender_report(track_id, sr);
}
}
pub fn sync_start_decode_times(&self) -> Vec<(u32, u64)> {
let anchored: Vec<(u32, f64, u32)> = self
.tracks
.iter()
.filter_map(|(id, st)| {
let anchor = st.sr_anchor.as_ref()?;
let first_ts = st.first_unwrapped?;
Some((
*id,
wall_seconds(anchor, st.clock_rate, first_ts),
st.clock_rate,
))
})
.collect();
if anchored.len() < 2 {
return Vec::new();
}
let origin = anchored
.iter()
.map(|(_, wall, _)| *wall)
.fold(f64::INFINITY, f64::min);
anchored
.into_iter()
.map(|(id, wall, clock_rate)| {
let raw_ticks = ((wall - origin) * f64::from(clock_rate)).max(0.0);
let ticks = (raw_ticks + 0.5) as u64;
(id, ticks)
})
.collect()
}
pub fn push(&mut self, track_id: u32, rtp_packet: &[u8]) -> Result<Vec<Sample>> {
let hdr = parse_rtp_header(rtp_packet)?;
let RtpStreamDepacketiser {
tracks,
loss_events,
..
} = self;
let Some((_, st)) = tracks.iter_mut().find(|(id, _)| *id == track_id) else {
return Ok(Vec::new());
};
let adm = st
.seq
.admit(st.reorder_depth, hdr.ssrc, hdr.sequence, rtp_packet);
let mut out = Vec::new();
if let Some((expected, got)) = adm.gap {
st.cur_pkts.clear();
st.cur_bytes = 0;
st.cur_ts = None;
loss_events.push_back(RtpLossEvent::SequenceGap {
track_id,
ssrc: hdr.ssrc,
expected,
got,
});
}
for pkt in &adm.released {
Self::push_one(st, loss_events, track_id, pkt, &mut out)?;
}
Ok(out)
}
fn push_one(
st: &mut TrackState,
loss_events: &mut VecDeque<RtpLossEvent>,
track_id: u32,
rtp_packet: &[u8],
out: &mut Vec<Sample>,
) -> Result<()> {
let hdr = parse_rtp_header(rtp_packet)?;
let ts = hdr.timestamp;
if let Some(cur) = st.cur_ts {
if cur != ts && !st.cur_pkts.is_empty() {
Self::drain_complete_or_discard(st, loss_events, track_id, out);
}
}
st.cur_ts = Some(ts);
st.cur_bytes += rtp_packet.len();
st.cur_pkts.push(rtp_packet.to_vec());
if st.cur_bytes > MAX_AU_BUFFER_BYTES {
st.cur_pkts.clear();
st.cur_bytes = 0;
st.cur_ts = None;
return Err(Error::BufferCapExceeded {
what: "RTP access-unit reassembly",
cap: MAX_AU_BUFFER_BYTES,
});
}
if matches!(st.kind, RtpMediaKind::H264) && hdr.marker {
Self::drain_complete_or_discard(st, loss_events, track_id, out);
st.cur_ts = None;
}
Ok(())
}
fn drain_complete_or_discard(
st: &mut TrackState,
loss_events: &mut VecDeque<RtpLossEvent>,
track_id: u32,
out: &mut Vec<Sample>,
) {
if Self::drain_complete(st, out).is_err() {
loss_events.push_back(RtpLossEvent::DamagedAccessUnit { track_id });
}
}
pub fn flush(&mut self, track_id: u32) -> Result<Vec<Sample>> {
let RtpStreamDepacketiser {
tracks,
loss_events,
..
} = self;
let Some((_, st)) = tracks.iter_mut().find(|(id, _)| *id == track_id) else {
return Ok(Vec::new());
};
let mut out = Vec::new();
if let Some((expected, got, released)) = st.seq.force_resolve() {
let ssrc = st.seq.ssrc.unwrap_or(0);
st.cur_pkts.clear();
st.cur_bytes = 0;
st.cur_ts = None;
loss_events.push_back(RtpLossEvent::SequenceGap {
track_id,
ssrc,
expected,
got,
});
for pkt in &released {
Self::push_one(st, loss_events, track_id, pkt, &mut out)?;
}
}
if !st.cur_pkts.is_empty() {
Self::drain_complete_or_discard(st, loss_events, track_id, &mut out);
st.cur_ts = None;
}
if let Some(p) = st.pending.take() {
let ts = to_ticks(p.unwrapped_ts);
out.push(Sample::new(
p.data,
Some(ts),
Some(ts),
Some(st.last_duration),
p.is_sync,
));
}
Ok(out)
}
fn drain_complete(st: &mut TrackState, out: &mut Vec<Sample>) -> Result<()> {
let pkts = core::mem::take(&mut st.cur_pkts);
st.cur_bytes = 0;
let aus = match st.kind {
RtpMediaKind::H264 => reassemble_video(&pkts)?,
RtpMediaKind::Aac => reassemble_audio(&pkts)?,
};
for au in aus {
let unwrapped = unwrap_ts(st.last_unwrapped, au.timestamp);
st.last_unwrapped = Some(unwrapped);
if st.first_unwrapped.is_none() {
st.first_unwrapped = Some(unwrapped);
}
if let Some(prev) = st.pending.take() {
let delta = unwrapped.saturating_sub(prev.unwrapped_ts);
let duration = u32::try_from(delta).unwrap_or(u32::MAX);
st.last_duration = duration;
let ts = to_ticks(prev.unwrapped_ts);
out.push(Sample::new(
prev.data,
Some(ts),
Some(ts),
Some(duration),
prev.is_sync,
));
}
st.pending = Some(PendingAu {
unwrapped_ts: unwrapped,
is_sync: au.is_sync,
data: au.data,
});
}
Ok(())
}
}
impl Stage for RtpStreamDepacketiser {
type In<'a> = &'a [u8];
type Out = Sample;
type Error = Error;
fn feed(&mut self, input: &[u8], _now: Timestamp) -> Result<()> {
let [(track_id, _)] = self.tracks.as_slice() else {
return Err(Error::InvalidInput(
"Stage::feed requires RtpStreamDepacketiser to be constructed with exactly one track",
));
};
let track_id = *track_id;
let samples = self.push(track_id, input)?;
self.stage_ready.extend(samples);
Ok(())
}
fn poll(&mut self) -> Option<Sample> {
self.stage_ready.pop_front()
}
fn finish(&mut self) -> Result<()> {
let [(track_id, _)] = self.tracks.as_slice() else {
return Err(Error::InvalidInput(
"Stage::finish requires RtpStreamDepacketiser to be constructed with exactly one track",
));
};
let track_id = *track_id;
let samples = self.flush(track_id)?;
self.stage_ready.extend(samples);
Ok(())
}
fn next_deadline(&self) -> Option<Timestamp> {
None
}
fn on_deadline(&mut self, _now: Timestamp) {}
fn demand(&self) -> Demand {
let saturated = self
.tracks
.first()
.map(|(_, st)| st.cur_bytes >= MAX_AU_BUFFER_BYTES)
.unwrap_or(false);
if saturated {
Demand::saturated()
} else {
Demand::default()
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::avc_config::{AVCConfigurationBox, AVCDecoderConfigurationRecord};
fn dummy_avc() -> CodecConfig {
CodecConfig::Avc {
config: AVCConfigurationBox::new(AVCDecoderConfigurationRecord {
configuration_version: 1,
profile_indication: 0x42,
profile_compatibility: 0,
level_indication: 0x1E,
length_size_minus_one: 3,
sps: alloc::vec![],
pps: alloc::vec![],
chroma_format: None,
bit_depth_luma_minus8: None,
bit_depth_chroma_minus8: None,
sps_ext: alloc::vec![],
}),
width: 0,
height: 0,
}
}
fn vpkt(seq: u16, ts: u32, marker: bool, nal: &[u8]) -> Vec<u8> {
let mut p = alloc::vec![0x80u8, if marker { 0x80 | 96 } else { 96 }];
p.extend_from_slice(&seq.to_be_bytes());
p.extend_from_slice(&ts.to_be_bytes());
p.extend_from_slice(&[0, 0, 0, 0]);
p.extend_from_slice(nal);
p
}
#[test]
fn video_stream_recovers_durations_and_sync() {
let mut d = RtpStreamDepacketiser::new(alloc::vec![RtpStreamTrack::new(
1,
RtpMediaKind::H264,
dummy_avc(),
90_000,
)]);
let idr = [0x65u8, 0xAA];
let non = [0x41u8, 0xBB];
assert!(d.push(1, &vpkt(1, 1000, true, &idr)).unwrap().is_empty());
let s0 = d.push(1, &vpkt(2, 4000, true, &non)).unwrap();
assert_eq!(s0.len(), 1);
assert_eq!(s0[0].duration, Some(3000));
assert!(s0[0].flags.is_sync);
assert_eq!(s0[0].composition_offset(), 0);
let s1 = d.push(1, &vpkt(3, 7000, true, &non)).unwrap();
assert_eq!(s1.len(), 1);
assert_eq!(s1[0].duration, Some(3000));
assert!(!s1[0].flags.is_sync);
let s2 = d.flush(1).unwrap();
assert_eq!(s2.len(), 1);
assert_eq!(s2[0].duration, Some(3000));
}
#[test]
fn stage_feed_poll_finish_matches_push_flush() {
let idr = [0x65u8, 0xAA];
let non = [0x41u8, 0xBB];
let packets = [
vpkt(1, 1000, true, &idr),
vpkt(2, 4000, true, &non),
vpkt(3, 7000, true, &non),
];
let mut oracle = RtpStreamDepacketiser::new(alloc::vec![RtpStreamTrack::new(
1,
RtpMediaKind::H264,
dummy_avc(),
90_000,
)]);
let mut oracle_samples: Vec<Sample> = Vec::new();
for pkt in &packets {
oracle_samples.extend(oracle.push(1, pkt).unwrap());
}
oracle_samples.extend(oracle.flush(1).unwrap());
let mut staged = RtpStreamDepacketiser::new(alloc::vec![RtpStreamTrack::new(
1,
RtpMediaKind::H264,
dummy_avc(),
90_000,
)]);
let mut staged_samples: Vec<Sample> = Vec::new();
for pkt in &packets {
Stage::feed(&mut staged, pkt, Timestamp::ZERO).unwrap();
while let Some(s) = Stage::poll(&mut staged) {
staged_samples.push(s);
}
}
Stage::finish(&mut staged).unwrap();
while let Some(s) = Stage::poll(&mut staged) {
staged_samples.push(s);
}
assert_eq!(staged_samples.len(), oracle_samples.len());
for (s, o) in staged_samples.iter().zip(oracle_samples.iter()) {
assert_eq!(s.dts, o.dts);
assert_eq!(s.pts, o.pts);
assert_eq!(s.duration, o.duration);
assert_eq!(s.flags.is_sync, o.flags.is_sync);
assert_eq!(s.data, o.data);
}
}
#[test]
fn stage_feed_errors_on_multi_track_construction() {
let mut d = RtpStreamDepacketiser::new(alloc::vec![
RtpStreamTrack::new(1, RtpMediaKind::H264, dummy_avc(), 90_000),
RtpStreamTrack::new(2, RtpMediaKind::H264, dummy_avc(), 90_000),
]);
let err = Stage::feed(&mut d, &vpkt(1, 1000, true, &[0x65]), Timestamp::ZERO).unwrap_err();
assert!(matches!(err, Error::InvalidInput(_)));
let err2 = Stage::finish(&mut d).unwrap_err();
assert!(matches!(err2, Error::InvalidInput(_)));
}
fn fu_a_fragment(start: bool, extra_len: usize) -> Vec<u8> {
const NAL_TYPE_FU_A: u8 = 28;
const FU_START: u8 = 0x80;
const ORIG_TYPE_IDR: u8 = 5;
let fu_header = if start {
FU_START | ORIG_TYPE_IDR
} else {
ORIG_TYPE_IDR
};
let mut payload = alloc::vec![NAL_TYPE_FU_A, fu_header];
payload.extend(core::iter::repeat_n(0xABu8, extra_len));
payload
}
#[test]
fn runaway_fu_a_without_end_bit_is_bounded_not_unbounded() {
let mut d = RtpStreamDepacketiser::new(alloc::vec![RtpStreamTrack::new(
1,
RtpMediaKind::H264,
dummy_avc(),
90_000,
)]);
const FRAGMENT_FILLER: usize = 2048;
let mut hit_cap = false;
let mut resume_seq: u16 = 0;
for i in 0..4096u16 {
match d.push(
1,
&vpkt(i, 1000, false, &fu_a_fragment(i == 0, FRAGMENT_FILLER)),
) {
Ok(samples) => assert!(
samples.is_empty(),
"a never-completing AU must not emit a sample"
),
Err(e) => {
assert!(
matches!(e, crate::error::Error::BufferCapExceeded { .. }),
"unexpected error variant: {e:?}"
);
hit_cap = true;
resume_seq = i.wrapping_add(1);
break;
}
}
}
assert!(
hit_cap,
"expected MAX_AU_BUFFER_BYTES to trip well within {} fragments \
(never grow unbounded)",
4096
);
let idr = [0x65u8, 0xAA];
let non = [0x41u8, 0xBB];
assert!(
d.push(1, &vpkt(resume_seq, 4000, true, &idr))
.unwrap()
.is_empty()
);
let s = d
.push(1, &vpkt(resume_seq.wrapping_add(1), 7000, true, &non))
.unwrap();
assert_eq!(s.len(), 1);
assert_eq!(s[0].duration, Some(3000));
assert!(s[0].flags.is_sync);
assert!(
d.poll_loss_event().is_none(),
"no loss event expected: the byte-cap overflow is its own \
recorded corrective action, and the sequence numbers this \
resync uses are contiguous with the ones already accepted"
);
}
#[test]
fn track_specs_use_clock_rate_as_timescale() {
let d = RtpStreamDepacketiser::new(alloc::vec![RtpStreamTrack::new(
7,
RtpMediaKind::H264,
dummy_avc(),
90_000,
)]);
let specs = d.track_specs();
assert_eq!(specs.len(), 1);
assert_eq!(specs[0].track_id, 7);
assert_eq!(specs[0].timescale, 90_000);
}
}