use crate::TrackId;
use crate::error::{Error, Result};
use crate::pipeline::{CodecConfig, Sample, TrackSpec};
use crate::rtcp::SenderReport;
use crate::rtp::{RtpMediaKind, parse_rtp_header, reassemble_audio, reassemble_video};
use crate::rtp_dts::DtsDeriver;
use crate::slice_header::{AvcParamSets, AvcPicture};
use alloc::collections::{BTreeMap, VecDeque};
use alloc::vec::Vec;
use broadcast_common::{Demand, Parse, Stage, Timestamp};
const NTP_FRACTION_SCALE: f64 = 4_294_967_296.0;
const MAX_AU_BUFFER_BYTES: usize = 4 * 1024 * 1024;
pub const DEFAULT_REORDER_DEPTH: usize = 16;
const RTP_SEQ_MOD: u32 = 1 << 16;
const RTP_SEQ_MOD_PLUS_ONE: u32 = RTP_SEQ_MOD + 1;
const MAX_DROPOUT: u16 = 3000;
const MAX_MISORDER: u32 = 100;
const MIN_SEQUENTIAL: u8 = 2;
const MAX_WILD_JUMP_U16: u16 = (RTP_SEQ_MOD - MAX_MISORDER) as u16;
#[non_exhaustive]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RtpLossEvent {
SequenceGap {
track_id: TrackId,
ssrc: u32,
expected: u16,
got: u16,
},
DamagedAccessUnit {
track_id: TrackId,
},
NonMonotonicTimestamp {
track_id: TrackId,
previous_pts: i64,
pts: i64,
},
ReorderDepthExceeded {
track_id: TrackId,
depth: usize,
pts: i64,
dts: i64,
},
DroppedAccessUnit {
track_id: TrackId,
pts: i64,
},
}
#[non_exhaustive]
pub struct RtpStreamTrack {
pub track_id: TrackId,
pub kind: RtpMediaKind,
pub config: CodecConfig,
pub clock_rate: u32,
reorder_depth: usize,
}
impl RtpStreamTrack {
pub fn new(
track_id: TrackId,
kind: RtpMediaKind,
config: CodecConfig,
clock_rate: u32,
) -> Self {
Self {
track_id,
kind,
config,
clock_rate,
reorder_depth: DEFAULT_REORDER_DEPTH,
}
}
pub fn with_reorder_depth(mut self, reorder_depth: usize) -> Self {
self.reorder_depth = reorder_depth;
self
}
}
#[derive(Default)]
struct SeqState {
ssrc: Option<u32>,
expected: Option<u16>,
held: Vec<(u16, Vec<u8>)>,
validity: SeqValidity,
}
struct SeqValidity {
max_seq: u16,
bad_seq: u32,
cycles: u32,
probation: u8,
}
impl Default for SeqValidity {
fn default() -> Self {
Self {
max_seq: 0,
bad_seq: RTP_SEQ_MOD_PLUS_ONE,
cycles: 0,
probation: 0,
}
}
}
impl SeqValidity {
fn init_seq(&mut self, seq: u16) {
self.max_seq = seq;
self.bad_seq = RTP_SEQ_MOD_PLUS_ONE;
self.cycles = 0;
self.probation = 0;
}
fn update_seq(&mut self, seq: u16) -> SeqVerdict {
let udelta = seq.wrapping_sub(self.max_seq);
if self.probation != 0 {
if seq == self.max_seq.wrapping_add(1) {
self.probation -= 1;
self.max_seq = seq;
if self.probation == 0 {
self.init_seq(seq);
return SeqVerdict::Valid { restarted: false };
}
} else {
self.probation = MIN_SEQUENTIAL - 1;
self.max_seq = seq;
}
return SeqVerdict::Probation;
}
if udelta < MAX_DROPOUT {
if seq < self.max_seq {
self.cycles = self.cycles.wrapping_add(RTP_SEQ_MOD);
}
self.max_seq = seq;
SeqVerdict::Valid { restarted: false }
} else if udelta <= MAX_WILD_JUMP_U16 {
if u32::from(seq) == self.bad_seq {
self.init_seq(seq);
SeqVerdict::Valid { restarted: true }
} else {
self.bad_seq = u32::from(seq.wrapping_add(1)) & (RTP_SEQ_MOD - 1);
SeqVerdict::WildJumpArmed
}
} else {
SeqVerdict::Valid { restarted: false }
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum SeqVerdict {
Valid { restarted: bool },
Probation,
WildJumpArmed,
}
struct Admission {
gap: Option<(u16, u16)>,
resynced: bool,
released: Vec<Vec<u8>>,
}
impl SeqState {
fn admit(&mut self, reorder_depth: usize, ssrc: u32, seq: u16, packet: &[u8]) -> Admission {
if self.ssrc != Some(ssrc) || self.expected.is_none() {
self.ssrc = Some(ssrc);
self.expected = Some(seq.wrapping_add(1));
self.held.clear();
self.validity.max_seq = seq.wrapping_sub(1);
self.validity.bad_seq = RTP_SEQ_MOD_PLUS_ONE;
self.validity.cycles = 0;
self.validity.probation = MIN_SEQUENTIAL;
return Admission {
gap: None,
resynced: false,
released: alloc::vec![packet.to_vec()],
};
}
match self.validity.update_seq(seq) {
SeqVerdict::Probation => {
}
SeqVerdict::WildJumpArmed => {
return Admission {
gap: None,
resynced: false,
released: Vec::new(),
};
}
SeqVerdict::Valid { restarted } => {
if restarted {
self.expected = Some(seq.wrapping_add(1));
self.held.clear();
return Admission {
gap: None,
resynced: true,
released: alloc::vec![packet.to_vec()],
};
}
}
}
let expected = self.expected.expect("checked above");
if seq == expected {
self.expected = Some(seq.wrapping_add(1));
let mut released = alloc::vec![packet.to_vec()];
self.drain_contiguous(&mut released);
return Admission {
gap: None,
resynced: false,
released,
};
}
if (seq.wrapping_sub(expected) as i16) < 0 {
return Admission {
gap: None,
resynced: false,
released: Vec::new(),
};
}
if self.held.iter().any(|(s, _)| *s == seq) {
return Admission {
gap: None,
resynced: false,
released: Vec::new(),
};
}
self.held.push((seq, packet.to_vec()));
if self.held.len() <= reorder_depth {
return Admission {
gap: None,
resynced: false,
released: Vec::new(),
};
}
let (expected, got, released) = self
.force_resolve()
.expect("held is non-empty: just pushed to it");
Admission {
gap: Some((expected, got)),
resynced: false,
released,
}
}
fn force_resolve(&mut self) -> Option<(u16, u16, Vec<Vec<u8>>)> {
let expected = self.expected?;
if self.held.is_empty() {
return None;
}
let idx = self
.held
.iter()
.enumerate()
.min_by_key(|(_, (s, _))| s.wrapping_sub(expected))
.map(|(i, _)| i)?;
let (got, bytes) = self.held.remove(idx);
self.expected = Some(got.wrapping_add(1));
let mut released = alloc::vec![bytes];
self.drain_contiguous(&mut released);
Some((expected, got, released))
}
fn drain_contiguous(&mut self, out: &mut Vec<Vec<u8>>) {
while let Some(expected) = self.expected {
let Some(pos) = self.held.iter().position(|(s, _)| *s == expected) else {
break;
};
let (_, bytes) = self.held.remove(pos);
out.push(bytes);
self.expected = Some(expected.wrapping_add(1));
}
}
}
struct TrackState {
kind: RtpMediaKind,
config: CodecConfig,
clock_rate: u32,
cur_ts: Option<u32>,
cur_pkts: Vec<Vec<u8>>,
cur_bytes: usize,
last_unwrapped: Option<u64>,
first_unwrapped: Option<u64>,
pending: Vec<PendingAu>,
last_duration: u32,
declared_duration: Option<u32>,
reordered: bool,
seen_pts: BTreeMap<u64, u64>,
next_seq: u64,
au_window: usize,
min_positive_step: Option<i64>,
sr_anchor: Option<SrAnchor>,
reorder_depth: usize,
seq: SeqState,
dts: DtsDeriver,
param_sets: AvcParamSets,
}
struct SrAnchor {
ntp_seconds: f64,
raw_rtp_ts: u32,
}
fn wall_seconds(anchor: &SrAnchor, clock_rate: u32, au_unwrapped_ts: u64) -> f64 {
let anchor_unwrapped = unwrap_ts(Some(au_unwrapped_ts), anchor.raw_rtp_ts);
let delta_ticks = au_unwrapped_ts as i128 - i128::from(anchor_unwrapped);
anchor.ntp_seconds + (delta_ticks as f64) / f64::from(clock_rate)
}
struct PendingAu {
unwrapped_ts: u64,
is_sync: bool,
data: Vec<u8>,
}
pub struct RtpStreamDepacketiser {
tracks: Vec<(TrackId, TrackState)>,
stage_ready: VecDeque<Sample>,
loss_events: VecDeque<RtpLossEvent>,
}
fn unwrap_ts(prev: Option<u64>, ts: u32) -> u64 {
let Some(prev) = prev else {
return u64::from(ts);
};
let prev_low = prev as u32;
let delta = ts.wrapping_sub(prev_low) as i32;
if delta >= 0 {
prev + u64::from(delta as u32)
} else {
prev.saturating_sub(u64::from(delta.unsigned_abs()))
}
}
fn to_ticks(uw: u64) -> i64 {
uw.min(i64::MAX as u64) as i64
}
pub(crate) const MAX_REORDER_DEPTH: usize = 16;
const MAX_REORDER_WINDOW: usize = 256;
fn declared_au_duration(kind: RtpMediaKind, config: &CodecConfig, clock_rate: u32) -> Option<u32> {
let ticks = match (kind, config) {
(RtpMediaKind::H264, CodecConfig::Avc { config, .. }) => {
let sps = config.config.sps.first()?;
let timing = crate::sps::decode_avc_vui_timing(&sps.0).ok().flatten()?;
if timing.num_units_in_tick == 0 || timing.time_scale == 0 {
return None;
}
let num =
u64::from(clock_rate) * VUI_UNITS_PER_FRAME * u64::from(timing.num_units_in_tick);
let den = u64::from(timing.time_scale);
(num + den / 2) / den
}
(RtpMediaKind::Aac, CodecConfig::Aac { esds, .. }) => {
let asc =
crate::aac_asc::AudioSpecificConfig::parse(esds.decoder_specific_info_data()?)
.ok()?;
let rate = u64::from(asc.effective_sampling_frequency()?);
if rate == 0 {
return None;
}
let num = u64::from(asc.samples_per_frame()?) * u64::from(clock_rate);
(num + rate / 2) / rate
}
_ => return None,
};
u32::try_from(ticks).ok().filter(|&d| d > 0)
}
fn declared_reorder_depth(kind: RtpMediaKind, config: &CodecConfig) -> Option<usize> {
match (kind, config) {
(RtpMediaKind::H264, CodecConfig::Avc { config, .. }) => {
let sps = config.config.sps.first()?;
let depth = crate::sps::decode_avc_reorder_depth(&sps.0).ok()?.depth()?;
usize::try_from(depth).ok()
}
_ => None,
}
}
fn seed_param_sets(param_sets: &mut AvcParamSets, config: &CodecConfig) {
if let CodecConfig::Avc { config, .. } = config {
for nal in config
.config
.sps
.iter()
.map(|n| &n.0)
.chain(config.config.pps.iter().map(|n| &n.0))
{
param_sets.observe_nal(nal);
}
}
}
fn sample_is_field_picture(param_sets: &mut AvcParamSets, sample: &Sample) -> bool {
crate::annexb::iter_length_prefixed_nals(&sample.data)
.map(|nals| matches!(param_sets.classify_nals(nals), AvcPicture::Field(_)))
.unwrap_or(false)
}
fn derive_dts(
st: &mut TrackState,
loss_events: &mut VecDeque<RtpLossEvent>,
track_id: TrackId,
raw: Vec<Sample>,
finish: bool,
) -> Vec<Sample> {
if st.kind != RtpMediaKind::H264 {
return raw;
}
let period = known_duration(st).map(i64::from);
let mut out = Vec::with_capacity(raw.len());
let mut exceeded = Vec::new();
for s in raw {
if sample_is_field_picture(&mut st.param_sets, &s) {
st.dts.declare_field_coded();
}
st.dts.feed(s, period, &mut out, &mut exceeded);
}
if finish {
st.dts.finish(period, &mut out, &mut exceeded);
}
for e in exceeded {
loss_events.push_back(RtpLossEvent::ReorderDepthExceeded {
track_id,
depth: e.depth,
pts: e.pts,
dts: e.dts,
});
}
out
}
const VUI_UNITS_PER_FRAME: u64 = 2;
fn known_duration(st: &TrackState) -> Option<u32> {
if st.last_duration != 0 {
return Some(st.last_duration);
}
if st.kind == RtpMediaKind::Aac && st.declared_duration.is_some() {
return st.declared_duration;
}
if let Some(p) = st.min_positive_step
&& let Ok(p) = u32::try_from(p)
&& p > 0
{
return Some(p);
}
st.declared_duration
}
impl RtpStreamDepacketiser {
pub fn new(tracks: Vec<RtpStreamTrack>) -> Self {
let tracks = tracks
.into_iter()
.map(|t| {
let declared_duration = declared_au_duration(t.kind, &t.config, t.clock_rate);
let reorder_depth = declared_reorder_depth(t.kind, &t.config);
let mut param_sets = AvcParamSets::new();
seed_param_sets(&mut param_sets, &t.config);
(
t.track_id,
TrackState {
kind: t.kind,
config: t.config,
clock_rate: t.clock_rate,
cur_ts: None,
cur_pkts: Vec::new(),
cur_bytes: 0,
last_unwrapped: None,
first_unwrapped: None,
pending: Vec::new(),
last_duration: 0,
declared_duration,
reordered: false,
seen_pts: BTreeMap::new(),
next_seq: 0,
au_window: 0,
min_positive_step: None,
sr_anchor: None,
reorder_depth: t.reorder_depth,
seq: SeqState::default(),
dts: DtsDeriver::new(reorder_depth),
param_sets,
},
)
})
.collect();
Self {
tracks,
stage_ready: VecDeque::new(),
loss_events: VecDeque::new(),
}
}
pub fn track_specs(&self) -> Vec<TrackSpec> {
self.tracks
.iter()
.map(|(id, st)| TrackSpec::new(*id, st.clock_rate, st.config.clone()))
.collect()
}
pub fn set_track_config(&mut self, track_id: TrackId, config: CodecConfig) {
if let Some((_, st)) = self.tracks.iter_mut().find(|(id, _)| *id == track_id) {
st.declared_duration = declared_au_duration(st.kind, &config, st.clock_rate);
st.dts
.set_declared(declared_reorder_depth(st.kind, &config));
seed_param_sets(&mut st.param_sets, &config);
st.config = config;
}
}
fn state(&mut self, track_id: TrackId) -> Option<&mut TrackState> {
self.tracks
.iter_mut()
.find(|(id, _)| *id == track_id)
.map(|(_, st)| st)
}
pub fn poll_loss_event(&mut self) -> Option<RtpLossEvent> {
self.loss_events.pop_front()
}
pub fn push_sender_report(&mut self, track_id: TrackId, sr: SenderReport) {
if let Some(st) = self.state(track_id) {
let ntp_seconds = f64::from(sr.ntp_msw) + f64::from(sr.ntp_lsw) / NTP_FRACTION_SCALE;
st.sr_anchor = Some(SrAnchor {
ntp_seconds,
raw_rtp_ts: sr.rtp_timestamp,
});
}
}
pub fn push_rtcp(&mut self, track_id: TrackId, bytes: &[u8]) {
if let Ok(sr) = SenderReport::parse(bytes) {
self.push_sender_report(track_id, sr);
}
}
pub fn sync_start_decode_times(&self) -> Vec<(TrackId, u64)> {
let anchored: Vec<(TrackId, f64, u32)> = self
.tracks
.iter()
.filter_map(|(id, st)| {
let anchor = st.sr_anchor.as_ref()?;
let first_ts = st.first_unwrapped?;
Some((
*id,
wall_seconds(anchor, st.clock_rate, first_ts),
st.clock_rate,
))
})
.collect();
if anchored.len() < 2 {
return Vec::new();
}
let origin = anchored
.iter()
.map(|(_, wall, _)| *wall)
.fold(f64::INFINITY, f64::min);
anchored
.into_iter()
.map(|(id, wall, clock_rate)| {
let raw_ticks = ((wall - origin) * f64::from(clock_rate)).max(0.0);
let ticks = (raw_ticks + 0.5) as u64;
(id, ticks)
})
.collect()
}
pub fn push(&mut self, track_id: TrackId, rtp_packet: &[u8]) -> Result<Vec<Sample>> {
let hdr = parse_rtp_header(rtp_packet)?;
let RtpStreamDepacketiser {
tracks,
loss_events,
..
} = self;
let Some((_, st)) = tracks.iter_mut().find(|(id, _)| *id == track_id) else {
return Ok(Vec::new());
};
let adm = st
.seq
.admit(st.reorder_depth, hdr.ssrc, hdr.sequence, rtp_packet);
let mut out = Vec::new();
if adm.resynced {
let mut held = Vec::new();
Self::drain_pending(st, loss_events, track_id, &mut held);
for p in st.pending.drain(..) {
loss_events.push_back(RtpLossEvent::DroppedAccessUnit {
track_id,
pts: to_ticks(p.unwrapped_ts),
});
}
st.cur_pkts.clear();
st.cur_bytes = 0;
st.cur_ts = None;
st.last_unwrapped = None;
st.first_unwrapped = None;
st.reordered = false;
st.seen_pts.clear();
st.next_seq = 0;
st.au_window = 0;
st.min_positive_step = None;
let mut flushed = derive_dts(st, loss_events, track_id, held, true);
for pkt in adm.released {
Self::push_one(st, loss_events, track_id, pkt, &mut out)?;
}
flushed.extend(derive_dts(st, loss_events, track_id, out, false));
return Ok(flushed);
}
if let Some((expected, got)) = adm.gap {
st.cur_pkts.clear();
st.cur_bytes = 0;
st.cur_ts = None;
loss_events.push_back(RtpLossEvent::SequenceGap {
track_id,
ssrc: hdr.ssrc,
expected,
got,
});
}
for pkt in adm.released {
Self::push_one(st, loss_events, track_id, pkt, &mut out)?;
}
Ok(derive_dts(st, loss_events, track_id, out, false))
}
fn push_one(
st: &mut TrackState,
loss_events: &mut VecDeque<RtpLossEvent>,
track_id: TrackId,
rtp_packet: Vec<u8>,
out: &mut Vec<Sample>,
) -> Result<()> {
let hdr = parse_rtp_header(&rtp_packet)?;
let ts = hdr.timestamp;
let marker = hdr.marker;
if let Some(cur) = st.cur_ts
&& cur != ts
&& !st.cur_pkts.is_empty()
{
Self::drain_complete_or_discard(st, loss_events, track_id, out);
}
st.cur_ts = Some(ts);
st.cur_bytes += rtp_packet.len();
st.cur_pkts.push(rtp_packet);
if st.cur_bytes > MAX_AU_BUFFER_BYTES {
st.cur_pkts.clear();
st.cur_bytes = 0;
st.cur_ts = None;
return Err(Error::BufferCapExceeded {
what: "RTP access-unit reassembly",
cap: MAX_AU_BUFFER_BYTES,
});
}
if matches!(st.kind, RtpMediaKind::H264) && marker {
Self::drain_complete_or_discard(st, loss_events, track_id, out);
st.cur_ts = None;
}
Ok(())
}
fn drain_complete_or_discard(
st: &mut TrackState,
loss_events: &mut VecDeque<RtpLossEvent>,
track_id: TrackId,
out: &mut Vec<Sample>,
) {
if Self::drain_complete(st, loss_events, track_id, out).is_err() {
loss_events.push_back(RtpLossEvent::DamagedAccessUnit { track_id });
}
}
pub fn flush(&mut self, track_id: TrackId) -> Result<Vec<Sample>> {
let RtpStreamDepacketiser {
tracks,
loss_events,
..
} = self;
let Some((_, st)) = tracks.iter_mut().find(|(id, _)| *id == track_id) else {
return Ok(Vec::new());
};
let mut out = Vec::new();
if let Some((expected, got, released)) = st.seq.force_resolve() {
let ssrc = st.seq.ssrc.unwrap_or(0);
st.cur_pkts.clear();
st.cur_bytes = 0;
st.cur_ts = None;
loss_events.push_back(RtpLossEvent::SequenceGap {
track_id,
ssrc,
expected,
got,
});
for pkt in released {
Self::push_one(st, loss_events, track_id, pkt, &mut out)?;
}
}
if !st.cur_pkts.is_empty() {
Self::drain_complete_or_discard(st, loss_events, track_id, &mut out);
st.cur_ts = None;
}
Self::drain_pending(st, loss_events, track_id, &mut out);
Ok(derive_dts(st, loss_events, track_id, out, true))
}
fn drain_pending(
st: &mut TrackState,
loss_events: &mut VecDeque<RtpLossEvent>,
track_id: TrackId,
out: &mut Vec<Sample>,
) {
if st.reordered && st.kind == RtpMediaKind::H264 {
Self::resolve_window(st, loss_events, track_id, out, true);
} else if let Some(duration) = known_duration(st) {
for (index, p) in st.pending.drain(..).enumerate() {
let offset = u64::from(duration) * index as u64;
let ts = to_ticks(p.unwrapped_ts.saturating_add(offset));
out.push(Sample::new(
p.data,
Some(ts),
Some(ts),
Some(duration),
p.is_sync,
));
}
}
}
fn resolve_window(
st: &mut TrackState,
loss_events: &mut VecDeque<RtpLossEvent>,
track_id: TrackId,
out: &mut Vec<Sample>,
drain_all: bool,
) {
while !st.pending.is_empty() {
let head_ts = st.pending[0].unwrapped_ts;
let successor = st
.seen_pts
.range(head_ts.saturating_add(1)..)
.next()
.map(|(&t, _)| t - head_ts);
let period = st.declared_duration.map(u64::from).or_else(|| {
st.min_positive_step
.and_then(|p| u64::try_from(p).ok())
.filter(|&p| p > 0)
});
let ready = drain_all
|| match period {
Some(p) => {
successor.is_some_and(|gap| gap <= p)
|| st.pending.len() > MAX_REORDER_DEPTH
}
None => st.pending.len() > st.au_window,
};
if !ready {
break;
}
let Some(duration) = successor
.or(period)
.and_then(|d| u32::try_from(d).ok())
.filter(|&d| d > 0)
else {
break;
};
let p = st.pending.remove(0);
let ts = to_ticks(p.unwrapped_ts);
st.last_duration = duration;
if let Some(oldest) = st.pending.iter().map(|p| p.unwrapped_ts).min() {
while st
.seen_pts
.first_key_value()
.is_some_and(|(&t, _)| t < oldest)
{
st.seen_pts.pop_first();
}
}
out.push(Sample::new(
p.data,
Some(ts),
Some(ts),
Some(duration),
p.is_sync,
));
}
if st.pending.len() > MAX_REORDER_WINDOW {
let excess = st.pending.len() - MAX_REORDER_WINDOW;
for p in st.pending.drain(..excess) {
loss_events.push_back(RtpLossEvent::DroppedAccessUnit {
track_id,
pts: to_ticks(p.unwrapped_ts),
});
}
}
}
fn drain_complete(
st: &mut TrackState,
loss_events: &mut VecDeque<RtpLossEvent>,
track_id: TrackId,
out: &mut Vec<Sample>,
) -> Result<()> {
let pkts = core::mem::take(&mut st.cur_pkts);
st.cur_bytes = 0;
let aus = match st.kind {
RtpMediaKind::H264 => reassemble_video(&pkts)?,
RtpMediaKind::Aac => reassemble_audio(&pkts)?,
};
for au in aus {
let unwrapped = unwrap_ts(st.last_unwrapped, au.timestamp);
if let Some(prev) = st.last_unwrapped
&& unwrapped < prev
{
loss_events.push_back(RtpLossEvent::NonMonotonicTimestamp {
track_id,
previous_pts: to_ticks(prev),
pts: to_ticks(unwrapped),
});
}
if let Some(prev) = st.last_unwrapped {
let step = unwrapped as i64 - prev as i64;
if step > 0 && st.min_positive_step.is_none_or(|p| step < p) {
st.min_positive_step = Some(step);
}
}
st.last_unwrapped = Some(unwrapped);
if st.first_unwrapped.is_none() {
st.first_unwrapped = Some(unwrapped);
}
if st.kind == RtpMediaKind::H264
&& (st.reordered
|| st
.pending
.last()
.is_some_and(|p| unwrapped < p.unwrapped_ts))
{
if !st.reordered {
let first = st.next_seq - st.pending.len() as u64;
for (i, p) in st.pending.iter().enumerate() {
st.seen_pts.insert(p.unwrapped_ts, first + i as u64);
}
}
st.reordered = true;
if let Some((_, &pred)) = st.seen_pts.range(..unwrapped).next_back() {
let lag = usize::try_from(st.next_seq - pred).unwrap_or(usize::MAX);
st.au_window = st.au_window.max(lag.min(MAX_REORDER_DEPTH));
}
st.seen_pts.insert(unwrapped, st.next_seq);
st.next_seq += 1;
st.pending.push(PendingAu {
unwrapped_ts: unwrapped,
is_sync: au.is_sync,
data: au.data,
});
Self::resolve_window(st, loss_events, track_id, out, false);
continue;
}
let same_packet_group = st.kind == RtpMediaKind::Aac
&& st
.pending
.last()
.is_some_and(|p| p.unwrapped_ts == unwrapped);
if !same_packet_group && !st.pending.is_empty() {
let prev_ts = st.pending[0].unwrapped_ts;
let forward = unwrapped.saturating_sub(prev_ts);
let count = st.pending.len() as u64;
let delta = if forward > 0 {
Some(forward)
} else {
st.min_positive_step
.and_then(|p| u64::try_from(p).ok())
.filter(|&p| p > 0)
.or_else(|| st.declared_duration.map(|d| u64::from(d) * count))
};
let group: Vec<PendingAu> = st.pending.drain(..).collect();
let share = delta.map(|d| d / count).filter(|&e| e > 0);
match (delta, share) {
(Some(delta), Some(each)) => {
for (index, p) in group.into_iter().enumerate() {
let offset = each * index as u64;
let duration = if index as u64 + 1 == count {
delta - offset
} else {
each
};
let duration = u32::try_from(duration).unwrap_or(u32::MAX);
st.last_duration = duration;
let ts = to_ticks(p.unwrapped_ts.saturating_add(offset));
out.push(Sample::new(
p.data,
Some(ts),
Some(ts),
Some(duration),
p.is_sync,
));
}
}
_ => {
let per_au = (st.last_duration != 0)
.then_some(st.last_duration)
.or(st.declared_duration);
if let Some(each) = per_au {
for (index, p) in group.into_iter().enumerate() {
let offset = u64::from(each) * index as u64;
let ts = to_ticks(p.unwrapped_ts.saturating_add(offset));
out.push(Sample::new(
p.data,
Some(ts),
Some(ts),
Some(each),
p.is_sync,
));
}
}
}
}
}
st.pending.push(PendingAu {
unwrapped_ts: unwrapped,
is_sync: au.is_sync,
data: au.data,
});
st.next_seq += 1;
}
Ok(())
}
}
impl Stage for RtpStreamDepacketiser {
type In<'a> = &'a [u8];
type Out = Sample;
type Error = Error;
fn feed(&mut self, input: &[u8], _now: Timestamp) -> Result<()> {
let [(track_id, _)] = self.tracks.as_slice() else {
return Err(Error::InvalidInput(
"Stage::feed requires RtpStreamDepacketiser to be constructed with exactly one track",
));
};
let track_id = *track_id;
let samples = self.push(track_id, input)?;
self.stage_ready.extend(samples);
Ok(())
}
fn poll(&mut self) -> Option<Sample> {
self.stage_ready.pop_front()
}
fn finish(&mut self) -> Result<()> {
let [(track_id, _)] = self.tracks.as_slice() else {
return Err(Error::InvalidInput(
"Stage::finish requires RtpStreamDepacketiser to be constructed with exactly one track",
));
};
let track_id = *track_id;
let samples = self.flush(track_id)?;
self.stage_ready.extend(samples);
Ok(())
}
fn next_deadline(&self) -> Option<Timestamp> {
None
}
fn on_deadline(&mut self, _now: Timestamp) {}
fn demand(&self) -> Demand {
let saturated = self
.tracks
.first()
.map(|(_, st)| st.cur_bytes >= MAX_AU_BUFFER_BYTES)
.unwrap_or(false);
if saturated {
Demand::saturated()
} else {
Demand::default()
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::avc_config::{AVCConfigurationBox, AVCDecoderConfigurationRecord};
fn dummy_avc() -> CodecConfig {
CodecConfig::Avc {
config: AVCConfigurationBox::new(AVCDecoderConfigurationRecord {
configuration_version: 1,
profile_indication: 0x42,
profile_compatibility: 0,
level_indication: 0x1E,
length_size_minus_one: 3,
sps: alloc::vec![],
pps: alloc::vec![],
chroma_format: None,
bit_depth_luma_minus8: None,
bit_depth_chroma_minus8: None,
sps_ext: alloc::vec![],
}),
width: 0,
height: 0,
}
}
const SPS_25FPS_FIXED: &[u8] = &[
0x67, 0x64, 0x00, 0x0D, 0xAC, 0xD9, 0x41, 0x41, 0xFB, 0x01, 0x10, 0x00, 0x00, 0x03, 0x00,
0x10, 0x00, 0x00, 0x03, 0x03, 0x20, 0xF1, 0x42, 0x99, 0x60,
];
const SPS_NO_TIMING: &[u8] = &[
0x67, 0x64, 0x00, 0x0D, 0xAD, 0xC8, 0xBF, 0xFE, 0x03, 0xC1, 0x41, 0xF9,
];
const ASC_LC_48K_STEREO_1024: [u8; 2] = [0x11, 0x90];
const ASC_LC_48K_STEREO_960: [u8; 2] = [0x11, 0x94];
fn avc_with_sps(sps: &[u8]) -> CodecConfig {
let CodecConfig::Avc {
mut config,
width,
height,
} = dummy_avc()
else {
unreachable!()
};
config.config.sps = alloc::vec![crate::nalu_types::AvcSps(sps.to_vec())];
CodecConfig::Avc {
config,
width,
height,
}
}
fn declare_reorder_depth(d: &mut RtpStreamDepacketiser, depth: usize) {
for (_, st) in &mut d.tracks {
st.dts = DtsDeriver::new(Some(depth));
}
}
fn single_track(kind: RtpMediaKind, config: CodecConfig, clock: u32) -> RtpStreamDepacketiser {
RtpStreamDepacketiser::new(alloc::vec![RtpStreamTrack::new(
TrackId::new(1),
kind,
config,
clock,
)])
}
#[test]
fn test_sps_fixtures_declare_what_the_tests_assume() {
let t = crate::sps::decode_avc_vui_timing(SPS_25FPS_FIXED)
.unwrap()
.unwrap();
assert!(t.num_units_in_tick == 1 && t.time_scale == 50);
assert!(
crate::sps::decode_avc_vui_timing(SPS_NO_TIMING)
.unwrap()
.is_none()
);
}
#[test]
fn flush_after_first_h264_au_uses_sps_vui_frame_period() {
let id = TrackId::new(1);
let mut d = single_track(RtpMediaKind::H264, avc_with_sps(SPS_25FPS_FIXED), 90_000);
assert!(
d.push(id, &vpkt(1, 1000, true, &[0x65, 0xAA]))
.unwrap()
.is_empty()
);
let out = d.flush(id).unwrap();
assert_eq!(out.len(), 1);
assert_eq!(out[0].duration, Some(90_000 / 25));
}
#[test]
fn set_track_config_refreshes_the_declared_duration() {
let id = TrackId::new(1);
let mut d = single_track(RtpMediaKind::H264, avc_with_sps(SPS_NO_TIMING), 90_000);
d.set_track_config(id, avc_with_sps(SPS_25FPS_FIXED));
assert!(
d.push(id, &vpkt(1, 1000, true, &[0x65, 0xAA]))
.unwrap()
.is_empty()
);
assert_eq!(d.flush(id).unwrap()[0].duration, Some(3600));
}
#[test]
fn flush_without_any_known_duration_holds_instead_of_inventing_one() {
let id = TrackId::new(1);
let mut d = single_track(RtpMediaKind::H264, avc_with_sps(SPS_NO_TIMING), 90_000);
assert!(
d.push(id, &vpkt(1, 1000, true, &[0x65, 0xAA]))
.unwrap()
.is_empty()
);
assert!(d.flush(id).unwrap().is_empty());
let out = d.push(id, &vpkt(2, 4000, true, &[0x41, 0xBB])).unwrap();
assert_eq!(out.len(), 1);
assert_eq!(out[0].duration, Some(3000));
}
#[test]
fn flush_after_first_aac_au_uses_asc_frame_length() {
let id = TrackId::new(1);
for (asc, expected) in [(ASC_LC_48K_STEREO_1024, 1024), (ASC_LC_48K_STEREO_960, 960)] {
let cfg = crate::rtp_sdp::aac_config_from_asc_bytes(asc.to_vec()).unwrap();
let mut d = single_track(RtpMediaKind::Aac, cfg, 48_000);
assert!(
d.push(id, &aac_packet(1, 5000, &[&[1, 2]]))
.unwrap()
.is_empty()
);
let out = d.flush(id).unwrap();
assert_eq!(out.len(), 1);
assert_eq!(out[0].duration, Some(expected));
}
}
#[test]
fn aggregated_group_with_delta_below_au_count_uses_declared_au_duration() {
let id = TrackId::new(1);
let cfg =
crate::rtp_sdp::aac_config_from_asc_bytes(ASC_LC_48K_STEREO_1024.to_vec()).unwrap();
let mut d = single_track(RtpMediaKind::Aac, cfg, 48_000);
let aus: [&[u8]; 4] = [&[1], &[2], &[3], &[4]];
assert!(d.push(id, &aac_packet(1, 10_000, &aus)).unwrap().is_empty());
assert!(
d.push(id, &aac_packet(2, 10_002, &[&[5]]))
.unwrap()
.is_empty()
);
let out = d.push(id, &aac_packet(3, 12_000, &[&[6]])).unwrap();
let got: Vec<_> = out.iter().map(|s| (s.pts, s.duration)).collect();
assert_eq!(
got,
alloc::vec![
(Some(10_000), Some(1024)),
(Some(11_024), Some(1024)),
(Some(12_048), Some(1024)),
(Some(13_072), Some(1024)),
]
);
}
#[test]
fn aggregated_group_with_delta_below_au_count_and_no_known_duration_is_not_emitted() {
let id = TrackId::new(1);
let mut d = single_track(RtpMediaKind::Aac, dummy_avc(), 48_000);
let aus: [&[u8]; 4] = [&[1], &[2], &[3], &[4]];
assert!(d.push(id, &aac_packet(1, 10_000, &aus)).unwrap().is_empty());
assert!(
d.push(id, &aac_packet(2, 10_002, &[&[5]]))
.unwrap()
.is_empty()
);
let out = d.push(id, &aac_packet(3, 12_000, &[&[6]])).unwrap();
assert!(out.is_empty());
}
fn aac_packet(seq: u16, ts: u32, aus: &[&[u8]]) -> Vec<u8> {
let mut p = alloc::vec![0x80u8, 96];
p.extend_from_slice(&seq.to_be_bytes());
p.extend_from_slice(&ts.to_be_bytes());
p.extend_from_slice(&[0, 0, 0, 0]);
let headers_bits = (aus.len() * 16) as u16;
p.extend_from_slice(&headers_bits.to_be_bytes());
for au in aus {
p.extend_from_slice(&((au.len() as u16) << 3).to_be_bytes());
}
for au in aus {
p.extend_from_slice(au);
}
p
}
#[test]
fn aggregated_aac_packet_spreads_its_timestamp_over_its_aus() {
let id = TrackId::new(2);
let mut d = RtpStreamDepacketiser::new(alloc::vec![RtpStreamTrack::new(
id,
RtpMediaKind::Aac,
dummy_avc(),
48_000,
)]);
let aus: [&[u8]; 4] = [&[1, 1], &[2, 2], &[3, 3], &[4, 4]];
assert!(d.push(id, &aac_packet(1, 10_000, &aus)).unwrap().is_empty());
assert!(
d.push(id, &aac_packet(2, 14_096, &[&[5, 5]]))
.unwrap()
.is_empty()
);
let out = d.push(id, &aac_packet(3, 15_120, &[&[6, 6]])).unwrap();
let got: Vec<_> = out.iter().map(|s| (s.pts, s.duration)).collect();
assert_eq!(
got,
alloc::vec![
(Some(10_000), Some(1024)),
(Some(11_024), Some(1024)),
(Some(12_048), Some(1024)),
(Some(13_072), Some(1024)),
]
);
}
fn vpkt(seq: u16, ts: u32, marker: bool, nal: &[u8]) -> Vec<u8> {
let mut p = alloc::vec![0x80u8, if marker { 0x80 | 96 } else { 96 }];
p.extend_from_slice(&seq.to_be_bytes());
p.extend_from_slice(&ts.to_be_bytes());
p.extend_from_slice(&[0, 0, 0, 0]);
p.extend_from_slice(nal);
p
}
#[test]
fn video_stream_recovers_durations_and_sync() {
let mut d = RtpStreamDepacketiser::new(alloc::vec![RtpStreamTrack::new(
TrackId::new(1),
RtpMediaKind::H264,
dummy_avc(),
90_000,
)]);
declare_reorder_depth(&mut d, 0);
let idr = [0x65u8, 0xAA];
let non = [0x41u8, 0xBB];
assert!(
d.push(TrackId::new(1), &vpkt(1, 1000, true, &idr))
.unwrap()
.is_empty()
);
let s0 = d.push(TrackId::new(1), &vpkt(2, 4000, true, &non)).unwrap();
assert_eq!(s0.len(), 1);
assert_eq!(s0[0].duration, Some(3000));
assert!(s0[0].flags.is_sync);
assert_eq!(s0[0].composition_offset(), 0);
let s1 = d.push(TrackId::new(1), &vpkt(3, 7000, true, &non)).unwrap();
assert_eq!(s1.len(), 1);
assert_eq!(s1[0].duration, Some(3000));
assert!(!s1[0].flags.is_sync);
let s2 = d.flush(TrackId::new(1)).unwrap();
assert_eq!(s2.len(), 1);
assert_eq!(s2[0].duration, Some(3000));
}
#[test]
fn stage_feed_poll_finish_matches_push_flush() {
let idr = [0x65u8, 0xAA];
let non = [0x41u8, 0xBB];
let packets = [
vpkt(1, 1000, true, &idr),
vpkt(2, 4000, true, &non),
vpkt(3, 7000, true, &non),
];
let mut oracle = RtpStreamDepacketiser::new(alloc::vec![RtpStreamTrack::new(
TrackId::new(1),
RtpMediaKind::H264,
dummy_avc(),
90_000,
)]);
let mut oracle_samples: Vec<Sample> = Vec::new();
for pkt in &packets {
oracle_samples.extend(oracle.push(TrackId::new(1), pkt).unwrap());
}
oracle_samples.extend(oracle.flush(TrackId::new(1)).unwrap());
let mut staged = RtpStreamDepacketiser::new(alloc::vec![RtpStreamTrack::new(
TrackId::new(1),
RtpMediaKind::H264,
dummy_avc(),
90_000,
)]);
let mut staged_samples: Vec<Sample> = Vec::new();
for pkt in &packets {
Stage::feed(&mut staged, pkt, Timestamp::ZERO).unwrap();
while let Some(s) = Stage::poll(&mut staged) {
staged_samples.push(s);
}
}
Stage::finish(&mut staged).unwrap();
while let Some(s) = Stage::poll(&mut staged) {
staged_samples.push(s);
}
assert_eq!(staged_samples.len(), oracle_samples.len());
for (s, o) in staged_samples.iter().zip(oracle_samples.iter()) {
assert_eq!(s.dts, o.dts);
assert_eq!(s.pts, o.pts);
assert_eq!(s.duration, o.duration);
assert_eq!(s.flags.is_sync, o.flags.is_sync);
assert_eq!(s.data, o.data);
}
}
#[test]
fn stage_feed_errors_on_multi_track_construction() {
let mut d = RtpStreamDepacketiser::new(alloc::vec![
RtpStreamTrack::new(TrackId::new(1), RtpMediaKind::H264, dummy_avc(), 90_000),
RtpStreamTrack::new(TrackId::new(2), RtpMediaKind::H264, dummy_avc(), 90_000),
]);
let err = Stage::feed(&mut d, &vpkt(1, 1000, true, &[0x65]), Timestamp::ZERO).unwrap_err();
assert!(matches!(err, Error::InvalidInput(_)));
let err2 = Stage::finish(&mut d).unwrap_err();
assert!(matches!(err2, Error::InvalidInput(_)));
}
fn fu_a_fragment(start: bool, extra_len: usize) -> Vec<u8> {
const NAL_TYPE_FU_A: u8 = 28;
const FU_START: u8 = 0x80;
const ORIG_TYPE_IDR: u8 = 5;
let fu_header = if start {
FU_START | ORIG_TYPE_IDR
} else {
ORIG_TYPE_IDR
};
let mut payload = alloc::vec![NAL_TYPE_FU_A, fu_header];
payload.extend(core::iter::repeat_n(0xABu8, extra_len));
payload
}
#[test]
fn runaway_fu_a_without_end_bit_is_bounded_not_unbounded() {
let mut d = RtpStreamDepacketiser::new(alloc::vec![RtpStreamTrack::new(
TrackId::new(1),
RtpMediaKind::H264,
dummy_avc(),
90_000,
)]);
declare_reorder_depth(&mut d, 0);
const FRAGMENT_FILLER: usize = 2048;
let mut hit_cap = false;
let mut resume_seq: u16 = 0;
for i in 0..4096u16 {
match d.push(
TrackId::new(1),
&vpkt(i, 1000, false, &fu_a_fragment(i == 0, FRAGMENT_FILLER)),
) {
Ok(samples) => assert!(
samples.is_empty(),
"a never-completing AU must not emit a sample"
),
Err(e) => {
assert!(
matches!(e, crate::error::Error::BufferCapExceeded { .. }),
"unexpected error variant: {e:?}"
);
hit_cap = true;
resume_seq = i.wrapping_add(1);
break;
}
}
}
assert!(
hit_cap,
"expected MAX_AU_BUFFER_BYTES to trip well within {} fragments \
(never grow unbounded)",
4096
);
let idr = [0x65u8, 0xAA];
let non = [0x41u8, 0xBB];
assert!(
d.push(TrackId::new(1), &vpkt(resume_seq, 4000, true, &idr))
.unwrap()
.is_empty()
);
let s = d
.push(
TrackId::new(1),
&vpkt(resume_seq.wrapping_add(1), 7000, true, &non),
)
.unwrap();
assert_eq!(s.len(), 1);
assert_eq!(s[0].duration, Some(3000));
assert!(s[0].flags.is_sync);
assert!(
d.poll_loss_event().is_none(),
"no loss event expected: the byte-cap overflow is its own \
recorded corrective action, and the sequence numbers this \
resync uses are contiguous with the ones already accepted"
);
}
#[test]
fn track_specs_use_clock_rate_as_timescale() {
let d = RtpStreamDepacketiser::new(alloc::vec![RtpStreamTrack::new(
TrackId::new(7),
RtpMediaKind::H264,
dummy_avc(),
90_000,
)]);
let specs = d.track_specs();
assert_eq!(specs.len(), 1);
assert_eq!(specs[0].track_id, TrackId::new(7));
assert_eq!(specs[0].timescale, 90_000);
}
fn max_hold_in_aus(
presentation_order_of_wire: &[u32],
depth: usize,
) -> (usize, Vec<(u64, u32)>) {
let id = TrackId::new(1);
let mut d = single_track(RtpMediaKind::H264, avc_with_sps(SPS_25FPS_FIXED), 90_000);
declare_reorder_depth(&mut d, depth);
let mut emitted = Vec::new();
let mut max_hold = 0;
for (n, &frame) in presentation_order_of_wire.iter().enumerate() {
let nal: &[u8] = if n == 0 { &[0x65, 0xAA] } else { &[0x41, 0xBB] };
let ts = 1000 + frame * 3600;
for s in d.push(id, &vpkt(n as u16, ts, true, nal)).unwrap() {
emitted.push((s.pts.unwrap() as u64, s.duration.unwrap()));
}
max_hold = max_hold.max(n + 1 - emitted.len());
}
(max_hold, emitted)
}
#[test]
fn reorder_window_tracks_the_observed_depth_not_a_fixed_size() {
let one_b: Vec<u32> = [0, 2, 1, 4, 3, 6, 5, 8, 7, 10, 9, 12, 11, 14, 13].to_vec();
let (hold, emitted) = max_hold_in_aus(&one_b, 1);
assert!(hold <= 3 + 1, "one B frame held {hold} AUs");
assert!(emitted.iter().skip(3).all(|&(_, dur)| dur == 3600));
let two_b: Vec<u32> = [
0, 3, 1, 2, 6, 4, 5, 9, 7, 8, 12, 10, 11, 15, 13, 14, 18, 16, 17,
]
.to_vec();
let (hold, emitted) = max_hold_in_aus(&two_b, 2);
assert!(hold <= 4 + 2, "two B frames held {hold} AUs");
assert!(emitted.iter().skip(4).all(|&(_, dur)| dur == 3600));
let none: Vec<u32> = (0..12).collect();
assert_eq!(max_hold_in_aus(&none, 0).0, 1);
}
#[test]
fn reorder_window_is_capped() {
let mut wire = alloc::vec![0u32];
for k in (1..40u32).rev() {
wire.push(k);
}
let (hold, _) = max_hold_in_aus(&wire, 0);
assert!(hold <= MAX_REORDER_DEPTH + 1, "held {hold}");
}
}