use alloc::collections::BTreeMap;
use alloc::vec::Vec;
use core::marker::PhantomData;
use broadcast_common::Unpackage;
use mpeg_ps::program_stream::parse_all_packs;
use crate::ac3::{AC3_SAMPLES_PER_SYNCFRAME, Ac3SyncframeInfo};
use crate::annexb::iter_annexb_nals;
use crate::au::AccessUnitSplitter;
use crate::avc_config::{AVCConfigurationBox, AVCDecoderConfigurationRecord};
use crate::error::{Error, Result};
use crate::media::{Media, Track};
use crate::mp4esds::{DecoderConfigDescriptor, ESDescriptor, EsdsBox, SLConfigDescriptor};
use crate::mpeg_legacy::Mpeg2SeqHeader;
use crate::nal::{NalCodec, ParamKind, parameter_set_is_valid};
use crate::nalu_types::{AvcPps, AvcSps};
use crate::pipeline::{CodecConfig, Sample, TrackSpec};
use crate::ts_demux::{
ESDS_VIDEO_ES_ID, MPEG2_PICTURE_START_CODE, OTI_MPEG2_VIDEO_MAIN, STREAM_TYPE_VISUAL,
mpeg2_is_sync,
};
const STREAM_ID_VIDEO_LO: u8 = 0xE0;
const STREAM_ID_VIDEO_HI: u8 = 0xEF;
const STREAM_ID_PRIVATE_1: u8 = 0xBD;
const PRIVATE1_HEADER_LEN: usize = 4;
const PRIVATE1_SUBSTREAM_ID_OFFSET: usize = 0;
const PRIVATE1_NUM_FRAMES_OFFSET: usize = 1;
const PRIVATE1_FIRST_ACCESS_UNIT_POINTER_OFFSET: usize = 2;
const PRIVATE1_NUM_FRAMES_LEN: usize = 1;
const PRIVATE1_FIRST_ACCESS_UNIT_POINTER_LEN: usize = 2;
const _: () = assert!(
PRIVATE1_SUBSTREAM_ID_OFFSET == 0
&& PRIVATE1_NUM_FRAMES_OFFSET == PRIVATE1_SUBSTREAM_ID_OFFSET + 1
&& PRIVATE1_NUM_FRAMES_LEN == 1
&& PRIVATE1_FIRST_ACCESS_UNIT_POINTER_OFFSET
== PRIVATE1_NUM_FRAMES_OFFSET + PRIVATE1_NUM_FRAMES_LEN
&& PRIVATE1_FIRST_ACCESS_UNIT_POINTER_LEN == 2
&& PRIVATE1_HEADER_LEN == 4
);
const SUBSTREAM_ID_DTS_RANGE: core::ops::RangeInclusive<u8> = 0x88..=0x8F;
const SUBSTREAM_ID_LPCM_RANGE: core::ops::RangeInclusive<u8> = 0xA0..=0xA7;
const SUBSTREAM_ID_SUBPICTURE_RANGE: core::ops::RangeInclusive<u8> = 0x20..=0x3F;
const PRIVATE1_SUBSTREAM_ID_NONE: u8 = 0;
const AC3_SYNCWORD: [u8; 2] = [0x0B, 0x77];
const AC3_SYNCWORD_LEN: usize = AC3_SYNCWORD.len();
const SUBSTREAM_ID_AUDIO_LO: u8 = 0x80;
const NAL_LENGTH_SIZE_MINUS_ONE: u8 = 3;
const H264_NAL_SPS: u8 = 7;
const H264_NAL_PPS: u8 = 8;
const H264_NAL_IDR: u8 = 5;
const H264_NAL_TYPE_MASK: u8 = 0x1F;
const VIDEO_TIMESCALE: u32 = 90_000;
const AUDIO_SAMPLE_SIZE_BITS: u16 = 16;
const DEFAULT_FRAME_DURATION: i128 = 3600;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Codec {
Video,
Private,
Ac3,
Skipped,
}
impl Codec {
fn from_stream_id(stream_id: u8) -> Option<Self> {
match stream_id {
STREAM_ID_VIDEO_LO..=STREAM_ID_VIDEO_HI => Some(Codec::Video),
STREAM_ID_PRIVATE_1 => Some(Codec::Private),
_ => None,
}
}
fn classify_private1(
substream_id: u8,
header: &Private1Header,
payload: &[u8],
) -> Option<Self> {
if SUBSTREAM_ID_SUBPICTURE_RANGE.contains(&substream_id)
|| SUBSTREAM_ID_DTS_RANGE.contains(&substream_id)
|| SUBSTREAM_ID_LPCM_RANGE.contains(&substream_id)
{
return None;
}
if substream_id < SUBSTREAM_ID_AUDIO_LO {
return None;
}
let pointer = usize::from(header.first_access_unit_pointer);
let offset = pointer.saturating_sub(1);
let probe = payload.get(offset..)?;
if probe.len() < AC3_SYNCWORD_LEN || !probe.starts_with(&AC3_SYNCWORD) {
return None;
}
Some(Codec::Ac3)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct Private1Header {
substream_id: u8,
number_of_frames: u8,
first_access_unit_pointer: u16,
}
impl Private1Header {
fn parse(payload: &[u8]) -> Option<(Self, &[u8])> {
if payload.len() < PRIVATE1_HEADER_LEN {
return None;
}
let ptr = PRIVATE1_FIRST_ACCESS_UNIT_POINTER_OFFSET;
let header = Private1Header {
substream_id: payload[PRIVATE1_SUBSTREAM_ID_OFFSET],
number_of_frames: payload[PRIVATE1_NUM_FRAMES_OFFSET],
first_access_unit_pointer: u16::from_be_bytes([payload[ptr], payload[ptr + 1]]),
};
Some((header, &payload[PRIVATE1_HEADER_LEN..]))
}
fn is_consistent(&self, payload_len: usize) -> bool {
let body_len = payload_len.saturating_sub(PRIVATE1_HEADER_LEN);
let pointer = usize::from(self.first_access_unit_pointer);
if self.number_of_frames > 0 && pointer == 0 {
return false;
}
if pointer > body_len.saturating_add(1) {
return false;
}
true
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
struct StreamKey {
stream_id: u8,
substream_id: u8,
}
struct ElementaryStream {
codec: Codec,
es_bytes: Vec<u8>,
stamps: Vec<Stamp>,
}
#[derive(Debug, Clone, Copy)]
struct Stamp {
offset: usize,
pts: Option<u64>,
dts: Option<u64>,
}
struct AccessUnit {
pts: Option<u64>,
dts: Option<u64>,
}
#[derive(Debug, Default, Clone)]
pub struct PsDemux<'a> {
_marker: PhantomData<&'a [u8]>,
}
impl<'a> PsDemux<'a> {
pub fn new() -> Self {
Self {
_marker: PhantomData,
}
}
pub fn demux(&mut self, input: &'a [u8]) -> Result<Media> {
let (packs, _trailing) = parse_all_packs(input).map_err(Error::Ps)?;
let mut order: Vec<StreamKey> = Vec::new();
let mut streams: BTreeMap<StreamKey, ElementaryStream> = BTreeMap::new();
for pack in &packs {
for pes in &pack.pes_packets {
let sid = pes.stream_id.0;
let Some(codec) = Codec::from_stream_id(sid) else {
continue;
};
let (substream_id, header, payload): (u8, Option<Private1Header>, &[u8]) =
match codec {
Codec::Video => (PRIVATE1_SUBSTREAM_ID_NONE, None, pes.payload),
Codec::Private => {
let Some((header, body)) = Private1Header::parse(pes.payload) else {
continue;
};
(header.substream_id, Some(header), body)
}
Codec::Ac3 | Codec::Skipped => continue,
};
if payload.is_empty() {
continue;
}
let (pts, dts) = pes
.header
.as_ref()
.map(|h| (h.pts.map(|p| p.0), h.dts.map(|d| d.0)))
.unwrap_or((None, None));
let key = StreamKey {
stream_id: sid,
substream_id,
};
let es = streams.entry(key).or_insert_with(|| {
order.push(key);
ElementaryStream {
codec,
es_bytes: Vec::new(),
stamps: Vec::new(),
}
});
if es.codec == Codec::Private {
if let Some(c) =
header.and_then(|h| Codec::classify_private1(substream_id, &h, payload))
{
es.codec = c;
} else {
if !header.is_some_and(|h| h.is_consistent(pes.payload.len())) {
continue;
}
}
}
if es.codec == Codec::Private || es.codec == Codec::Skipped {
continue;
}
let payload = match header {
Some(header)
if es.es_bytes.is_empty() && header.is_consistent(pes.payload.len()) =>
{
let prefix = usize::from(header.first_access_unit_pointer)
.saturating_sub(1)
.min(payload.len());
&payload[prefix..]
}
_ => payload,
};
if payload.is_empty() {
continue;
}
let offset = es.es_bytes.len();
if pts.is_some() || dts.is_some() {
es.stamps.push(Stamp { offset, pts, dts });
}
es.es_bytes.extend_from_slice(payload);
}
}
let mut tracks: Vec<Track> = Vec::new();
let mut track_id: u32 = 1;
for key in &order {
let es = &streams[key];
let built = match es.codec {
Codec::Video if Mpeg2SeqHeader::find(&es.es_bytes).is_ok() => {
build_mpeg2_track(es, track_id)
}
Codec::Video => build_h264_track(es, track_id)?,
Codec::Ac3 => build_ac3_track(es, track_id),
Codec::Private | Codec::Skipped => None,
};
if let Some(track) = built {
tracks.push(track);
track_id += 1;
}
}
Ok(Media::new(tracks, VIDEO_TIMESCALE))
}
}
impl<'a> Unpackage for PsDemux<'a> {
type Input = &'a [u8];
type Media = Media;
type Error = Error;
fn unpackage(&mut self, input: &'a [u8]) -> Result<Media> {
self.demux(input)
}
}
type SplitUnits = Vec<(usize, usize)>;
type MaybeTrack = Result<Option<Track>>;
fn split_h264_access_units(data: &[u8]) -> Result<Option<SplitUnits>> {
let mut splitter = AccessUnitSplitter::new(NalCodec::Avc);
let Some(mut cursor) = crate::au::first_nal_start(data) else {
return Ok(None);
};
splitter.push(data)?;
splitter.finish();
let mut ranges = Vec::new();
while let Some(unit) = splitter.pop() {
let end = cursor.checked_add(unit.len()).ok_or(Error::InvalidInput(
"ps: H.264 access-unit offset overflowed",
))?;
if data.get(cursor..end) != Some(unit.as_slice()) {
return Err(Error::InvalidInput(
"ps: H.264 access-unit splitter returned bytes that are not a slice of its input",
));
}
ranges.push((cursor, end));
cursor = end;
}
Ok(if ranges.is_empty() {
None
} else {
Some(ranges)
})
}
fn split_mpeg2_pictures(data: &[u8]) -> Vec<(usize, usize)> {
let codes = crate::annexb::start_code_positions(data);
let mut starts: Vec<usize> = Vec::new();
for &pos in &codes {
if pos + 3 < data.len() && data[pos + 3] == MPEG2_PICTURE_START_CODE {
starts.push(pos);
}
}
if starts.is_empty() {
return if data.is_empty() {
Vec::new()
} else {
alloc::vec![(0, data.len())]
};
}
let mut ranges: Vec<(usize, usize)> = Vec::with_capacity(starts.len());
let n = starts.len();
for i in 0..n {
let start = if i == 0 { 0 } else { starts[i] };
let end = if i + 1 < n { starts[i + 1] } else { data.len() };
ranges.push((start, end));
}
ranges
}
fn unwrap_ts(prev_unwrapped: i128, prev_raw: u64, raw: u64) -> i128 {
broadcast_common::clock33::unwrap_delta(prev_unwrapped, prev_raw, raw)
}
fn assign_stamps(ranges: &[(usize, usize)], stamps: &[Stamp]) -> Vec<(Option<u64>, Option<u64>)> {
let mut out = alloc::vec![(None, None); ranges.len()];
let mut si = 0usize;
let (mut prev_pts_raw, mut prev_pts_uw): (Option<u64>, i128) = (None, 0);
let (mut prev_dts_raw, mut prev_dts_uw): (Option<u64>, i128) = (None, 0);
for (ai, &(start, _end)) in ranges.iter().enumerate() {
while si < stamps.len() && stamps[si].offset <= start {
let s = stamps[si];
let pts_uw = s.pts.map(|p| match prev_pts_raw {
Some(pr) => {
let uw = unwrap_ts(prev_pts_uw, pr, p);
prev_pts_uw = uw;
prev_pts_raw = Some(p);
uw
}
None => {
prev_pts_uw = p as i128;
prev_pts_raw = Some(p);
p as i128
}
});
let dts_uw = s.dts.map(|d| match prev_dts_raw {
Some(pr) => {
let uw = unwrap_ts(prev_dts_uw, pr, d);
prev_dts_uw = uw;
prev_dts_raw = Some(d);
uw
}
None => {
prev_dts_uw = d as i128;
prev_dts_raw = Some(d);
d as i128
}
});
if out[ai].0.is_none() && out[ai].1.is_none() {
out[ai] = (pts_uw.map(|v| v as u64), dts_uw.map(|v| v as u64));
}
si += 1;
}
}
out
}
fn build_h264_track(es: &ElementaryStream, track_id: u32) -> MaybeTrack {
let Some(ranges) = split_h264_access_units(&es.es_bytes)? else {
return Ok(None);
};
let stamped = assign_stamps(&ranges, &es.stamps);
let mut sps: Option<Vec<u8>> = None;
let mut pps: Option<Vec<u8>> = None;
let units: Vec<AccessUnit> = ranges
.iter()
.enumerate()
.map(|(i, &(start, end))| {
for nal in iter_annexb_nals(&es.es_bytes[start..end]) {
match nal[0] & H264_NAL_TYPE_MASK {
H264_NAL_SPS
if sps.is_none()
&& parameter_set_is_valid(NalCodec::Avc, ParamKind::Sps, nal) =>
{
sps = Some(nal.to_vec())
}
H264_NAL_PPS
if pps.is_none()
&& parameter_set_is_valid(NalCodec::Avc, ParamKind::Pps, nal) =>
{
pps = Some(nal.to_vec())
}
_ => {}
}
}
AccessUnit {
pts: stamped[i].0,
dts: stamped[i].1,
}
})
.collect();
let (Some(sps), Some(pps)) = (sps, pps) else {
return Ok(None);
};
if sps.len() < 4 {
return Ok(None);
}
let record = AVCDecoderConfigurationRecord {
configuration_version: 1,
profile_indication: sps[1],
profile_compatibility: sps[2],
level_indication: sps[3],
length_size_minus_one: NAL_LENGTH_SIZE_MINUS_ONE,
sps: alloc::vec![AvcSps(sps)],
pps: alloc::vec![AvcPps(pps)],
chroma_format: None,
bit_depth_luma_minus8: None,
bit_depth_chroma_minus8: None,
sps_ext: alloc::vec![],
};
let config = AVCConfigurationBox::new(record);
let dts = interpolate_dts(&units);
let pts = interpolate_pts(&units, &dts);
let mut order: Vec<usize> = (0..units.len()).collect();
order.sort_by_key(|&i| dts[i]);
let samples: Vec<Sample> = order
.iter()
.enumerate()
.map(|(pos, &i)| {
let dur = frame_duration(&order, &dts, pos);
let (start, end) = ranges[i];
let au = &es.es_bytes[start..end];
let is_idr = au_is_idr(au);
Sample::from_annexb(
au,
Some(to_ticks(dts[i])),
Some(to_ticks(pts[i])),
Some(dur),
is_idr,
)
})
.collect();
let anchor = order.first().map(|&i| dts[i].max(0) as u64).unwrap_or(0);
Ok(Some(Track::new_at(
TrackSpec::new(
track_id,
VIDEO_TIMESCALE,
CodecConfig::Avc {
config,
width: 0,
height: 0,
},
),
samples,
anchor,
)))
}
fn build_mpeg2_track(es: &ElementaryStream, track_id: u32) -> Option<Track> {
let seq = Mpeg2SeqHeader::find(&es.es_bytes).ok()?;
let ranges = split_mpeg2_pictures(&es.es_bytes);
if ranges.is_empty() {
return None;
}
let stamped = assign_stamps(&ranges, &es.stamps);
let units: Vec<AccessUnit> = stamped
.iter()
.map(|&(pts, dts)| AccessUnit { pts, dts })
.collect();
let esds = EsdsBox::new(ESDescriptor::new(
ESDS_VIDEO_ES_ID,
0,
Some(DecoderConfigDescriptor::new(
OTI_MPEG2_VIDEO_MAIN,
STREAM_TYPE_VISUAL,
false,
0,
0,
0,
None,
)),
Some(SLConfigDescriptor::predefined_two()),
));
let dts = interpolate_dts(&units);
let pts = interpolate_pts(&units, &dts);
let mut order: Vec<usize> = (0..units.len()).collect();
order.sort_by_key(|&i| dts[i]);
let samples: Vec<Sample> = order
.iter()
.enumerate()
.map(|(pos, &i)| {
let dur = frame_duration(&order, &dts, pos);
let (start, end) = ranges[i];
let au = &es.es_bytes[start..end];
let is_sync = mpeg2_is_sync(au);
Sample::new(
au.to_vec(),
Some(to_ticks(dts[i])),
Some(to_ticks(pts[i])),
Some(dur),
is_sync,
)
})
.collect();
let anchor = order.first().map(|&i| dts[i].max(0) as u64).unwrap_or(0);
Some(Track::new_at(
TrackSpec::new(
track_id,
VIDEO_TIMESCALE,
CodecConfig::Mpeg2Video {
esds,
width: seq.width,
height: seq.height,
},
),
samples,
anchor,
))
}
fn to_ticks(uw: i128) -> i64 {
uw.clamp(i64::MIN as i128, i64::MAX as i128) as i64
}
fn au_is_idr(au: &[u8]) -> bool {
iter_annexb_nals(au).any(|nal| (nal[0] & H264_NAL_TYPE_MASK) == H264_NAL_IDR)
}
fn stamped_frame_duration(units: &[AccessUnit]) -> i128 {
let stamped: Vec<(usize, i128)> = units
.iter()
.enumerate()
.filter_map(|(i, u)| u.dts.map(|d| (i, d as i128)))
.collect();
if stamped.len() < 2 {
return DEFAULT_FRAME_DURATION;
}
let (i0, d0) = stamped[0];
let (i1, d1) = *stamped.last().unwrap();
let span_idx = (i1 - i0) as i128;
let span_dts = d1 - d0;
if span_idx > 0 && span_dts > 0 {
(span_dts / span_idx).max(1)
} else {
DEFAULT_FRAME_DURATION
}
}
fn interpolate_dts(units: &[AccessUnit]) -> Vec<i128> {
let dur = stamped_frame_duration(units);
let n = units.len();
let mut dts = alloc::vec![0i128; n];
let anchor = units
.iter()
.enumerate()
.find_map(|(i, u)| u.dts.map(|d| (i, d as i128)));
let (anchor_idx, anchor_dts) = anchor.unwrap_or((0, 0));
for (i, slot) in dts.iter_mut().enumerate() {
*slot = match units[i].dts {
Some(d) => d as i128,
None => anchor_dts + (i as i128 - anchor_idx as i128) * dur,
};
}
dts
}
fn interpolate_pts(units: &[AccessUnit], dts: &[i128]) -> Vec<i128> {
units
.iter()
.enumerate()
.map(|(i, u)| u.pts.map(|p| p as i128).unwrap_or(dts[i]))
.collect()
}
fn frame_duration(order: &[usize], dts: &[i128], pos: usize) -> u32 {
let n = order.len();
let dur = if pos + 1 < n {
(dts[order[pos + 1]] - dts[order[pos]]).max(0)
} else if pos > 0 {
(dts[order[pos]] - dts[order[pos - 1]]).max(0)
} else {
DEFAULT_FRAME_DURATION
};
dur as u32
}
fn split_ac3_frames(data: &[u8]) -> Vec<(usize, usize)> {
crate::ac3::split_ac3_syncframe_ranges(data, true)
}
fn build_ac3_track(es: &ElementaryStream, track_id: u32) -> Option<Track> {
let info = Ac3SyncframeInfo::from_es(&es.es_bytes).ok()?;
let sample_rate = info.sample_rate;
let channel_count = info.channel_count() as u16;
let config = info.into_dac3();
let frames = split_ac3_frames(&es.es_bytes);
if frames.is_empty() {
return None;
}
let rescale_90k = |t90: u64| -> i64 {
((t90 as u128 * sample_rate as u128) / VIDEO_TIMESCALE as u128) as i64
};
let stamped = assign_stamps(&frames, &es.stamps);
let mut samples: Vec<Sample> = Vec::with_capacity(frames.len());
let mut next_dts: Option<i64> = None;
let mut anchor_wire90: i128 = 0;
let mut ticks_since_anchor: i64 = 0;
for (i, &(s, e)) in frames.iter().enumerate() {
let stamp90 = stamped[i].1.or(stamped[i].0);
let dts = match (stamp90, next_dts) {
(Some(t90), Some(running)) => {
let expected_wire90 = anchor_wire90
+ (ticks_since_anchor as i128 * VIDEO_TIMESCALE as i128)
/ sample_rate.max(1) as i128;
let drift = (t90 as i128 - expected_wire90).abs();
if drift > crate::ts_demux::audio_discontinuity_threshold_90k(sample_rate) {
anchor_wire90 = t90 as i128;
ticks_since_anchor = 0;
rescale_90k(t90)
} else {
running
}
}
(Some(t90), None) => {
anchor_wire90 = t90 as i128;
ticks_since_anchor = 0;
rescale_90k(t90)
}
(None, Some(running)) => running,
(None, None) => {
samples.push(Sample::from_raw(
es.es_bytes[s..e].to_vec(),
None,
None,
Some(AC3_SAMPLES_PER_SYNCFRAME),
));
continue;
}
};
next_dts = Some(dts + AC3_SAMPLES_PER_SYNCFRAME as i64);
ticks_since_anchor += AC3_SAMPLES_PER_SYNCFRAME as i64;
samples.push(Sample::from_raw(
es.es_bytes[s..e].to_vec(),
Some(dts),
Some(dts),
Some(AC3_SAMPLES_PER_SYNCFRAME),
));
}
let anchor = samples
.first()
.and_then(|s| s.dts)
.map(|d| d.max(0) as u64)
.unwrap_or(0);
Some(Track::new_at(
TrackSpec::new(
track_id,
sample_rate,
CodecConfig::Ac3 {
config,
channel_count,
sample_rate,
sample_size: AUDIO_SAMPLE_SIZE_BITS,
},
),
samples,
anchor,
))
}
#[cfg(test)]
mod wrap_tests {
use super::*;
const WRAP: u64 = broadcast_common::clock33::WRAP_33BIT;
#[test]
fn assign_stamps_unrolls_across_the_33bit_wrap() {
let ranges = [(0usize, 10usize), (10, 20), (20, 30)];
let stamps = [
Stamp {
offset: 0,
pts: Some(WRAP - 90_000),
dts: Some(WRAP - 90_000),
},
Stamp {
offset: 10,
pts: Some(90_000),
dts: Some(90_000),
},
Stamp {
offset: 20,
pts: Some(WRAP - 45_000),
dts: Some(WRAP - 45_000),
},
];
let out = assign_stamps(&ranges, &stamps);
assert_eq!(out[0].0, Some(WRAP - 90_000));
assert_eq!(out[1].0, Some(WRAP + 90_000));
assert_eq!(out[2].0, Some(WRAP - 45_000));
assert_eq!(out[1].1, Some(WRAP + 90_000));
}
}