use alloc::format;
use alloc::string::{String, ToString};
use alloc::vec;
use alloc::vec::Vec;
use core::marker::PhantomData;
use broadcast_common::{Package, Parse, Unpackage};
use crate::ac3::{Ac3SpecificBox, Ec3SpecificBox};
use broadcast_hls::{ByteRange, DecimalSeconds, MapTag, MediaPlaylist, MediaSegment};
use crate::ac4::Ac4SpecificBox;
use crate::box_types::{BOX_HEADER_MIN_SIZE, parse_box};
use crate::display_track::{SpsTracker, TrackCodec};
use crate::dts::DtsSpecificBox;
use crate::error::{Error, Result};
use crate::flac::FlacSpecificBox;
use crate::init_segment::{
MovieBox, OpaqueBox, SampleDescriptionBox, SampleEntryVariant, StblChild, TrackBox,
sampling_rate_override,
};
use crate::ir::{
AspectRatioIdc, CodecConfig, ColourDescription, DisplayGeometry, DisplayInfo,
FragmentTrackData, InputDegradation, Sample, SubtitleFormat, SubtitleSampleEntry, TrackSpec,
VideoFormat, VideoSignalType,
};
use crate::movie_fragment::{MovieFragmentBox, SAMPLE_FLAG_IS_NON_SYNC};
use crate::mp4esds::EsdsBox;
use crate::mpeg_legacy::{Mpeg2SeqHeader, MpegAudioFrameHeader, MpegAudioLayer};
use crate::mpegh::{MHAC_FOURCC, MHADecoderConfigurationRecord};
use crate::opus::OpusSpecificBox;
use crate::pipeline::{build_init_segment, build_media_segment};
use crate::visual_ext::{ColourInformationBox, PixelAspectRatioBox};
use alloc::sync::Arc;
pub use crate::ir::{Media, PcrSample, SkippedTrack, Track, TrackEncryption};
const DEFAULT_MOVIE_TIMESCALE: u32 = 1000;
const OTI_MPEG2_AUDIO: u8 = 0x69;
const OTI_MPEG1_AUDIO: u8 = 0x6B;
#[derive(Debug, Default, Clone)]
pub struct Fmp4Demux<'a> {
_marker: PhantomData<&'a [u8]>,
}
impl Fmp4Demux<'_> {
pub fn new() -> Self {
Self {
_marker: PhantomData,
}
}
}
struct TrackBuilder {
spec: TrackSpec,
samples: Vec<Sample>,
start_decode_time: Option<u64>,
next_dts: i64,
in_band: bool,
entry_extra_boxes: Vec<crate::sample_entries::OpaqueBox>,
}
impl<'a> Unpackage for Fmp4Demux<'a> {
type Input = &'a [u8];
type Media = Media;
type Error = Error;
fn unpackage(&mut self, input: &'a [u8]) -> Result<Media> {
let moov_bytes =
find_top_box(input, b"moov").ok_or(Error::UnexpectedBox { expected: "moov" })?;
let moov = MovieBox::parse(moov_bytes)?;
let movie_timescale = moov.mvhd.timescale;
let trex_defaults: &[crate::init_segment::TrackExtendsBox] =
moov.mvex.as_ref().map_or(&[], |m| m.trex.as_slice());
let mut builders: Vec<TrackBuilder> = Vec::with_capacity(moov.tracks.len());
let mut skipped: Vec<SkippedTrack> = Vec::new();
for trak in &moov.tracks {
match track_spec_from_trak(trak) {
Ok(spec) => {
let (in_band, entry_extra_boxes) = track_entry_inband_and_extras(trak);
builders.push(TrackBuilder {
spec,
samples: Vec::new(),
start_decode_time: None,
next_dts: 0,
in_band,
entry_extra_boxes,
})
}
Err(err) => skipped.push(skipped_track(err, trak.tkhd.track_id)),
}
}
let mut offset = 0usize;
let mdats = crate::frag_offsets::mdat_ranges(input);
let mut pending_moofs: Vec<(usize, MovieFragmentBox)> = Vec::new();
let mut zero_width_budget = crate::movie_fragment::MAX_ZERO_WIDTH_TRUN_SAMPLES;
let mut degraded: Vec<InputDegradation> = Vec::new();
while offset + BOX_HEADER_MIN_SIZE <= input.len() {
let (bx, consumed) = match parse_box(&input[offset..]) {
Ok(parsed) => parsed,
Err(Error::BufferTooShort { .. })
| Err(Error::UuidBufferTooShort { .. })
| Err(Error::LargesizeBufferTooShort { .. }) => break,
Err(e) => return Err(e),
};
let ty = bx.header.box_type.0;
if &ty == b"moof" {
match MovieFragmentBox::parse_body_budgeted(bx.body, &mut zero_width_budget) {
Ok(mut moof) => {
apply_trex_defaults(&mut moof, trex_defaults);
pending_moofs.push((offset, moof));
}
Err(_) => degraded.push(InputDegradation::MalformedUnit {
position: offset as u64,
}),
}
} else if &ty == b"mdat" {
for (moof_off, moof) in pending_moofs.drain(..) {
for _ in 0..absorb_fragment(input, &mdats, moof_off, &moof, &mut builders) {
degraded.push(InputDegradation::MalformedUnit {
position: moof_off as u64,
});
}
}
zero_width_budget = crate::movie_fragment::MAX_ZERO_WIDTH_TRUN_SAMPLES;
}
if consumed == 0 {
break;
}
offset += consumed;
}
let tracks = builders
.into_iter()
.map(|mut b| {
refine_legacy_config(&mut b.spec.config, &b.samples);
normalise_track_nal_lengths(&mut b.spec, &mut b.samples)?;
if b.in_band {
tag_fmp4_inband_display(&mut b);
}
if matches!(b.spec.config, CodecConfig::Hevc { .. }) {
b.spec.hdr10plus = b.samples.first().is_some_and(|s| {
crate::hdr10plus::Hdr10PlusMetadata::sample_carries_hdr10plus(&s.data)
});
}
Ok(Track {
spec: b.spec,
samples: b.samples,
start_decode_time: b.start_decode_time.unwrap_or(0),
encryption: None,
})
})
.collect::<Result<Vec<_>>>()?;
Ok(Media {
tracks,
movie_timescale,
pcr: Vec::new(),
skipped,
degraded,
degraded_at: Vec::new(),
})
}
}
fn apply_trex_defaults(moof: &mut MovieFragmentBox, trex: &[crate::init_segment::TrackExtendsBox]) {
for traf in &mut moof.traf {
let Some(defaults) = trex.iter().find(|t| t.track_id == traf.tfhd.track_id) else {
continue;
};
let tfhd = &mut traf.tfhd;
tfhd.default_sample_duration
.get_or_insert(defaults.default_sample_duration);
tfhd.default_sample_flags
.get_or_insert(defaults.default_sample_flags);
if defaults.default_sample_size != 0 {
tfhd.default_sample_size
.get_or_insert(defaults.default_sample_size);
}
}
}
fn absorb_fragment(
file: &[u8],
mdats: &[crate::frag_offsets::MdatRange],
moof_off: usize,
moof: &MovieFragmentBox,
builders: &mut [TrackBuilder],
) -> usize {
let mut bad = 0usize;
for traf in &moof.traf {
if absorb_traf(file, mdats, moof_off, moof, traf, builders).is_err() {
bad += 1;
}
}
bad
}
fn absorb_traf(
file: &[u8],
mdats: &[crate::frag_offsets::MdatRange],
moof_off: usize,
moof: &MovieFragmentBox,
traf: &crate::movie_fragment::TrackFragmentBox,
builders: &mut [TrackBuilder],
) -> Result<()> {
{
let tfhd = &traf.tfhd;
let Some(builder) = builders
.iter_mut()
.find(|b| b.spec.track_id == tfhd.track_id)
else {
return Ok(());
};
let mut start_decode_time = builder.start_decode_time;
let mut next_dts = builder.next_dts;
if let Some(tfdt) = &traf.tfdt {
let base = tfdt.base_media_decode_time();
if start_decode_time.is_none() {
start_decode_time = Some(base);
}
next_dts = crate::frag_offsets::tfdt_as_i64(base)?;
}
let mut samples = Vec::new();
for r in crate::frag_offsets::sample_ranges_in(file, mdats, moof_off, moof, tfhd.track_id)?
{
let is_sync = r.flags & SAMPLE_FLAG_IS_NON_SYNC == 0;
let dts = next_dts;
let pts = crate::frag_offsets::add_offset(dts, r.composition_offset)?;
samples.push(Sample {
data: file[r.start..r.end].to_vec().into(),
dts: Some(dts),
pts: Some(pts),
duration: Some(r.duration),
flags: crate::ir::SampleFlags::new(is_sync),
provenance: None,
display: None,
});
next_dts = crate::frag_offsets::add_duration(dts, r.duration)?;
}
builder.start_decode_time = start_decode_time;
builder.next_dts = next_dts;
builder.samples.extend(samples);
}
Ok(())
}
#[derive(Debug, Clone)]
pub struct CmafMux {
pub sequence_number: u32,
}
impl Default for CmafMux {
fn default() -> Self {
Self { sequence_number: 1 }
}
}
impl CmafMux {
pub fn new(sequence_number: u32) -> Self {
Self { sequence_number }
}
}
impl Package for CmafMux {
type Media = Media;
type Output = Vec<u8>;
type Error = Error;
fn package(&mut self, media: &Media) -> Result<Vec<u8>> {
if media.tracks.is_empty() {
return Err(Error::InvalidInput("cannot package a Media with no tracks"));
}
let specs: Vec<TrackSpec> = media.tracks.iter().map(|t| t.spec.clone()).collect();
let movie_timescale = if media.movie_timescale == 0 {
DEFAULT_MOVIE_TIMESCALE
} else {
media.movie_timescale
};
let mut out = build_init_segment(&specs, movie_timescale)?;
let fragments: Vec<FragmentTrackData<'_>> = media
.tracks
.iter()
.map(|t| FragmentTrackData {
track_id: t.spec.track_id,
base_media_decode_time: t.start_decode_time,
samples: &t.samples,
})
.collect();
let media_seg = build_media_segment(self.sequence_number, &fragments)?;
out.extend_from_slice(&media_seg);
Ok(out)
}
}
#[derive(Debug, Clone, Copy)]
pub(crate) struct TimelineOrigin {
ticks: i64,
timescale: u32,
}
impl TimelineOrigin {
pub(crate) fn of<'t>(tracks: impl IntoIterator<Item = &'t Track>) -> Self {
let mut best: Option<(i64, u32)> = None;
for t in tracks {
let Some(d) = t.samples.iter().find_map(|s| s.dts) else {
continue;
};
let ts = t.spec.timescale.max(1);
let earlier = match best {
None => true,
Some((bd, bts)) => (d as i128) * (bts as i128) < (bd as i128) * (ts as i128),
};
if earlier {
best = Some((d, ts));
}
}
let (ticks, timescale) = best.unwrap_or((0, 1));
Self { ticks, timescale }
}
pub(crate) fn in_timescale(&self, timescale: u32) -> i64 {
let n = self.ticks as i128 * timescale.max(1) as i128;
let v = n.div_euclid(self.timescale.max(1) as i128);
v.clamp(i64::MIN as i128, i64::MAX as i128) as i64
}
}
pub(crate) fn relative_decode_times(track: &Track, origin: &TimelineOrigin) -> Vec<i64> {
let base = origin.in_timescale(track.spec.timescale);
let mut out = Vec::with_capacity(track.samples.len());
let mut next: i64 = 0;
for s in &track.samples {
let t = match s.dts {
Some(d) => d.saturating_sub(base),
None => next,
};
out.push(t);
next = t.saturating_add(i64::from(s.duration.unwrap_or(0)));
}
out
}
pub(crate) fn presentation_span_secs(tracks: &[&Track]) -> f64 {
let origin = TimelineOrigin::of(tracks.iter().copied());
let mut span: Option<(f64, f64)> = None;
for t in tracks {
let ts = f64::from(t.spec.timescale.max(1));
for (s, dts) in t.samples.iter().zip(relative_decode_times(t, &origin)) {
let pts = (dts + i64::from(s.composition_offset())) as f64 / ts;
let end = pts + f64::from(s.duration.unwrap_or(0)) / ts;
span = Some(match span {
None => (pts, end),
Some((lo, hi)) => (lo.min(pts), hi.max(end)),
});
}
}
span.map_or(0.0, |(lo, hi)| (hi - lo).max(0.0))
}
fn ceil_secs(secs: f64) -> u32 {
let whole = secs as u32; if (whole as f64) < secs {
whole.saturating_add(1)
} else {
whole
}
}
#[derive(Debug, Clone)]
pub struct HlsPackager {
pub version: u8,
pub media_sequence: u64,
pub uri_prefix: String,
}
impl Default for HlsPackager {
fn default() -> Self {
Self {
version: 7,
media_sequence: 0,
uri_prefix: String::from("seg"),
}
}
}
impl Package for HlsPackager {
type Media = Media;
type Output = String;
type Error = Error;
fn package(&mut self, media: &Media) -> Result<String> {
if media.tracks.is_empty() {
return Err(Error::InvalidInput("cannot package a Media with no tracks"));
}
let kept: Vec<&Track> = media
.tracks
.iter()
.filter(|t| t.samples.iter().any(|s| s.duration.is_some()))
.collect();
let specs: Vec<TrackSpec> = kept.iter().map(|t| t.spec.clone()).collect();
let movie_timescale = if media.movie_timescale == 0 {
DEFAULT_MOVIE_TIMESCALE
} else {
media.movie_timescale
};
let init_len = if specs.is_empty() {
0
} else {
build_init_segment(&specs, movie_timescale)?.len() as u64
};
let Some(first) = kept.first() else {
return Err(Error::InvalidInput(
"cannot package a Media whose every track is timestamp-less \
(section-carried): an HLS media playlist needs at least one \
segment with a real EXTINF duration",
));
};
let duration = presentation_span_secs(&kept);
let target_secs = ceil_secs(duration);
let uri = format!("{}{}.m4s", self.uri_prefix, first.spec.track_id);
let segments = vec![MediaSegment {
map: Some(MapTag {
uri: uri.clone(),
byte_range: Some(ByteRange {
length: init_len,
offset: Some(0),
}),
extra_attrs: Vec::new(),
}),
uri,
duration: DecimalSeconds::new(duration).expect("presentation span is finite, >= 0"),
discontinuous: false,
parts: vec![],
..Default::default()
}];
let playlist = MediaPlaylist {
version: self.version,
target_duration: target_secs,
media_sequence: self.media_sequence,
discontinuity_sequence: 0,
segments,
endlist: true,
extra_tags: vec![],
low_latency: None,
iframes_only: false,
open_segment: None,
..Default::default()
};
Ok(playlist.to_m3u8()?)
}
}
pub(crate) use crate::box_types::find_top_box;
pub(crate) fn track_spec_from_trak(trak: &TrackBox) -> Result<TrackSpec> {
let track_id = trak.tkhd.track_id;
let mdia = trak
.mdia
.as_ref()
.ok_or(Error::UnexpectedBox { expected: "mdia" })?;
let timescale = mdia
.mdhd
.as_ref()
.ok_or(Error::UnexpectedBox { expected: "mdhd" })?
.timescale;
let minf = mdia
.minf
.as_ref()
.ok_or(Error::UnexpectedBox { expected: "minf" })?;
let stbl = minf
.stbl
.as_ref()
.ok_or(Error::UnexpectedBox { expected: "stbl" })?;
let stsd: SampleDescriptionBox = stbl
.children
.iter()
.find_map(|c| match c {
StblChild::Stsd(s) => Some(Ok(s.clone())),
StblChild::Opaque(bytes) if bytes.len() >= 8 && &bytes[4..8] == b"stsd" => {
Some(SampleDescriptionBox::parse(bytes.as_slice()))
}
_ => None,
})
.ok_or(Error::UnexpectedBox { expected: "stsd" })??;
let entry = stsd.entries.first().ok_or(Error::UnexpectedBox {
expected: "stsd entry",
})?;
let config = codec_config_from_entry(entry)?;
let spec = TrackSpec::new(track_id, timescale, config);
let spec = match entry {
SampleEntryVariant::Avc1(e) => overlay_sample_entry_display(spec, &e.extra_boxes),
SampleEntryVariant::Hevc1(e) => overlay_sample_entry_display(spec, &e.extra_boxes),
_ => spec,
};
Ok(spec)
}
pub(crate) fn overlay_sample_entry_geometry(
display: Option<DisplayGeometry>,
config: &CodecConfig,
extra_boxes: &[crate::sample_entries::OpaqueBox],
) -> Option<DisplayGeometry> {
let pasp = extra_boxes
.iter()
.filter(|b| b.box_type == *b"pasp")
.find_map(|b| PixelAspectRatioBox::parse(&b.data).ok());
let colour = extra_boxes
.iter()
.filter(|b| b.box_type == *b"colr")
.find_map(|b| {
let c = ColourInformationBox::parse(&b.data).ok()?;
if c.colour_type != *b"nclx" {
return None;
}
let n = c.nclx?;
let colour = ColourDescription::from_wire(
u8::try_from(n.colour_primaries).ok()?,
u8::try_from(n.transfer_characteristics).ok()?,
u8::try_from(n.matrix_coefficients).ok()?,
);
Some((colour, n.full_range_flag))
});
let pasp_ratio = pasp.and_then(|p| {
if p.h_spacing == 0 || p.v_spacing == 0 {
return None;
}
let g = gcd_u32(p.h_spacing, p.v_spacing);
let (rw, rh) = (
u16::try_from(p.h_spacing / g).ok()?,
u16::try_from(p.v_spacing / g).ok()?,
);
Some(if rw == rh {
AspectRatioIdc::from_wire(1, None)
} else {
AspectRatioIdc::from_wire(255, Some((rw, rh)))
})
});
if pasp_ratio.is_none() && colour.is_none() {
return display;
}
let (visual_w, visual_h) = match config {
CodecConfig::Avc { width, height, .. } | CodecConfig::Hevc { width, height, .. } => {
(u32::from(*width), u32::from(*height))
}
_ => (0, 0),
};
let signalled_format = display
.as_ref()
.and_then(|g| g.video_signal.map(|v| v.video_format));
let mut display = display.unwrap_or_else(|| DisplayGeometry::new(visual_w, visual_h, None));
if let Some(ratio) = pasp_ratio
&& display.sample_aspect_ratio().is_none()
{
display.aspect_ratio = Some(ratio);
}
if let Some((colour, full_range)) = colour {
display.video_signal = Some(VideoSignalType::new(
signalled_format.unwrap_or(VideoFormat::Unspecified),
full_range,
Some(colour),
));
}
Some(display)
}
fn overlay_sample_entry_display(
mut spec: TrackSpec,
extra_boxes: &[crate::sample_entries::OpaqueBox],
) -> TrackSpec {
let config = spec.config.clone();
spec.display = overlay_sample_entry_geometry(spec.display, &config, extra_boxes);
if let Some(hdr) = crate::hdr::HdrStaticMetadata::from_sample_entry_boxes(
extra_boxes.iter().map(|b| (&b.box_type, b.data.as_slice())),
) {
spec.hdr = Some(hdr);
}
if let Some(dovi) = crate::dovi::DoviDecoderConfigurationRecord::from_sample_entry_boxes(
extra_boxes.iter().map(|b| (&b.box_type, b.data.as_slice())),
) {
spec.dovi = Some(dovi);
}
spec
}
fn track_entry_inband_and_extras(trak: &TrackBox) -> (bool, Vec<crate::sample_entries::OpaqueBox>) {
let Some(mdia) = trak.mdia.as_ref() else {
return (false, Vec::new());
};
let Some(minf) = mdia.minf.as_ref() else {
return (false, Vec::new());
};
let Some(stbl) = minf.stbl.as_ref() else {
return (false, Vec::new());
};
let stsd: SampleDescriptionBox = match stbl.children.iter().find_map(|c| match c {
StblChild::Stsd(s) => Some(Ok(s.clone())),
StblChild::Opaque(bytes) if bytes.len() >= 8 && &bytes[4..8] == b"stsd" => {
Some(SampleDescriptionBox::parse(bytes.as_slice()))
}
_ => None,
}) {
Some(Ok(s)) => s,
_ => return (false, Vec::new()),
};
let Some(entry) = stsd.entries.first() else {
return (false, Vec::new());
};
match entry {
SampleEntryVariant::Avc1(e) => (
e.codec_type == *b"avc3" || e.codec_type == *b"avc4",
e.extra_boxes.clone(),
),
SampleEntryVariant::Hevc1(e) => (
e.codec_type == *b"hev1" || e.codec_type == crate::dovi::DVHE_FOURCC,
e.extra_boxes.clone(),
),
_ => (false, Vec::new()),
}
}
fn tag_fmp4_inband_display(b: &mut TrackBuilder) {
let config = b.spec.config.clone();
let codec = match &config {
CodecConfig::Avc { .. } => TrackCodec::Avc,
CodecConfig::Hevc { .. } => TrackCodec::Hevc,
_ => return,
};
let mut tracker = SpsTracker::new(codec);
let raw_seed = DisplayGeometry::from_codec_config(&config);
tracker.seed_state(raw_seed);
let extra = b.entry_extra_boxes.clone();
let mut cached_raw: Option<Arc<DisplayInfo>> = None;
let mut cached_overlaid: Option<Arc<DisplayInfo>> = None;
for sample in b.samples.iter_mut() {
let length_size = config_nal_length_size(&config).unwrap_or(crate::annexb::NAL_LENGTH_SIZE);
if let Ok(nals) = crate::annexb::iter_length_prefixed_nals_with(&sample.data, length_size) {
let mut annexb = Vec::with_capacity(sample.data.len());
for nal in nals {
annexb.extend_from_slice(&[0, 0, 0, 1]);
annexb.extend_from_slice(nal);
}
let _ = tracker.observe(&annexb);
}
let raw_state = tracker.state().cloned();
let display_arc = match raw_state {
None => None,
Some(raw) => {
let overlaid =
if let (Some(prev_raw), Some(prev_ov)) = (&cached_raw, &cached_overlaid) {
if Arc::ptr_eq(prev_raw, &raw) {
prev_ov.clone()
} else {
let ov = overlay_display_info(&raw, &config, &extra);
cached_raw = Some(raw.clone());
cached_overlaid = Some(ov.clone());
ov
}
} else {
let ov = overlay_display_info(&raw, &config, &extra);
cached_raw = Some(raw.clone());
cached_overlaid = Some(ov.clone());
ov
};
Some(overlaid)
}
};
sample.display = display_arc;
}
}
fn overlay_display_info(
raw: &Arc<DisplayInfo>,
config: &CodecConfig,
extra: &[crate::sample_entries::OpaqueBox],
) -> Arc<DisplayInfo> {
let mut info = (**raw).clone();
info.geometry = overlay_sample_entry_geometry(info.geometry, config, extra);
Arc::new(info)
}
fn gcd_u32(mut a: u32, mut b: u32) -> u32 {
while b != 0 {
let t = b;
b = a % b;
a = t;
}
a.max(1)
}
pub(crate) fn skipped_track(err: Error, track_id: crate::TrackId) -> SkippedTrack {
let fourcc = match &err {
Error::UnsupportedSampleEntry { fourcc } => fourcc.clone(),
_ => String::from("unknown"),
};
SkippedTrack {
fourcc,
reason: err.to_string(),
track_id: Some(track_id),
}
}
fn entry_sampling_rate(samplerate_16_16: u32, config_boxes: &[OpaqueBox]) -> u32 {
sampling_rate_override(config_boxes).unwrap_or(samplerate_16_16 >> 16)
}
fn codec_config_from_entry(entry: &SampleEntryVariant) -> Result<CodecConfig> {
match entry {
SampleEntryVariant::Avc1(avc) => Ok(CodecConfig::Avc {
config: avc.config.clone(),
width: avc.visual.width,
height: avc.visual.height,
}),
SampleEntryVariant::Hevc1(hevc) => Ok(CodecConfig::Hevc {
config: hevc.config.clone(),
width: hevc.visual.width,
height: hevc.visual.height,
}),
SampleEntryVariant::Vvc(vvc) => {
let (width, height) = vvc
.config
.config
.dimensions()
.and_then(|(w, h)| Some((u16::try_from(w).ok()?, u16::try_from(h).ok()?)))
.unwrap_or((vvc.visual.width, vvc.visual.height));
Ok(CodecConfig::Vvc {
config: vvc.config.clone(),
width,
height,
})
}
SampleEntryVariant::Av01(av1) => Ok(CodecConfig::Av1 {
config: av1.config.clone(),
width: av1.visual.width,
height: av1.visual.height,
}),
SampleEntryVariant::Vp09(vp9) => Ok(CodecConfig::Vp9 {
config: vp9.config.clone(),
width: vp9.visual.width,
height: vp9.visual.height,
}),
SampleEntryVariant::Mp4v(mp4v) => {
let esds = EsdsBox::parse_body(config_box_body(&mp4v.config_boxes, b"esds")?)?;
Ok(CodecConfig::Mpeg2Video {
esds,
width: mp4v.visual.width,
height: mp4v.visual.height,
})
}
SampleEntryVariant::Mp4a(mp4a) => {
let esds = esds_from_config_boxes(&mp4a.config_boxes)?;
let oti = esds
.es_descriptor
.decoder_config
.as_ref()
.map(|dc| dc.object_type_indication.0);
if oti == Some(OTI_MPEG2_AUDIO) || oti == Some(OTI_MPEG1_AUDIO) {
Ok(CodecConfig::MpegAudio {
esds,
layer: MpegAudioLayer::LayerII,
channel_count: mp4a.channelcount,
sample_rate: entry_sampling_rate(mp4a.samplerate, &mp4a.config_boxes),
sample_size: mp4a.samplesize,
})
} else {
Ok(CodecConfig::Aac {
esds,
channel_count: mp4a.channelcount,
sample_rate: entry_sampling_rate(mp4a.samplerate, &mp4a.config_boxes),
sample_size: mp4a.samplesize,
})
}
}
SampleEntryVariant::Ac3(ac3) => {
let config = Ac3SpecificBox::parse(config_box_body(&ac3.config_boxes, b"dac3")?)?;
Ok(CodecConfig::Ac3 {
config,
channel_count: ac3.channelcount,
sample_rate: entry_sampling_rate(ac3.samplerate, &ac3.config_boxes),
sample_size: ac3.samplesize,
})
}
SampleEntryVariant::Ec3(ec3) => {
let config = Ec3SpecificBox::parse(config_box_body(&ec3.config_boxes, b"dec3")?)?;
Ok(CodecConfig::Eac3 {
config,
channel_count: ec3.channelcount,
sample_rate: entry_sampling_rate(ec3.samplerate, &ec3.config_boxes),
sample_size: ec3.samplesize,
})
}
SampleEntryVariant::Opus(opus) => {
let config = OpusSpecificBox::parse(config_box_body(&opus.config_boxes, b"dOps")?)?;
Ok(CodecConfig::Opus {
config,
channel_count: opus.channelcount,
sample_rate: entry_sampling_rate(opus.samplerate, &opus.config_boxes),
sample_size: opus.samplesize,
})
}
SampleEntryVariant::Flac(flac) => {
let config = FlacSpecificBox::parse(config_box_body(&flac.config_boxes, b"dfLa")?)?;
Ok(CodecConfig::Flac {
config,
channel_count: flac.channelcount,
sample_rate: entry_sampling_rate(flac.samplerate, &flac.config_boxes),
sample_size: flac.samplesize,
})
}
SampleEntryVariant::Dts(dts) => {
let config = DtsSpecificBox::parse(config_box_body(&dts.config_boxes, b"ddts")?)?;
Ok(CodecConfig::Dts {
config,
codec_fourcc: dts.codec_type,
channel_count: dts.channelcount,
sample_rate: entry_sampling_rate(dts.samplerate, &dts.config_boxes),
sample_size: dts.samplesize,
})
}
SampleEntryVariant::Mha(mha) => {
let config = MHADecoderConfigurationRecord::parse(config_box_body(
&mha.config_boxes,
&MHAC_FOURCC,
)?)?;
Ok(CodecConfig::MpegH {
config,
channel_count: mha.channelcount,
sample_rate: entry_sampling_rate(mha.samplerate, &mha.config_boxes),
sample_size: mha.samplesize,
})
}
SampleEntryVariant::Ac4(ac4) => {
let config = Ac4SpecificBox::parse(config_box_body(
&ac4.config_boxes,
&crate::ac4::DAC4_FOURCC,
)?)?;
Ok(CodecConfig::Ac4 {
config,
channel_count: ac4.channelcount,
sample_rate: entry_sampling_rate(ac4.samplerate, &ac4.config_boxes),
sample_size: ac4.samplesize,
})
}
SampleEntryVariant::Stpp(s) => Ok(CodecConfig::Subtitle {
format: SubtitleFormat::Ttml,
sample_entry: Some(SubtitleSampleEntry::Ttml((**s).clone())),
}),
SampleEntryVariant::Wvtt(w) => Ok(CodecConfig::Subtitle {
format: SubtitleFormat::WebVtt,
sample_entry: Some(SubtitleSampleEntry::WebVtt((**w).clone())),
}),
SampleEntryVariant::Unknown(entry) => Err(Error::UnsupportedSampleEntry {
fourcc: String::from_utf8_lossy(&entry.box_type).into_owned(),
}),
}
}
pub(crate) fn refine_legacy_config(config: &mut CodecConfig, samples: &[Sample]) {
let Some(first) = samples.first() else {
return;
};
match config {
CodecConfig::Mpeg2Video { width, height, .. }
if let Ok(sh) = Mpeg2SeqHeader::find(&first.data) =>
{
*width = sh.width;
*height = sh.height;
}
CodecConfig::MpegAudio { layer, .. }
if let Ok(hdr) = MpegAudioFrameHeader::parse(&first.data) =>
{
*layer = hdr.layer;
}
_ => {}
}
}
fn config_nal_length_size(config: &CodecConfig) -> Option<usize> {
match config {
CodecConfig::Avc { config, .. } => {
Some(usize::from(config.config.length_size_minus_one) + 1)
}
CodecConfig::Hevc { config, .. } => {
Some(usize::from(config.config.length_size_minus_one) + 1)
}
_ => None,
}
}
pub(crate) fn normalise_track_nal_lengths(
spec: &mut TrackSpec,
samples: &mut [Sample],
) -> Result<()> {
let Some(length_size) = config_nal_length_size(&spec.config) else {
return Ok(());
};
if length_size == crate::annexb::NAL_LENGTH_SIZE {
return Ok(());
}
for sample in samples.iter_mut() {
let data = crate::annexb::normalise_nal_length_size(&sample.data, length_size)?;
sample.data = data.into();
}
match &mut spec.config {
CodecConfig::Avc { config, .. } => {
config.config.length_size_minus_one = crate::annexb::NAL_LENGTH_SIZE_MINUS_ONE;
}
CodecConfig::Hevc { config, .. } => {
config.config.length_size_minus_one = crate::annexb::NAL_LENGTH_SIZE_MINUS_ONE;
}
_ => {}
}
Ok(())
}
fn esds_from_config_boxes(boxes: &[OpaqueBox]) -> Result<EsdsBox> {
EsdsBox::parse_body(config_box_body(boxes, b"esds")?)
}
fn config_box_body<'b>(boxes: &'b [OpaqueBox], fourcc: &[u8; 4]) -> Result<&'b [u8]> {
boxes
.iter()
.find(|b| &b.box_type == fourcc)
.map(|b| b.data.as_slice())
.ok_or(Error::UnexpectedBox {
expected: "config box in audio sample entry",
})
}
#[cfg(test)]
mod tests {
use super::*;
use crate::TrackId;
use crate::subtitle_entries::WvttSampleEntry;
#[test]
fn wvtt_sample_entry_demuxes_to_subtitle_webvtt() {
let wvtt = WvttSampleEntry::new("WEBVTT\n");
let entry = SampleEntryVariant::Wvtt(alloc::boxed::Box::new(wvtt));
let config = codec_config_from_entry(&entry).expect("wvtt must now demux");
assert!(
matches!(
config,
CodecConfig::Subtitle {
format: SubtitleFormat::WebVtt,
sample_entry: Some(SubtitleSampleEntry::WebVtt(ref e)),
} if e.config.config == "WEBVTT\n"
),
"wvtt must map to CodecConfig::Subtitle{{format: SubtitleFormat::WebVtt}}, got {config:?}"
);
}
#[test]
fn unknown_sample_entry_errors_naming_the_fourcc() {
let unknown = OpaqueBox::new(*b"xyz9", vec![1, 2, 3]);
let entry = SampleEntryVariant::Unknown(unknown);
let err = codec_config_from_entry(&entry).unwrap_err();
match err {
Error::UnsupportedSampleEntry { fourcc } => assert_eq!(fourcc, "xyz9"),
other => panic!("expected UnsupportedSampleEntry, got {other:?}"),
}
}
fn minimal_avc_config() -> crate::avc_config::AVCConfigurationBox {
use crate::avc_config::{AVCConfigurationBox, AVCDecoderConfigurationRecord};
use crate::nalu_types::{AvcPps, AvcSps};
AVCConfigurationBox {
config: AVCDecoderConfigurationRecord {
configuration_version: 1,
profile_indication: 66,
profile_compatibility: 0,
level_indication: 0x1E,
length_size_minus_one: 3,
sps: vec![AvcSps(vec![0x67, 0x42, 0x00, 0x1E, 0xAB, 0x40])],
pps: vec![AvcPps(vec![0x68, 0xCE, 0x3C, 0x80])],
chroma_format: None,
bit_depth_luma_minus8: None,
bit_depth_chroma_minus8: None,
sps_ext: Vec::new(),
},
}
}
#[test]
fn gapped_fmp4_reseeds_absolute_dts_from_each_fragments_own_tfdt() {
const TIMESCALE: u32 = 90_000;
const FRAME_DUR: u32 = 3_000;
const FRAMES_PER_FRAGMENT: u32 = 4;
const GAP_TICKS: u64 = FRAME_DUR as u64;
let spec = TrackSpec::new(
TrackId::new(1),
TIMESCALE,
CodecConfig::Avc {
config: minimal_avc_config(),
width: 16,
height: 16,
},
);
let init = build_init_segment(core::slice::from_ref(&spec), TIMESCALE)
.expect("init segment builds");
let frames = |base: i64| -> Vec<Sample> {
(0..FRAMES_PER_FRAGMENT)
.map(|i| {
let nal = [0x41u8, 0xAA, 0xBB];
let mut data = (nal.len() as u32).to_be_bytes().to_vec();
data.extend_from_slice(&nal);
let dts = base + i64::from(i) * i64::from(FRAME_DUR);
Sample::new(data, Some(dts), Some(dts), Some(FRAME_DUR), i == 0)
})
.collect()
};
let first_base = 0u64;
let contiguous_second_base = u64::from(FRAMES_PER_FRAGMENT * FRAME_DUR);
let gapped_second_base = contiguous_second_base + GAP_TICKS;
let f1 = frames(first_base as i64);
let f2 = frames(gapped_second_base as i64);
let seg1 = build_media_segment(
1,
&[FragmentTrackData::new(TrackId::new(1), first_base, &f1)],
)
.expect("fragment 1 builds");
let seg2 = build_media_segment(
2,
&[FragmentTrackData::new(
TrackId::new(1),
gapped_second_base,
&f2,
)],
)
.expect("fragment 2 builds");
let mut file = init;
file.extend_from_slice(&seg1);
file.extend_from_slice(&seg2);
let media = Fmp4Demux::new()
.unpackage(&file)
.expect("demux gapped fMP4");
let track = &media.tracks[0];
assert_eq!(
track.samples.len(),
(FRAMES_PER_FRAGMENT * 2) as usize,
"both fragments' samples must be present"
);
assert_eq!(
track.start_decode_time, first_base,
"start_decode_time stays the FIRST fragment's anchor, not the last"
);
let first_post_gap = &track.samples[FRAMES_PER_FRAGMENT as usize];
assert_eq!(
first_post_gap.dts,
Some(gapped_second_base as i64),
"post-gap dts must come from the second fragment's tfdt \
({gapped_second_base}), not the pure duration sum \
({contiguous_second_base})"
);
for (i, s) in track.samples[FRAMES_PER_FRAGMENT as usize..]
.iter()
.enumerate()
{
let expected = gapped_second_base as i64 + (i as i64) * i64::from(FRAME_DUR);
assert_eq!(s.dts, Some(expected), "sample {i} after the gap");
}
}
#[test]
fn contiguous_fmp4_dts_is_unchanged_by_the_per_fragment_tfdt_reseed() {
const TIMESCALE: u32 = 90_000;
const FRAME_DUR: u32 = 3_000;
const FRAMES: u32 = 3;
let spec = TrackSpec::new(
TrackId::new(7),
TIMESCALE,
CodecConfig::Avc {
config: minimal_avc_config(),
width: 16,
height: 16,
},
);
let init = build_init_segment(core::slice::from_ref(&spec), TIMESCALE).unwrap();
let frames = |base: i64| -> Vec<Sample> {
(0..FRAMES)
.map(|i| {
let nal = [0x41u8, 0xAA, 0xBB];
let mut data = (nal.len() as u32).to_be_bytes().to_vec();
data.extend_from_slice(&nal);
let dts = base + i64::from(i) * i64::from(FRAME_DUR);
Sample::new(data, Some(dts), Some(dts), Some(FRAME_DUR), i == 0)
})
.collect()
};
let second_base = u64::from(FRAMES * FRAME_DUR);
let f1 = frames(0);
let f2 = frames(second_base as i64);
let mut file = init;
file.extend_from_slice(
&build_media_segment(1, &[FragmentTrackData::new(TrackId::new(7), 0, &f1)]).unwrap(),
);
file.extend_from_slice(
&build_media_segment(
2,
&[FragmentTrackData::new(TrackId::new(7), second_base, &f2)],
)
.unwrap(),
);
let media = Fmp4Demux::new().unpackage(&file).unwrap();
let got: Vec<i64> = media.tracks[0]
.samples
.iter()
.filter_map(|s| s.dts)
.collect();
let want: Vec<i64> = (0..FRAMES * 2)
.map(|i| i64::from(i) * i64::from(FRAME_DUR))
.collect();
assert_eq!(got, want, "a gapless stream must be unaffected by the fix");
}
#[test]
fn hls_packager_omits_a_timestampless_track_instead_of_emitting_extinf_zero() {
const TIMESCALE: u32 = 90_000;
const FRAME_DUR: u32 = 3_000;
let timed = Track::new(
TrackSpec::new(
TrackId::new(1),
TIMESCALE,
CodecConfig::Avc {
config: minimal_avc_config(),
width: 16,
height: 16,
},
),
(0..3)
.map(|i| {
Sample::new(
vec![0u8; 4],
Some(i64::from(i) * i64::from(FRAME_DUR)),
Some(i64::from(i) * i64::from(FRAME_DUR)),
Some(FRAME_DUR),
true,
)
})
.collect::<Vec<_>>(),
);
let sections = Track::new(
TrackSpec::new(
TrackId::new(2),
TIMESCALE,
CodecConfig::Data {
stream_type: 0x86,
descriptors: Vec::new(),
carriage: crate::ir::DataCarriage::Sections,
},
),
vec![Sample::new(vec![0xFCu8; 8], None, None, None, true)],
);
let playlist = HlsPackager::default()
.package(&Media::new(vec![timed, sections], TIMESCALE))
.expect("a Media with one timed track still packages");
let zero_extinf = playlist.lines().any(|l| {
l.strip_prefix("#EXTINF:")
.and_then(|rest| rest.trim_end_matches(',').parse::<f64>().ok())
.is_some_and(|secs| secs == 0.0)
});
assert!(
!zero_extinf,
"no zero-duration EXTINF may be emitted, got:\n{playlist}"
);
assert!(
!playlist.contains("seg2.m4s"),
"the timestamp-less track must be omitted entirely, got:\n{playlist}"
);
assert!(
playlist.contains("seg1.m4s"),
"the timed track must still be present, got:\n{playlist}"
);
let only_sections = Track::new(
TrackSpec::new(
TrackId::new(2),
TIMESCALE,
CodecConfig::Data {
stream_type: 0x86,
descriptors: Vec::new(),
carriage: crate::ir::DataCarriage::Sections,
},
),
vec![Sample::new(vec![0xFCu8; 8], None, None, None, true)],
);
assert!(
HlsPackager::default()
.package(&Media::new(vec![only_sections], TIMESCALE))
.is_err(),
"an all-timestamp-less Media must error, not render an empty playlist"
);
}
#[test]
fn fmp4_demux_normalises_two_byte_nal_lengths() {
use crate::annexb::NAL_LENGTH_SIZE_MINUS_ONE;
const TIMESCALE: u32 = 90_000;
use crate::ir::{FragmentTrackData, Sample, SampleFlags};
use crate::pipeline::{CodecConfig, TrackSpec, build_init_segment, build_media_segment};
let mut avcc = minimal_avc_config();
avcc.config.length_size_minus_one = 1;
let spec = TrackSpec::new(
TrackId::new(1),
TIMESCALE,
CodecConfig::Avc {
config: avcc,
width: 640,
height: 360,
},
);
let sample = Sample {
data: vec![0x00, 0x05, 0x65, 0x88, 0x84, 0x00, 0x21, 0x00, 0x01, 0x41].into(),
dts: Some(0),
pts: Some(0),
duration: Some(1000),
flags: SampleFlags::new(true),
provenance: None,
display: None,
};
let init = build_init_segment(std::slice::from_ref(&spec), TIMESCALE).expect("init");
let hdr = init
.windows(4)
.position(|w| w == b"avcC")
.expect("avcC present");
assert_eq!(
init[hdr + 8] & 0x03,
1,
"the built init segment must declare lengthSizeMinusOne = 1"
);
let frag = build_media_segment(
1,
&[FragmentTrackData::new(
TrackId::new(1),
0,
std::slice::from_ref(&sample),
)],
)
.expect("media segment");
let mut file = init;
file.extend_from_slice(&frag);
let media = Fmp4Demux::new()
.unpackage(&file)
.expect("demux the 2-byte fMP4");
let track = &media.tracks[0];
assert_eq!(
track.samples[0].data.as_ref(),
[
0x00, 0x00, 0x00, 0x05, 0x65, 0x88, 0x84, 0x00, 0x21, 0x00, 0x00, 0x00, 0x01, 0x41,
],
"2-byte NAL lengths must be rewritten to 4-byte"
);
let CodecConfig::Avc { config, .. } = &track.spec.config else {
panic!("expected CodecConfig::Avc");
};
assert_eq!(
config.config.length_size_minus_one, NAL_LENGTH_SIZE_MINUS_ONE,
"the reported avcC must declare the 4-byte length size"
);
}
}