use crate::TrackId;
use alloc::format;
use alloc::string::{String, ToString};
use alloc::vec::Vec;
use broadcast_common::{Parse, Serialize};
use crate::aac_asc::AudioSpecificConfig;
use crate::box_types::{BoxHeader, BoxType, UUID_TYPE_SIZE};
use crate::error::{Error, Result};
use crate::media::{Media, TimelineOrigin, Track, relative_decode_times};
use crate::movie_fragment::{
MediaRunSample, MovieFragmentBox, MovieFragmentHeaderBox, TrackFragmentBox, media_fragment_run,
};
use crate::pipeline::{CodecConfig, Sample};
use crate::segments::{MediaDataBox, SegmentTypeBox};
use quick_xml::Writer;
use quick_xml::events::{BytesDecl, BytesEnd, BytesStart, Event};
pub const SMOOTH_TIMESCALE: u64 = 10_000_000;
const MAJOR_VERSION: u32 = 2;
const MINOR_VERSION: u32 = 0;
pub const TFXD_UUID: [u8; UUID_TYPE_SIZE] = [
0x6d, 0x1d, 0x9b, 0x05, 0x42, 0xd5, 0x44, 0xe6, 0x80, 0xe2, 0x14, 0x1d, 0xaf, 0xf7, 0x57, 0xb2,
];
pub const FOURCC_H264: &str = "H264";
pub const FOURCC_AVC1: &str = "AVC1";
pub const FOURCC_AACL: &str = "AACL";
pub const FOURCC_AACH: &str = "AACH";
pub const FOURCC_STPP: &str = "stpp";
pub const FOURCC_WVTT: &str = "wvtt";
pub const SUBTYPE_SUBT: &str = "SUBT";
const AUDIO_TAG_AAC: u32 = 255;
const AAC_BITS_PER_SAMPLE: u16 = 16;
const START_CODE: [u8; 4] = [0x00, 0x00, 0x00, 0x01];
const STYP_MAJOR_BRAND: [u8; 4] = *b"msdh";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum SmoothStreamType {
Video,
Audio,
Text,
}
impl SmoothStreamType {
pub fn name(&self) -> &'static str {
match self {
SmoothStreamType::Video => "video",
SmoothStreamType::Audio => "audio",
SmoothStreamType::Text => "text",
}
}
}
broadcast_common::impl_spec_display!(SmoothStreamType);
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize))]
pub struct TfxdBox {
pub fragment_absolute_time: u64,
pub fragment_duration: u64,
}
impl TfxdBox {
const VERSION: u8 = 1;
const PAYLOAD_LEN: usize = 16;
pub fn new(fragment_absolute_time: u64, fragment_duration: u64) -> Self {
Self {
fragment_absolute_time,
fragment_duration,
}
}
pub fn parse_body(body: &[u8]) -> Result<Self> {
let need = 4 + Self::PAYLOAD_LEN;
if body.len() < need {
return Err(Error::BufferTooShort {
need,
have: body.len(),
what: "tfxd body",
});
}
let p = &body[4..];
let fragment_absolute_time =
u64::from_be_bytes([p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7]]);
let fragment_duration =
u64::from_be_bytes([p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]]);
Ok(Self {
fragment_absolute_time,
fragment_duration,
})
}
}
impl Serialize for TfxdBox {
type Error = Error;
fn serialized_len(&self) -> usize {
let hdr = BoxHeader::new(0, BoxType::from_bytes(*b"uuid"), Some(TFXD_UUID));
hdr.serialized_len() + 4 + Self::PAYLOAD_LEN
}
fn serialize_into(&self, buf: &mut [u8]) -> Result<usize> {
let need = self.serialized_len();
if buf.len() < need {
return Err(Error::OutputBufferTooSmall {
need,
have: buf.len(),
});
}
let hdr = BoxHeader::new(need as u64, BoxType::from_bytes(*b"uuid"), Some(TFXD_UUID));
let mut c = hdr.serialize_into(buf)?;
buf[c] = Self::VERSION;
buf[c + 1] = 0;
buf[c + 2] = 0;
buf[c + 3] = 0;
c += 4;
buf[c..c + 8].copy_from_slice(&self.fragment_absolute_time.to_be_bytes());
c += 8;
buf[c..c + 8].copy_from_slice(&self.fragment_duration.to_be_bytes());
c += 8;
Ok(c)
}
}
#[derive(Debug, Clone)]
pub struct SmoothFragment {
pub track_id: TrackId,
pub sequence_number: u32,
pub start_time: u64,
pub duration: u64,
pub data: Vec<u8>,
}
#[derive(Debug, Clone)]
pub struct SmoothOutput {
pub manifest: String,
pub fragments: Vec<SmoothFragment>,
}
#[derive(Debug, Clone)]
pub struct SmoothPackager {
pub target_duration_secs: u32,
}
impl Default for SmoothPackager {
fn default() -> Self {
Self {
target_duration_secs: 2,
}
}
}
struct StreamInfo {
stream_type: SmoothStreamType,
fourcc: &'static str,
codec_private_data: Vec<u8>,
bitrate: u64,
width: Option<u32>,
height: Option<u32>,
sampling_rate: Option<u32>,
channels: Option<u16>,
fragments: Vec<FragmentTiming>,
total_duration: u64,
}
struct FragmentTiming {
start_time: u64,
duration: u64,
}
impl SmoothPackager {
fn to_smooth_ticks(ticks: u64, media_timescale: u32) -> u64 {
let ts = media_timescale.max(1) as u64;
let num = ticks.saturating_mul(SMOOTH_TIMESCALE);
(num + ts / 2) / ts
}
fn video_codec_private_data(config: &crate::avc_config::AVCConfigurationBox) -> Vec<u8> {
let mut out = Vec::new();
for sps in &config.config.sps {
out.extend_from_slice(&START_CODE);
out.extend_from_slice(&sps.0);
}
for pps in &config.config.pps {
out.extend_from_slice(&START_CODE);
out.extend_from_slice(&pps.0);
}
out
}
fn audio_codec_private_data(esds: &crate::mp4esds::EsdsBox) -> Result<Vec<u8>> {
let dsi = esds.require_decoder_specific_info()?;
Ok(dsi.to_vec())
}
fn build_track(
&self,
track: &Track,
origin: &TimelineOrigin,
next_seq: &mut u32,
fragments_out: &mut Vec<SmoothFragment>,
) -> Result<StreamInfo> {
let media_timescale = track.spec.timescale.max(1);
let (stream_type, fourcc, codec_private_data, width, height, sampling_rate, channels) =
resolve_codec(&track.spec.config)?;
let timeline = smooth_timeline(track, origin, media_timescale)?;
let groups = segment_samples(&track.samples, media_timescale, self.target_duration_secs);
let mut fragment_timings = Vec::with_capacity(groups.len());
let mut total_bytes = 0u64;
for range in groups {
let group = &track.samples[range.clone()];
let times = &timeline[range];
let start_smooth = times.first().map_or(0, |t| t.start);
let dur_smooth: u64 = times.iter().map(|t| u64::from(t.duration)).sum();
let data = build_smooth_fragment(
track.spec.track_id,
*next_seq,
start_smooth,
dur_smooth,
group,
times,
)?;
for s in group.iter() {
total_bytes += s.data.len() as u64;
}
fragments_out.push(SmoothFragment {
track_id: track.spec.track_id,
sequence_number: *next_seq,
start_time: start_smooth,
duration: dur_smooth,
data,
});
fragment_timings.push(FragmentTiming {
start_time: start_smooth,
duration: dur_smooth,
});
*next_seq += 1;
}
let total_duration: u64 = fragment_timings.iter().map(|f| f.duration).sum();
let bitrate = {
let bits_ts = total_bytes
.saturating_mul(8)
.saturating_mul(SMOOTH_TIMESCALE);
(bits_ts + total_duration / 2)
.checked_div(total_duration)
.unwrap_or(0)
}
.max(1);
Ok(StreamInfo {
stream_type,
fourcc,
codec_private_data,
bitrate,
width,
height,
sampling_rate,
channels,
fragments: fragment_timings,
total_duration,
})
}
pub fn package_track_fragment(
&self,
track: &Track,
sequence_number: u32,
start_smooth: u64,
) -> Result<SmoothFragment> {
if track.samples.is_empty() {
return Err(Error::InvalidInput(
"cannot package a Smooth fragment from a track with no samples",
));
}
let media_timescale = track.spec.timescale.max(1);
let origin = TimelineOrigin::of(core::iter::once(track));
let times = smooth_timeline(track, &origin, media_timescale)?;
let dur_smooth: u64 = times.iter().map(|t| u64::from(t.duration)).sum();
let data = build_smooth_fragment(
track.spec.track_id,
sequence_number,
start_smooth,
dur_smooth,
&track.samples,
×,
)?;
Ok(SmoothFragment {
track_id: track.spec.track_id,
sequence_number,
start_time: start_smooth,
duration: dur_smooth,
data,
})
}
fn render_manifest(&self, streams: &[StreamInfo]) -> String {
let mut writer = new_writer();
let w = &mut writer;
declaration(w);
let duration = streams.iter().map(|s| s.total_duration).max().unwrap_or(0);
open(
w,
"SmoothStreamingMedia",
&[
("MajorVersion", MAJOR_VERSION.to_string()),
("MinorVersion", MINOR_VERSION.to_string()),
("Duration", duration.to_string()),
("TimeScale", SMOOTH_TIMESCALE.to_string()),
],
);
let mut seen: Vec<(SmoothStreamType, usize)> = Vec::new();
for s in streams {
let ty = s.stream_type.name();
let ordinal = match seen.iter_mut().find(|(t, _)| *t == s.stream_type) {
Some((_, n)) => {
*n += 1;
*n
}
None => {
seen.push((s.stream_type, 1));
1
}
};
let name = if ordinal == 1 {
ty.to_string()
} else {
format!("{ty}_{ordinal}")
};
let url = format!("QualityLevels({{bitrate}})/Fragments({name}={{start time}})");
let mut attrs = alloc::vec![("Type", ty.to_string())];
if ordinal > 1 {
attrs.push(("Name", name));
}
let subtype = if s.stream_type == SmoothStreamType::Text {
SUBTYPE_SUBT.to_string()
} else {
String::new()
};
attrs.extend([
("Subtype", subtype),
("Chunks", s.fragments.len().to_string()),
("QualityLevels", "1".to_string()),
("Url", url),
]);
open(w, "StreamIndex", &attrs);
let mut ql = alloc::vec![
("Index", "0".to_string()),
("Bitrate", s.bitrate.to_string()),
("FourCC", s.fourcc.to_string()),
];
match s.stream_type {
SmoothStreamType::Video if let (Some(wd), Some(ht)) = (s.width, s.height) => {
ql.push(("MaxWidth", wd.to_string()));
ql.push(("MaxHeight", ht.to_string()));
ql.push((
"CodecPrivateData",
::hex::encode_upper(&s.codec_private_data),
));
}
SmoothStreamType::Video => {
ql.push((
"CodecPrivateData",
::hex::encode_upper(&s.codec_private_data),
));
}
SmoothStreamType::Text => {}
SmoothStreamType::Audio => {
if let Some(sr) = s.sampling_rate {
ql.push(("SamplingRate", sr.to_string()));
}
if let Some(ch) = s.channels {
ql.push(("Channels", ch.to_string()));
}
ql.push(("BitsPerSample", AAC_BITS_PER_SAMPLE.to_string()));
ql.push(("AudioTag", AUDIO_TAG_AAC.to_string()));
ql.push((
"CodecPrivateData",
::hex::encode_upper(&s.codec_private_data),
));
}
}
empty(w, "QualityLevel", &ql);
for (i, f) in s.fragments.iter().enumerate() {
let mut c = alloc::vec![("n", i.to_string())];
if i == 0 {
c.push(("t", f.start_time.to_string()));
}
c.push(("d", f.duration.to_string()));
empty(w, "c", &c);
}
close(w, "StreamIndex");
}
close(w, "SmoothStreamingMedia");
finish(writer)
}
}
impl broadcast_common::Package for SmoothPackager {
type Media = Media;
type Output = SmoothOutput;
type Error = Error;
fn package(&mut self, media: &Media) -> Result<SmoothOutput> {
if media.tracks.is_empty() {
return Err(Error::InvalidInput("cannot package a Media with no tracks"));
}
let mut fragments = Vec::new();
let mut streams = Vec::with_capacity(media.tracks.len());
let origin = TimelineOrigin::of(&media.tracks);
for track in &media.tracks {
let mut next_seq = 1u32;
streams.push(self.build_track(track, &origin, &mut next_seq, &mut fragments)?);
}
let manifest = self.render_manifest(&streams);
Ok(SmoothOutput {
manifest,
fragments,
})
}
}
type ResolvedCodec = (
SmoothStreamType,
&'static str,
Vec<u8>,
Option<u32>,
Option<u32>,
Option<u32>,
Option<u16>,
);
fn resolve_codec(config: &CodecConfig) -> Result<ResolvedCodec> {
match config {
CodecConfig::Avc { config, .. } => {
let cpd = SmoothPackager::video_codec_private_data(config);
let (w, h) = config
.config
.sps
.first()
.and_then(|sps| sps.decode().ok())
.map(|i| (i.width, i.height))
.ok_or(Error::InvalidInput(
"AVC track has no decodable SPS for Smooth geometry",
))?;
Ok((
SmoothStreamType::Video,
FOURCC_H264,
cpd,
Some(w),
Some(h),
None,
None,
))
}
CodecConfig::Aac {
esds,
channel_count,
sample_rate,
..
} => {
let cpd = SmoothPackager::audio_codec_private_data(esds)?;
let rate = AudioSpecificConfig::parse(&cpd)
.ok()
.and_then(|asc| asc.effective_sampling_frequency())
.unwrap_or(*sample_rate);
Ok((
SmoothStreamType::Audio,
FOURCC_AACL,
cpd,
None,
None,
Some(rate),
Some(*channel_count),
))
}
CodecConfig::Subtitle {
sample_entry: Some(entry),
..
} => {
let fourcc = match entry {
crate::pipeline::SubtitleSampleEntry::Ttml(_) => FOURCC_STPP,
crate::pipeline::SubtitleSampleEntry::WebVtt(_) => FOURCC_WVTT,
};
Ok((
SmoothStreamType::Text,
fourcc,
Vec::new(),
None,
None,
None,
None,
))
}
_ => Err(Error::InvalidInput(
"Smooth Streaming supports only H.264 video, AAC-LC audio and ISOBMFF (stpp/wvtt) subtitles",
)),
}
}
fn segment_samples(
samples: &[Sample],
media_timescale: u32,
target_secs: u32,
) -> Vec<core::ops::Range<usize>> {
let mut groups = Vec::new();
if samples.is_empty() {
return groups;
}
let target_ticks = (target_secs.max(1) as u64).saturating_mul(media_timescale.max(1) as u64);
let mut start = 0usize;
let mut acc_dur = 0u64;
for (i, s) in samples.iter().enumerate() {
if i > start && s.flags.is_sync && acc_dur >= target_ticks {
groups.push(start..i);
start = i;
acc_dur = 0;
}
acc_dur += s.duration.unwrap_or(0) as u64;
}
groups.push(start..samples.len());
groups
}
#[derive(Debug, Clone, Copy)]
struct SmoothSampleTime {
start: u64,
duration: u32,
composition_offset: i32,
}
fn smooth_timeline(
track: &Track,
origin: &TimelineOrigin,
media_timescale: u32,
) -> Result<Vec<SmoothSampleTime>> {
let times = relative_decode_times(track, origin);
let starts: Vec<u64> = times
.iter()
.map(|&t| SmoothPackager::to_smooth_ticks(t.max(0) as u64, media_timescale))
.collect();
track
.samples
.iter()
.enumerate()
.map(|(i, s)| {
let start = starts[i];
let own_end = SmoothPackager::to_smooth_ticks(
(times[i].max(0) as u64).saturating_add(u64::from(s.duration.unwrap_or(0))),
media_timescale,
);
let end = match starts.get(i + 1) {
Some(&next) if next >= start => next,
_ => own_end,
};
let duration = u32::try_from(end - start).map_err(|_| Error::InvalidValue {
field: "trun sample_duration",
value: end - start,
reason: "exceeds 32 bits in the 10 MHz Smooth timescale",
})?;
let cto = to_smooth_signed(i64::from(s.composition_offset()), media_timescale);
let composition_offset = i32::try_from(cto).map_err(|_| Error::InvalidValue {
field: "trun sample_composition_time_offset",
value: cto.unsigned_abs(),
reason: "exceeds 32 bits in the 10 MHz Smooth timescale",
})?;
Ok(SmoothSampleTime {
start,
duration,
composition_offset,
})
})
.collect()
}
fn to_smooth_signed(ticks: i64, media_timescale: u32) -> i64 {
let magnitude = SmoothPackager::to_smooth_ticks(ticks.unsigned_abs(), media_timescale);
let v = i64::try_from(magnitude).unwrap_or(i64::MAX);
if ticks < 0 { -v } else { v }
}
fn build_smooth_fragment(
track_id: TrackId,
sequence_number: u32,
start_smooth: u64,
dur_smooth: u64,
samples: &[Sample],
times: &[SmoothSampleTime],
) -> Result<Vec<u8>> {
let styp = SegmentTypeBox {
major_brand: STYP_MAJOR_BRAND,
minor_version: 0,
compatible_brands: alloc::vec![STYP_MAJOR_BRAND, *b"msix"],
};
let (tfhd, trun) = media_fragment_run(
track_id,
samples.iter().zip(times).map(|(s, t)| MediaRunSample {
duration: t.duration,
size: s.data.len(),
is_sync: s.flags.is_sync,
composition_offset: t.composition_offset,
}),
)?;
let tfxd = TfxdBox::new(start_smooth, dur_smooth);
let mut moof = MovieFragmentBox::new(
MovieFragmentHeaderBox::new(sequence_number),
alloc::vec![TrackFragmentBox::new(tfhd, None, alloc::vec![trun])],
);
let tfxd_len = tfxd.serialized_len();
let moof_size = moof.serialized_len() + tfxd_len;
let mdat_payload_len: u64 = samples.iter().map(|s| s.data.len() as u64).sum();
let mdat_header =
BoxHeader::for_payload(BoxType::from_bytes(*b"mdat"), None, mdat_payload_len).header_size();
let mdat_start = moof_size + mdat_header;
let data_offset = i32::try_from(mdat_start).map_err(|_| Error::InvalidValue {
field: "trun data_offset",
value: mdat_start as u64,
reason: "moof too large for the signed 32-bit trun data_offset",
})?;
moof.traf[0].trun[0].data_offset = Some(data_offset);
let base_moof = moof.try_to_bytes()?;
let with_tfxd = splice_tfxd_into_traf(&base_moof, &tfxd)?;
debug_assert_eq!(with_tfxd.len(), moof_size);
let mut mdat_data = Vec::new();
for s in samples {
mdat_data.extend_from_slice(&s.data);
}
let mdat = MediaDataBox { data: mdat_data };
let total = styp.serialized_len() + with_tfxd.len() + mdat.serialized_len();
let mut out = alloc::vec![0u8; total];
let mut c = 0usize;
c += styp.serialize_into(&mut out[c..])?;
out[c..c + with_tfxd.len()].copy_from_slice(&with_tfxd);
c += with_tfxd.len();
mdat.serialize_into(&mut out[c..])?;
Ok(out)
}
fn splice_tfxd_into_traf(moof: &[u8], tfxd: &TfxdBox) -> Result<Vec<u8>> {
if moof.len() < 8 || &moof[4..8] != b"moof" {
return Err(Error::UnexpectedBox { expected: "moof" });
}
let mfhd_size = u32::from_be_bytes([moof[8], moof[9], moof[10], moof[11]]) as usize;
let traf_off = 8 + mfhd_size;
if moof.len() < traf_off + 8 || &moof[traf_off + 4..traf_off + 8] != b"traf" {
return Err(Error::UnexpectedBox { expected: "traf" });
}
let old_traf_size = u32::from_be_bytes([
moof[traf_off],
moof[traf_off + 1],
moof[traf_off + 2],
moof[traf_off + 3],
]) as usize;
let tfxd_bytes = tfxd.try_to_bytes()?;
let mut out = Vec::with_capacity(moof.len() + tfxd_bytes.len());
out.extend_from_slice(moof);
out.extend_from_slice(&tfxd_bytes);
let too_big = |size: usize| Error::InvalidValue {
field: "moof/traf box size",
value: size as u64,
reason: "exceeds the 32-bit box size of the serialized moof",
};
let new_moof_size = u32::try_from(out.len()).map_err(|_| too_big(out.len()))?;
out[0..4].copy_from_slice(&new_moof_size.to_be_bytes());
let traf_size = old_traf_size + tfxd_bytes.len();
let new_traf_size = u32::try_from(traf_size).map_err(|_| too_big(traf_size))?;
out[traf_off..traf_off + 4].copy_from_slice(&new_traf_size.to_be_bytes());
Ok(out)
}
type XmlWriter = Writer<Vec<u8>>;
fn new_writer() -> XmlWriter {
Writer::new_with_indent(Vec::new(), b' ', 2)
}
fn declaration(w: &mut XmlWriter) {
let _ = w.write_event(Event::Decl(BytesDecl::new("1.0", Some("utf-8"), None)));
}
fn element_tag<'a>(name: &'a str, attrs: &'a [(&str, String)]) -> BytesStart<'a> {
let mut start = BytesStart::new(name);
for (key, value) in attrs {
start.push_attribute((*key, value.as_str()));
}
start
}
fn open(w: &mut XmlWriter, name: &str, attrs: &[(&str, String)]) {
let _ = w.write_event(Event::Start(element_tag(name, attrs)));
}
fn empty(w: &mut XmlWriter, name: &str, attrs: &[(&str, String)]) {
let _ = w.write_event(Event::Empty(element_tag(name, attrs)));
}
fn close(w: &mut XmlWriter, name: &str) {
let _ = w.write_event(Event::End(BytesEnd::new(name)));
}
fn finish(w: XmlWriter) -> String {
let mut out = String::from_utf8_lossy(&w.into_inner()).into_owned();
out.push('\n');
out
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn writer_escapes_attributes_and_text() {
const SPECIALS: &str = "a&b<c>d\"e'f";
let mut writer = new_writer();
let w = &mut writer;
open(w, "Root", &[("k", SPECIALS.to_string())]);
empty(w, "Leaf", &[("v", SPECIALS.to_string())]);
let _ = w.write_event(Event::Start(BytesStart::new("T")));
let _ = w.write_event(Event::Text(quick_xml::events::BytesText::new(SPECIALS)));
close(w, "T");
close(w, "Root");
let out = finish(writer);
assert_eq!(
out,
concat!(
"<Root k=\"a&b<c>d"e'f\">\n",
" <Leaf v=\"a&b<c>d"e'f\"/>\n",
" <T>a&b<c>d"e'f</T>\n",
"</Root>\n"
)
);
let parsed = crate::smooth_parse::SmoothManifest::parse(&out);
assert!(matches!(
parsed,
Err(crate::smooth_parse::SmoothParseError::UnexpectedElement { .. })
));
}
#[test]
fn tfxd_round_trip() {
let t = TfxdBox::new(0x0011_2233_4455_6677, 0x0089_ABCD);
let bytes = t.try_to_bytes().unwrap();
assert_eq!(bytes.len(), 44);
assert_eq!(&bytes[4..8], b"uuid");
assert_eq!(&bytes[8..24], &TFXD_UUID);
let p = TfxdBox::parse_body(&bytes[24..]).unwrap();
assert_eq!(p, t);
assert_eq!(p.try_to_bytes().unwrap(), bytes);
}
#[test]
fn segment_samples_keyframe_aligned() {
let mk = |sync: bool| Sample {
data: alloc::vec![0u8; 4].into(),
dts: None,
pts: None,
duration: Some(1),
flags: crate::ir::SampleFlags::new(sync),
provenance: None,
};
let samples = alloc::vec![mk(true), mk(false), mk(true), mk(false)];
let groups = segment_samples(&samples, 1, 2);
assert_eq!(groups.len(), 2);
assert_eq!(groups[0].len(), 2);
assert_eq!(groups[1].len(), 2);
}
#[test]
fn to_smooth_ticks_scales() {
assert_eq!(SmoothPackager::to_smooth_ticks(90_000, 90_000), 10_000_000);
}
}