mod export;
mod import;
pub use export::*;
pub use import::*;
#[cfg(test)]
mod export_test;
#[cfg(test)]
mod import_test;
use std::task::Poll;
use bytes::Bytes;
use hang::catalog::{AudioCodec, AudioConfig, VideoCodec, VideoConfig};
use mp4_atom::Atom;
use crate::container::{Container, Frame, Timestamp};
#[derive(Debug, Clone, thiserror::Error)]
#[non_exhaustive]
pub enum Error {
#[error("mp4: {0}")]
Mp4(std::sync::Arc<mp4_atom::Error>),
#[error("moq: {0}")]
Moq(#[from] moq_net::Error),
#[error("flac: {0}")]
Flac(#[from] crate::codec::flac::Error),
#[error("missing keyframe: a group must open on a keyframe")]
MissingKeyframe(#[from] crate::container::MissingKeyframe),
#[error("timestamp overflow")]
TimestampOverflow(#[from] moq_net::TimeOverflow),
#[error("no traf in moof")]
NoTraf,
#[error("no tfdt in traf")]
NoTfdt,
#[error("PTS overflow")]
PtsOverflow,
#[error("missing moof")]
NoMoof,
#[error("missing mdat")]
NoMdat,
#[error("missing moov")]
NoMoov,
#[error("no tracks in moov")]
NoTracks,
#[error("multiple tracks in moov, use Trak instead")]
MultipleTracks,
#[error("can't synthesize CMAF init for {0}")]
UnsupportedSynthesis(String),
#[error("subtitle tracks are not supported")]
UnsupportedSubtitle,
#[error("unknown track handler: {0:?}")]
UnknownTrackHandler([u8; 4]),
#[error("missing codec")]
MissingCodec,
#[error("multiple codecs")]
MultipleCodecs,
#[error("unknown codec: {0:?}")]
UnknownCodec(mp4_atom::FourCC),
#[error("unsupported codec: {0:?}")]
UnsupportedCodec(Box<mp4_atom::Codec>),
#[error("unsupported codec: MPEG2")]
UnsupportedMpeg2,
#[error("duplicate moof")]
DuplicateMoof,
#[error("missing trun")]
MissingTrun,
#[error("missing tfdt")]
MissingTfdt,
#[error("video codec {0} needs a description (codec config record) to synthesize a CMAF init")]
MissingVideoDescription(String),
#[error("video track {0} missing in catalog")]
MissingVideoTrack(String),
#[error("audio track {0} missing in catalog")]
MissingAudioTrack(String),
#[error("invalid data offset")]
InvalidDataOffset,
#[error("unknown track {0}")]
UnknownTrack(u32),
#[error("no keyframe at start of group")]
NoKeyframe,
#[error("track sample range {start}..{end} is out of bounds of mdat (len {len})")]
SampleRangeOutOfBounds { start: usize, end: usize, len: usize },
#[error("no catalog snapshot")]
NoCatalogSnapshot,
#[error("encode_fragment called with no frames")]
NoFrames,
#[error("audio codec {0} needs a description (AudioSpecificConfig) to synthesize a CMAF init")]
MissingAudioDescription(String),
#[error("multi-sample fragment has a non-final sample with no duration; DTS is unrecoverable")]
MissingSampleDuration,
}
impl From<mp4_atom::Error> for Error {
fn from(err: mp4_atom::Error) -> Self {
Error::Mp4(std::sync::Arc::new(err))
}
}
pub type Result<T> = std::result::Result<T, Error>;
pub struct Wire {
trak: Box<mp4_atom::Trak>,
}
impl Wire {
pub fn new(trak: mp4_atom::Trak) -> Self {
Self { trak: Box::new(trak) }
}
pub fn from_init(init_data: &[u8]) -> Result<Self> {
use mp4_atom::DecodeMaybe;
let mut cursor = std::io::Cursor::new(init_data);
while let Some(atom) = mp4_atom::Any::decode_maybe(&mut cursor)? {
if let mp4_atom::Any::Moov(mut moov) = atom {
return match moov.trak.len() {
1 => Ok(Self::new(moov.trak.remove(0))),
0 => Err(Error::NoTracks),
_ => Err(Error::MultipleTracks),
};
}
}
Err(Error::NoMoov)
}
pub fn trak(&self) -> &mp4_atom::Trak {
&self.trak
}
}
impl Container for Wire {
type Error = Error;
fn write(&self, group: &mut moq_net::GroupProducer, frames: &[Frame]) -> std::result::Result<(), Self::Error> {
let timescale = self.trak.mdia.mdhd.timescale as u64;
let track_id = self.trak.tkhd.track_id;
encode(group, frames, timescale, track_id)
}
fn poll_read(
&self,
group: &mut moq_net::GroupConsumer,
waiter: &kio::Waiter,
) -> Poll<std::result::Result<crate::container::Read, Self::Error>> {
use std::task::ready;
let Some(data) = ready!(group.poll_read_frame(waiter)?) else {
return Poll::Ready(Ok(crate::container::Read::Done));
};
let timescale = self.trak.mdia.mdhd.timescale as u64;
Poll::Ready(Ok(crate::container::Read::Fragment(decode(data, timescale)?)))
}
}
pub(crate) fn decode(data: Bytes, timescale: u64) -> Result<Vec<Frame>> {
use mp4_atom::DecodeMaybe;
let mut cursor = std::io::Cursor::new(&data);
let mut moof = None;
let mut mdat_data = None;
while let Some(atom) = mp4_atom::Any::decode_maybe(&mut cursor)? {
match atom {
mp4_atom::Any::Moof(m) => moof = Some(m),
mp4_atom::Any::Mdat(m) => mdat_data = Some(m.data),
_ => {}
}
}
let moof = moof.ok_or(Error::NoMoof)?;
let mdat_data = mdat_data.ok_or(Error::NoMdat)?;
let traf = moof.traf.first().ok_or(Error::NoTraf)?;
let tfdt = traf.tfdt.as_ref().ok_or(Error::NoTfdt)?;
let base_dts = tfdt.base_media_decode_time;
let default_size = traf.tfhd.default_sample_size;
let default_duration = traf.tfhd.default_sample_duration;
let total_samples: usize = traf.trun.iter().map(|t| t.entries.len()).sum();
let mut frames = Vec::new();
let mut offset = 0usize;
let mut dts = base_dts;
let mut sample_index = 0usize;
for trun in &traf.trun {
for entry in &trun.entries {
let size = entry.size.or(default_size).unwrap_or(0) as usize;
let end = offset + size;
if end > mdat_data.len() {
return Err(Error::SampleRangeOutOfBounds {
start: offset,
end,
len: mdat_data.len(),
});
}
let cts = entry.cts.unwrap_or_default() as i64;
let pts = dts.checked_add_signed(cts).ok_or(Error::PtsOverflow)?;
let timestamp = Timestamp::from_scale(pts, timescale)?;
let payload = Bytes::copy_from_slice(&mdat_data[offset..end]);
let flags = entry.flags.unwrap_or(0);
let keyframe = (flags >> 24) & 0x3 == 0x2;
let sample_duration = entry.duration.or(default_duration).filter(|d| *d != 0);
let is_last = sample_index + 1 == total_samples;
if sample_duration.is_none() && !is_last {
return Err(Error::MissingSampleDuration);
}
let duration = sample_duration
.map(|d| Timestamp::from_scale(d as u64, timescale))
.transpose()?;
frames.push(Frame {
timestamp,
payload,
keyframe,
duration,
});
offset = end;
dts += sample_duration.unwrap_or(0) as u64;
sample_index += 1;
}
}
Ok(frames)
}
pub(crate) fn encode(
group: &mut moq_net::GroupProducer,
frames: &[Frame],
timescale: u64,
track_id: u32,
) -> Result<()> {
if frames.is_empty() {
return Ok(());
}
let sequence_number = group.frame_count() as u32;
let bytes = encode_fragment(track_id, timescale, sequence_number, frames)?;
let mut writer = group.create_frame(moq_net::Frame {
size: bytes.len() as u64,
})?;
writer.write(bytes)?;
writer.finish()?;
Ok(())
}
pub(crate) fn encode_fragment(track_id: u32, timescale: u64, sequence_number: u32, frames: &[Frame]) -> Result<Bytes> {
use mp4_atom::Encode;
if frames.is_empty() {
return Ok(Bytes::new());
}
let base_dts = frames[0].timestamp.as_scale(timescale) as u64;
let mut dts = base_dts;
let entries: Vec<_> = frames
.iter()
.map(|f| {
let flags = if f.keyframe { 0x0200_0000 } else { 0x0001_0000 };
let duration = f.duration.map(|d| d.as_scale(timescale) as u32);
let pts = f.timestamp.as_scale(timescale) as i128;
let cts = pts - i128::from(dts);
let cts = i32::try_from(cts).map_err(|_| Error::PtsOverflow)?;
if let Some(duration) = duration {
dts = dts.checked_add(u64::from(duration)).ok_or(Error::PtsOverflow)?;
}
Ok(mp4_atom::TrunEntry {
duration,
size: Some(f.payload.len() as u32),
flags: Some(flags),
cts: (cts != 0).then_some(cts),
})
})
.collect::<Result<_>>()?;
let mdat_data: Vec<u8> = frames.iter().flat_map(|f| f.payload.iter().copied()).collect();
let build_moof = |data_offset| mp4_atom::Moof {
mfhd: mp4_atom::Mfhd { sequence_number },
traf: vec![mp4_atom::Traf {
tfhd: mp4_atom::Tfhd {
track_id,
..Default::default()
},
tfdt: Some(mp4_atom::Tfdt {
base_media_decode_time: base_dts,
}),
trun: vec![mp4_atom::Trun {
data_offset: Some(data_offset),
entries: entries.clone(),
}],
..Default::default()
}],
};
let mut buf = Vec::new();
build_moof(0).encode(&mut buf)?;
let moof_size = buf.len();
buf.clear();
build_moof((moof_size + 8) as i32).encode(&mut buf)?;
let mdat = mp4_atom::Mdat { data: mdat_data };
mdat.encode(&mut buf)?;
Ok(Bytes::from(buf))
}
pub(crate) fn synthesize_video_trak(
track_id: u32,
timescale: u64,
config: &VideoConfig,
description: Option<&[u8]>,
) -> Result<mp4_atom::Trak> {
let width = config.coded_width.unwrap_or(0) as u16;
let height = config.coded_height.unwrap_or(0) as u16;
let visual = mp4_atom::Visual {
data_reference_index: 1,
width,
height,
..Default::default()
};
let require_description = || description.ok_or_else(|| Error::MissingVideoDescription(config.codec.to_string()));
let sample_entry = match &config.codec {
VideoCodec::H264(_) => {
let mut cursor = std::io::Cursor::new(require_description()?);
let avcc = mp4_atom::Avcc::decode_body(&mut cursor).map_err(Error::from)?;
mp4_atom::Codec::from(mp4_atom::Avc1 {
visual,
avcc,
..Default::default()
})
}
VideoCodec::H265(h265) => {
let mut cursor = std::io::Cursor::new(require_description()?);
let hvcc = mp4_atom::Hvcc::decode_body(&mut cursor).map_err(Error::from)?;
if h265.in_band {
mp4_atom::Codec::from(mp4_atom::Hev1 {
visual,
hvcc,
..Default::default()
})
} else {
mp4_atom::Codec::from(mp4_atom::Hvc1 {
visual,
hvcc,
..Default::default()
})
}
}
VideoCodec::AV1(av1) => mp4_atom::Codec::from(mp4_atom::Av01 {
visual,
av1c: crate::codec::av1::av1c_from_av1(av1),
..Default::default()
}),
VideoCodec::VP8 => mp4_atom::Codec::from(mp4_atom::Vp08 {
visual,
vpcc: crate::codec::vp8::vpcc(),
..Default::default()
}),
VideoCodec::VP9(vp9) => mp4_atom::Codec::from(mp4_atom::Vp09 {
visual,
vpcc: crate::codec::vp9::vpcc(vp9),
..Default::default()
}),
other => return Err(Error::UnsupportedSynthesis(format!("video codec {:?}", other))),
};
Ok(build_video_trak(track_id, timescale, sample_entry, width, height))
}
pub(crate) fn synthesize_audio_trak(track_id: u32, timescale: u64, config: &AudioConfig) -> Result<mp4_atom::Trak> {
use mp4_atom::Decode;
let audio = mp4_atom::Audio {
data_reference_index: 1,
channel_count: config.channel_count as u16,
sample_size: 16,
sample_rate: mp4_atom::FixedPoint::from(config.sample_rate as u16),
};
let sample_entry = match &config.codec {
AudioCodec::Opus => mp4_atom::Codec::from(mp4_atom::Opus {
audio,
dops: mp4_atom::Dops {
output_channel_count: config.channel_count as u8,
pre_skip: 0,
input_sample_rate: config.sample_rate,
output_gain: 0,
},
btrt: None,
}),
AudioCodec::AAC(_) => {
let description = config
.description
.as_ref()
.ok_or_else(|| Error::MissingAudioDescription(config.codec.to_string()))?;
let mut cursor = std::io::Cursor::new(description.as_ref());
let dec_specific = mp4_atom::esds::DecoderSpecific::decode(&mut cursor)?;
let bitrate = config.bitrate.unwrap_or(0) as u32;
mp4_atom::Codec::from(mp4_atom::Mp4a {
audio,
esds: mp4_atom::Esds {
es_desc: mp4_atom::esds::EsDescriptor {
es_id: 0,
dec_config: mp4_atom::esds::DecoderConfig {
object_type_indication: 0x40, stream_type: 0x05, up_stream: 0,
buffer_size_db: Default::default(),
max_bitrate: bitrate,
avg_bitrate: bitrate,
dec_specific,
},
sl_config: Default::default(),
},
},
btrt: None,
taic: None,
})
}
AudioCodec::Flac => {
let description = config
.description
.as_ref()
.ok_or_else(|| Error::MissingAudioDescription(config.codec.to_string()))?;
let info = crate::codec::flac::Config::parse(&mut description.as_ref())?;
let stream_info = mp4_atom::FlacMetadataBlock::StreamInfo {
minimum_block_size: info.min_block_size,
maximum_block_size: info.max_block_size,
minimum_frame_size: info.min_frame_size.min(0xFF_FFFF).try_into().expect("fits in u24"),
maximum_frame_size: info.max_frame_size.min(0xFF_FFFF).try_into().expect("fits in u24"),
sample_rate: info.sample_rate,
num_channels_minus_one: info.channel_count.saturating_sub(1) as u8,
bits_per_sample_minus_one: info.bits_per_sample.saturating_sub(1) as u8,
number_of_interchannel_samples: info.total_samples,
md5_checksum: info.md5.to_vec(),
};
mp4_atom::Codec::from(mp4_atom::Flac {
audio,
dfla: mp4_atom::Dfla {
blocks: vec![stream_info],
},
})
}
other => return Err(Error::UnsupportedSynthesis(format!("audio codec {:?}", other))),
};
Ok(build_audio_trak(track_id, timescale, sample_entry))
}
fn build_video_trak(
track_id: u32,
timescale: u64,
sample_entry: mp4_atom::Codec,
width: u16,
height: u16,
) -> mp4_atom::Trak {
mp4_atom::Trak {
tkhd: mp4_atom::Tkhd {
track_id,
enabled: true,
width: mp4_atom::FixedPoint::from(width),
height: mp4_atom::FixedPoint::from(height),
..Default::default()
},
mdia: build_mdia(timescale, b"vide", true, sample_entry),
..Default::default()
}
}
fn build_audio_trak(track_id: u32, timescale: u64, sample_entry: mp4_atom::Codec) -> mp4_atom::Trak {
mp4_atom::Trak {
tkhd: mp4_atom::Tkhd {
track_id,
enabled: true,
..Default::default()
},
mdia: build_mdia(timescale, b"soun", false, sample_entry),
..Default::default()
}
}
fn build_mdia(timescale: u64, handler: &[u8; 4], is_video: bool, sample_entry: mp4_atom::Codec) -> mp4_atom::Mdia {
mp4_atom::Mdia {
mdhd: mp4_atom::Mdhd {
timescale: timescale as u32,
..Default::default()
},
hdlr: mp4_atom::Hdlr {
handler: mp4_atom::FourCC::new(handler),
name: String::new(),
},
minf: mp4_atom::Minf {
vmhd: is_video.then(mp4_atom::Vmhd::default),
smhd: (!is_video).then(mp4_atom::Smhd::default),
dinf: mp4_atom::Dinf {
dref: mp4_atom::Dref {
urls: vec![mp4_atom::Url::default()],
},
},
stbl: mp4_atom::Stbl {
stsd: mp4_atom::Stsd {
codecs: vec![sample_entry],
},
..Default::default()
},
..Default::default()
},
}
}
pub(crate) fn default_video_timescale(config: &VideoConfig) -> u64 {
if let Some(fps) = config.framerate {
(fps * 1000.0) as u64
} else {
90000
}
}
#[cfg(test)]
mod tests {
use super::*;
fn ts(micros: u64) -> Timestamp {
Timestamp::from_micros(micros).unwrap()
}
#[test]
fn decode_reads_trun_sample_duration() {
use mp4_atom::Encode;
let timescale = 1_000_000;
let moof = mp4_atom::Moof {
mfhd: mp4_atom::Mfhd { sequence_number: 0 },
traf: vec![mp4_atom::Traf {
tfhd: mp4_atom::Tfhd {
track_id: 1,
..Default::default()
},
tfdt: Some(mp4_atom::Tfdt {
base_media_decode_time: 0,
}),
trun: vec![mp4_atom::Trun {
data_offset: Some(0),
entries: vec![
mp4_atom::TrunEntry {
size: Some(2),
duration: Some(33_333),
..Default::default()
},
mp4_atom::TrunEntry {
size: Some(2),
duration: Some(33_333),
..Default::default()
},
],
}],
..Default::default()
}],
};
let mut buf = Vec::new();
moof.encode(&mut buf).unwrap();
mp4_atom::Mdat {
data: vec![0xDE, 0xAD, 0xBE, 0xEF],
}
.encode(&mut buf)
.unwrap();
let frames = decode(Bytes::from(buf), timescale).unwrap();
assert_eq!(frames.len(), 2);
assert_eq!(frames[0].timestamp, ts(0));
assert_eq!(frames[0].duration, Some(ts(33_333)));
assert_eq!(frames[1].timestamp, ts(33_333));
assert_eq!(frames[1].duration, Some(ts(33_333)));
}
#[test]
fn duration_round_trips_through_encode() {
let timescale = 1_000_000;
let input = vec![Frame {
timestamp: ts(0),
payload: Bytes::from_static(&[0xDE, 0xAD]),
keyframe: true,
duration: Some(ts(33_333)),
}];
let fragment = encode_fragment(1, timescale, 0, &input).unwrap();
let frames = decode(fragment, timescale).unwrap();
assert_eq!(frames.len(), 1);
assert_eq!(frames[0].duration, Some(ts(33_333)));
}
#[test]
fn reordered_pts_round_trips_with_cts() {
let timescale = 1_000_000;
let input = vec![
Frame {
timestamp: ts(0),
payload: Bytes::from_static(&[0x00]),
keyframe: true,
duration: Some(ts(33_000)),
},
Frame {
timestamp: ts(99_000),
payload: Bytes::from_static(&[0x01]),
keyframe: false,
duration: Some(ts(33_000)),
},
Frame {
timestamp: ts(33_000),
payload: Bytes::from_static(&[0x02]),
keyframe: false,
duration: Some(ts(33_000)),
},
];
let fragment = encode_fragment(1, timescale, 0, &input).unwrap();
let frames = decode(fragment, timescale).unwrap();
assert_eq!(frames.len(), input.len());
for (actual, expected) in frames.iter().zip(&input) {
assert_eq!(actual.timestamp, expected.timestamp);
assert_eq!(actual.duration, expected.duration);
assert_eq!(actual.payload, expected.payload);
}
}
#[test]
fn decode_without_duration_reports_none() {
let timescale = 90_000;
let frames = vec![Frame {
timestamp: ts(0),
payload: Bytes::from_static(&[0xDE, 0xAD]),
keyframe: true,
duration: None,
}];
let fragment = encode_fragment(1, timescale, 0, &frames).unwrap();
let frames = decode(fragment, timescale).unwrap();
assert_eq!(frames.len(), 1);
assert_eq!(frames[0].duration, None);
}
#[test]
fn decode_zero_duration_reports_none() {
use mp4_atom::Encode;
let timescale = 24_000;
let moof = mp4_atom::Moof {
mfhd: mp4_atom::Mfhd { sequence_number: 0 },
traf: vec![mp4_atom::Traf {
tfhd: mp4_atom::Tfhd {
track_id: 1,
default_sample_duration: Some(0),
default_sample_size: Some(2),
..Default::default()
},
tfdt: Some(mp4_atom::Tfdt {
base_media_decode_time: 2_000,
}),
trun: vec![mp4_atom::Trun {
data_offset: Some(0),
entries: vec![mp4_atom::TrunEntry {
size: None,
duration: None,
..Default::default()
}],
}],
..Default::default()
}],
};
let mut buf = Vec::new();
moof.encode(&mut buf).unwrap();
mp4_atom::Mdat { data: vec![0xDE, 0xAD] }.encode(&mut buf).unwrap();
let frames = decode(Bytes::from(buf), timescale).unwrap();
assert_eq!(frames.len(), 1);
assert_eq!(frames[0].timestamp.as_micros(), 83_333);
assert_eq!(frames[0].duration, None);
}
}