use std::collections::VecDeque;
use std::io::{self, BufReader, Read, Seek, SeekFrom};
use crate::error::{Error, Result};
use crate::util::bytes::Reader;
const MAX_MOOV_SIZE: u64 = 64 << 20;
const MAX_MOOF_SIZE: u64 = 16 << 20;
pub const MAX_SAMPLE_SIZE: u32 = 64 << 20;
const MAX_TRACKS: usize = 64;
const MAX_SAMPLES: u64 = 1 << 26;
pub const PCM_PACKET_SAMPLES: u32 = 1024;
pub type FourCc = [u8; 4];
pub fn fourcc_str(f: FourCc) -> String {
if f.iter().all(|b| b.is_ascii_graphic() || *b == b' ') {
f.iter().map(|&b| b as char).collect()
} else {
format!("0x{:08x}", u32::from_be_bytes(f))
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TrackType {
Video,
Audio,
Other(FourCc),
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Codec {
Avc { avcc: Option<Vec<u8>> },
Hevc { hvcc: Vec<u8> },
Aac { asc: Vec<u8> },
Opus { head: Vec<u8> },
Alac { config: Vec<u8> },
Pcm {
bits: u16,
big_endian: bool,
float: bool,
},
Other {
fourcc: FourCc,
object_type: Option<u8>,
},
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Edit {
pub duration: u64,
pub media_time: i64,
pub rate: i32,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Track {
pub id: u32,
pub track_type: TrackType,
pub timescale: u32,
pub duration: u64,
pub fourcc: FourCc,
pub codec: Codec,
pub sample_descriptions: u32,
pub width: u32,
pub height: u32,
pub sample_rate: u32,
pub channels: u16,
pub matrix: [i32; 9],
pub edits: Vec<Edit>,
pub time_offset: i64,
pub sample_count: u64,
pub total_bytes: u64,
pub reorders: bool,
table: SampleTable,
defaults: Defaults,
}
pub const IDENTITY_MATRIX: [i32; 9] = [0x1_0000, 0, 0, 0, 0x1_0000, 0, 0, 0, 0x4000_0000];
impl Track {
pub fn matrix_is_identity(&self) -> bool {
self.matrix[..2] == IDENTITY_MATRIX[..2] && self.matrix[3..5] == IDENTITY_MATRIX[3..5]
}
pub fn media_edits(&self) -> usize {
self.edits.iter().filter(|e| e.media_time != -1).count()
}
pub fn duration_secs(&self) -> Option<f64> {
(self.timescale > 0 && self.duration > 0)
.then(|| self.duration as f64 / self.timescale as f64)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Sample {
pub track_id: u32,
pub dts: i64,
pub pts: i64,
pub duration: u32,
pub keyframe: bool,
pub pos: u64,
pub data: Vec<u8>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
struct SampleTable {
stts: Vec<(u32, u32)>,
ctts: Vec<(u32, i32)>,
stss: Option<Vec<u32>>,
sizes: Sizes,
stsc: Vec<(u32, u32)>,
chunks: Vec<u64>,
group: u32,
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum Sizes {
Constant { size: u32, count: u32 },
Table(Vec<u32>),
}
impl Default for Sizes {
fn default() -> Sizes {
Sizes::Constant { size: 0, count: 0 }
}
}
impl Sizes {
fn count(&self) -> u32 {
match self {
Sizes::Constant { count, .. } => *count,
Sizes::Table(t) => t.len() as u32,
}
}
fn get(&self, i: u32) -> u32 {
match self {
Sizes::Constant { size, .. } => *size,
Sizes::Table(t) => t.get(i as usize).copied().unwrap_or(0),
}
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
struct Defaults {
duration: u32,
size: u32,
flags: u32,
}
#[derive(Debug, Clone, Default)]
struct Cursor {
sample: u32,
stts_i: usize,
stts_left: u32,
last_delta: u32,
ctts_i: usize,
ctts_left: u32,
stsc_i: usize,
chunk: usize,
in_chunk: u32,
per_chunk: u32,
offset: u64,
stss_i: usize,
dts: i64,
started: bool,
done: bool,
}
#[derive(Debug, Clone, Copy)]
struct SampleRef {
track: usize,
dts: i64,
pts: i64,
duration: u32,
keyframe: bool,
pos: u64,
size: u32,
}
impl Cursor {
fn enter_chunk(&mut self, t: &SampleTable) -> bool {
while self.chunk < t.chunks.len() {
let c = self.chunk as u64 + 1;
while self.stsc_i + 1 < t.stsc.len() && c >= t.stsc[self.stsc_i + 1].0 as u64 {
self.stsc_i += 1;
}
self.per_chunk = t.stsc.get(self.stsc_i).map_or(0, |e| e.1);
if t.stsc.first().is_some_and(|e| c >= e.0 as u64) && self.per_chunk > 0 {
self.offset = t.chunks[self.chunk];
self.in_chunk = 0;
return true;
}
self.chunk += 1;
}
false
}
fn next(&mut self, t: &SampleTable, track: usize) -> Option<SampleRef> {
if self.done || self.sample >= t.sizes.count() {
self.done = true;
return None;
}
if !self.started {
self.started = true;
if !self.enter_chunk(t) {
self.done = true;
return None;
}
} else if self.in_chunk >= self.per_chunk {
self.chunk += 1;
if !self.enter_chunk(t) {
self.done = true;
return None;
}
}
while self.stts_left == 0 && self.stts_i < t.stts.len() {
self.stts_left = t.stts[self.stts_i].0;
self.last_delta = t.stts[self.stts_i].1;
self.stts_i += 1;
}
let delta = self.last_delta;
self.stts_left = self.stts_left.saturating_sub(1);
while self.ctts_left == 0 && self.ctts_i < t.ctts.len() {
self.ctts_left = t.ctts[self.ctts_i].0;
self.ctts_i += 1;
}
let cts = if self.ctts_left > 0 {
self.ctts_left -= 1;
t.ctts[self.ctts_i - 1].1 as i64
} else {
0
};
let number = self.sample + 1;
let keyframe = match &t.stss {
None => true,
Some(list) => {
while self.stss_i < list.len() && list[self.stss_i] < number {
self.stss_i += 1;
}
list.get(self.stss_i) == Some(&number)
}
};
let size = t.sizes.get(self.sample);
let r = SampleRef {
track,
dts: self.dts,
pts: self.dts.saturating_add(cts),
duration: delta,
keyframe,
pos: self.offset,
size,
};
self.dts = self.dts.saturating_add(delta as i64);
self.offset = self.offset.saturating_add(size as u64);
self.in_chunk += 1;
self.sample += 1;
Some(r)
}
fn continues_chunk(&self, t: &SampleTable) -> bool {
self.started
&& !self.done
&& self.in_chunk < self.per_chunk
&& self.sample < t.sizes.count()
}
}
fn boxes(data: &[u8]) -> impl Iterator<Item = Result<(FourCc, &[u8])>> + '_ {
let mut r = Reader::new(data, "mp4");
let mut failed = false;
std::iter::from_fn(move || {
if failed || r.remaining() < 8 {
return None;
}
let rv = (|| {
let start = r.position();
let size = r.u32_be()? as u64;
let typ: FourCc = r.array()?;
let size = match size {
0 => (r.len() - start) as u64,
1 => r.u64_be()?,
n => n,
};
let header = (r.position() - start) as u64;
if size < header {
return Err(r.error("box size smaller than its header"));
}
let body = usize::try_from(size - header).map_err(|_| r.error("box too large"))?;
Ok((typ, r.bytes(body)?))
})();
failed = rv.is_err();
Some(rv)
})
}
fn child<'a>(data: &'a [u8], typ: &FourCc) -> Result<Option<&'a [u8]>> {
for b in boxes(data) {
let (t, body) = b?;
if &t == typ {
return Ok(Some(body));
}
}
Ok(None)
}
fn full_box<'a>(body: &'a [u8], what: &'static str) -> Result<(u8, u32, Reader<'a>)> {
let mut r = Reader::new(body, what);
let vf = r.u32_be()?;
Ok(((vf >> 24) as u8, vf & 0x00ff_ffff, r))
}
fn count(r: &mut Reader<'_>, entry_size: usize) -> Result<usize> {
let n = r.u32_be()? as usize;
if n.checked_mul(entry_size).is_none_or(|b| b > r.remaining()) {
return Err(r.error("entry count exceeds box size"));
}
if n as u64 > MAX_SAMPLES {
return Err(r.error("too many entries"));
}
Ok(n)
}
fn parse_mvhd(body: &[u8]) -> Result<u32> {
let (v, _, mut r) = full_box(body, "mp4 mvhd")?;
r.skip(if v == 1 { 16 } else { 8 })?;
r.u32_be()
}
fn parse_elst(body: &[u8]) -> Result<Vec<Edit>> {
let (v, _, mut r) = full_box(body, "mp4 elst")?;
let n = count(&mut r, if v == 1 { 20 } else { 12 })?;
let mut out = Vec::with_capacity(n);
for _ in 0..n {
let (duration, media_time) = if v == 1 {
(r.u64_be()?, r.i64_be()?)
} else {
(r.u32_be()? as u64, r.i32_be()? as i64)
};
let rate = r.i32_be()?;
out.push(Edit {
duration,
media_time,
rate,
});
}
Ok(out)
}
fn edit_offset(edits: &[Edit], movie_timescale: u32, timescale: u32) -> i64 {
let mut empty: u64 = 0;
for e in edits {
if e.media_time == -1 {
empty = empty.saturating_add(e.duration);
continue;
}
let empty_ticks = if movie_timescale == 0 {
0
} else {
(empty as u128 * timescale as u128 / movie_timescale as u128).min(i64::MAX as u128)
as i64
};
return empty_ticks.saturating_sub(e.media_time);
}
0
}
fn descriptor<'a>(r: &mut Reader<'a>) -> Result<(u8, Reader<'a>)> {
let tag = r.u8()?;
let mut len = 0usize;
for _ in 0..4 {
let b = r.u8()?;
len = (len << 7) | (b & 0x7f) as usize;
if b & 0x80 == 0 {
break;
}
}
Ok((tag, r.sub(len)?))
}
fn parse_esds(body: &[u8]) -> Result<(u8, Vec<u8>)> {
let (_, _, mut r) = full_box(body, "mp4 esds")?;
let (tag, mut es) = descriptor(&mut r)?;
if tag != 0x03 {
return Err(es.error("esds without ES_Descriptor"));
}
es.skip(2)?; let flags = es.u8()?;
if flags & 0x80 != 0 {
es.skip(2)?; }
if flags & 0x40 != 0 {
let n = es.u8()? as usize;
es.skip(n)?; }
if flags & 0x20 != 0 {
es.skip(2)?; }
while !es.is_at_end() {
let (tag, mut dc) = descriptor(&mut es)?;
if tag != 0x04 {
continue;
}
let object_type = dc.u8()?;
dc.skip(12)?; while !dc.is_at_end() {
let (tag, dsi) = descriptor(&mut dc)?;
if tag == 0x05 {
return Ok((object_type, dsi.rest().to_vec()));
}
}
return Ok((object_type, Vec::new()));
}
Err(Error::malformed("mp4 esds: no DecoderConfigDescriptor"))
}
fn dops_to_opus_head(body: &[u8]) -> Result<Vec<u8>> {
let mut r = Reader::new(body, "mp4 dOps");
if r.u8()? != 0 {
return Err(r.error("unsupported dOps version"));
}
let channels = r.u8()?;
let pre_skip = r.u16_be()?;
let rate = r.u32_be()?;
let gain = r.i16_be()?;
let family = r.u8()?;
let mut head = Vec::with_capacity(19 + 2 + channels as usize);
head.extend_from_slice(b"OpusHead");
head.push(1);
head.push(channels);
head.extend_from_slice(&pre_skip.to_le_bytes());
head.extend_from_slice(&rate.to_le_bytes());
head.extend_from_slice(&gain.to_le_bytes());
head.push(family);
if family != 0 {
let streams = r.u8()?;
let coupled = r.u8()?;
head.push(streams);
head.push(coupled);
head.extend_from_slice(r.bytes(channels as usize)?);
}
Ok(head)
}
fn audio_child<'a>(data: &'a [u8], typ: &FourCc) -> Result<Option<&'a [u8]>> {
if let Some(b) = child(data, typ)? {
return Ok(Some(b));
}
match child(data, b"wave")? {
Some(wave) => child(wave, typ),
None => Ok(None),
}
}
struct Entry {
fourcc: FourCc,
codec: Codec,
width: u32,
height: u32,
sample_rate: u32,
channels: u16,
}
fn parse_stsd(body: &[u8], track_type: TrackType) -> Result<(Entry, u32)> {
let (_, _, mut r) = full_box(body, "mp4 stsd")?;
let n = r.u32_be()?;
let (fourcc, entry) = boxes(r.rest())
.next()
.ok_or_else(|| Error::malformed("mp4 stsd: no sample entry"))??;
let mut e = Entry {
fourcc,
codec: Codec::Other {
fourcc,
object_type: None,
},
width: 0,
height: 0,
sample_rate: 0,
channels: 0,
};
let mut r = Reader::new(entry, "mp4 sample entry");
r.skip(8)?; match track_type {
TrackType::Video => {
r.skip(16)?;
e.width = r.u16_be()? as u32;
e.height = r.u16_be()? as u32;
r.skip(50)?;
let kids = r.rest();
e.codec = match &fourcc {
b"avc1" | b"avc2" | b"avc3" | b"avc4" => Codec::Avc {
avcc: child(kids, b"avcC")?.map(<[u8]>::to_vec),
},
b"hvc1" | b"hev1" => Codec::Hevc {
hvcc: child(kids, b"hvcC")?
.ok_or_else(|| Error::malformed("mp4: HEVC sample entry without hvcC"))?
.to_vec(),
},
_ => e.codec,
};
}
TrackType::Audio => {
let version = r.u16_be()?;
r.skip(6)?; e.channels = r.u16_be()?;
let bits = r.u16_be()?;
r.skip(4)?; e.sample_rate = r.u32_be()? >> 16;
let mut flags = 0u32;
let mut bits = bits;
match version {
0 => {}
1 => r.skip(16)?,
2 => {
r.skip(4)?; let rate = f64::from_bits(r.u64_be()?);
e.sample_rate = if rate.is_finite() && (1.0..=1e7).contains(&rate) {
rate.round() as u32
} else {
0
};
e.channels = u16::try_from(r.u32_be()?).unwrap_or(0);
r.skip(4)?; bits = u16::try_from(r.u32_be()?).unwrap_or(0);
flags = r.u32_be()?;
r.skip(8)?;
}
_ => return Err(r.error("unknown sound sample description version")),
}
let kids = r.rest();
e.codec = match &fourcc {
b"mp4a" => match audio_child(kids, b"esds")? {
Some(esds) => {
let (ot, dsi) = parse_esds(esds)?;
match ot {
0x40 | 0x66 | 0x67 | 0x68 => Codec::Aac { asc: dsi },
_ => Codec::Other {
fourcc,
object_type: Some(ot),
},
}
}
None => e.codec,
},
b"Opus" => Codec::Opus {
head: dops_to_opus_head(
child(kids, b"dOps")?
.ok_or_else(|| Error::malformed("mp4: Opus entry without dOps"))?,
)?,
},
b"alac" => {
let cfg = audio_child(kids, b"alac")?
.ok_or_else(|| Error::malformed("mp4: ALAC entry without config"))?;
let (_, _, cr) = full_box(cfg, "mp4 alac")?;
if cr.remaining() < 24 {
return Err(Error::malformed("mp4: ALAC config too short"));
}
Codec::Alac {
config: cr.rest().to_vec(),
}
}
b"sowt" => Codec::Pcm {
bits,
big_endian: false,
float: false,
},
b"twos" => Codec::Pcm {
bits,
big_endian: true,
float: false,
},
b"lpcm" => Codec::Pcm {
bits,
big_endian: flags & 2 != 0,
float: flags & 1 != 0,
},
_ => e.codec,
};
}
TrackType::Other(_) => {}
}
Ok((e, n))
}
fn parse_stbl(body: &[u8], track_type: TrackType) -> Result<(Entry, u32, SampleTable)> {
let mut t = SampleTable::default();
let mut entry = None;
let mut have_sizes = false;
for b in boxes(body) {
let (typ, body) = b?;
match &typ {
b"stsd" => entry = Some(parse_stsd(body, track_type)?),
b"stts" => {
let (_, _, mut r) = full_box(body, "mp4 stts")?;
let n = count(&mut r, 8)?;
t.stts = (0..n)
.map(|_| Ok((r.u32_be()?, r.u32_be()?)))
.collect::<Result<_>>()?;
}
b"ctts" => {
let (_, _, mut r) = full_box(body, "mp4 ctts")?;
let n = count(&mut r, 8)?;
t.ctts = (0..n)
.map(|_| Ok((r.u32_be()?, r.i32_be()?)))
.collect::<Result<_>>()?;
}
b"stss" => {
let (_, _, mut r) = full_box(body, "mp4 stss")?;
let n = count(&mut r, 4)?;
t.stss = Some((0..n).map(|_| r.u32_be()).collect::<Result<_>>()?);
}
b"stsz" => {
let (_, _, mut r) = full_box(body, "mp4 stsz")?;
let size = r.u32_be()?;
if size != 0 {
let count = r.u32_be()?;
if count as u64 > MAX_SAMPLES {
return Err(r.error("too many samples"));
}
t.sizes = Sizes::Constant { size, count };
} else {
let n = count(&mut r, 4)?;
t.sizes = Sizes::Table((0..n).map(|_| r.u32_be()).collect::<Result<_>>()?);
}
have_sizes = true;
}
b"stz2" => {
let (_, _, mut r) = full_box(body, "mp4 stz2")?;
r.skip(3)?;
let field = r.u8()?;
let n = r.u32_be()? as usize;
let bytes = match field {
4 => n.div_ceil(2),
8 => n,
16 => n.saturating_mul(2),
_ => return Err(r.error("invalid stz2 field size")),
};
if bytes > r.remaining() || n as u64 > MAX_SAMPLES {
return Err(r.error("entry count exceeds box size"));
}
let data = r.bytes(bytes)?;
let sizes = (0..n)
.map(|i| match field {
4 => ((data[i / 2] >> if i % 2 == 0 { 4 } else { 0 }) & 0x0f) as u32,
8 => data[i] as u32,
_ => u16::from_be_bytes([data[2 * i], data[2 * i + 1]]) as u32,
})
.collect();
t.sizes = Sizes::Table(sizes);
have_sizes = true;
}
b"stsc" => {
let (_, _, mut r) = full_box(body, "mp4 stsc")?;
let n = count(&mut r, 12)?;
let mut prev = 0;
for _ in 0..n {
let first = r.u32_be()?;
let per = r.u32_be()?;
r.skip(4)?; if first == 0 || first < prev {
return Err(r.error("stsc first_chunk not increasing"));
}
prev = first;
t.stsc.push((first, per));
}
}
b"stco" => {
let (_, _, mut r) = full_box(body, "mp4 stco")?;
let n = count(&mut r, 4)?;
t.chunks = (0..n)
.map(|_| r.u32_be().map(u64::from))
.collect::<Result<_>>()?;
}
b"co64" => {
let (_, _, mut r) = full_box(body, "mp4 co64")?;
let n = count(&mut r, 8)?;
t.chunks = (0..n).map(|_| r.u64_be()).collect::<Result<_>>()?;
}
_ => {}
}
}
let (entry, n) = entry.ok_or_else(|| Error::malformed("mp4: stbl without stsd"))?;
if !have_sizes && !t.chunks.is_empty() {
return Err(Error::malformed("mp4: stbl without stsz"));
}
Ok((entry, n, t))
}
fn scan_table(t: &SampleTable) -> (u64, bool) {
let mut total = 0u64;
match &t.sizes {
Sizes::Constant { size, count } => total = *size as u64 * *count as u64,
Sizes::Table(s) => {
for &n in s {
total += n as u64;
}
}
}
if t.ctts.is_empty() {
return (total, false);
}
let mut c = Cursor::default();
let mut last: Option<i64> = None;
let mut reorders = false;
while let Some(s) = c.next(t, 0) {
if last.is_some_and(|l| s.pts <= l) {
reorders = true;
break;
}
last = Some(s.pts);
}
(total, reorders)
}
fn parse_trak(body: &[u8], movie_timescale: u32) -> Result<Option<Track>> {
let mut id = 0;
let mut edits = Vec::new();
let mut mdia = None;
let mut matrix = IDENTITY_MATRIX;
for b in boxes(body) {
let (typ, body) = b?;
match &typ {
b"tkhd" => {
let (v, _, mut r) = full_box(body, "mp4 tkhd")?;
r.skip(if v == 1 { 16 } else { 8 })?;
id = r.u32_be()?;
r.skip(if v == 1 { 4 + 8 } else { 4 + 4 })?;
r.skip(16)?;
for m in matrix.iter_mut() {
*m = r.i32_be()?;
}
}
b"edts" => {
if let Some(elst) = child(body, b"elst")? {
edits = parse_elst(elst)?;
}
}
b"mdia" => mdia = Some(body),
_ => {}
}
}
let mdia = mdia.ok_or_else(|| Error::malformed("mp4: trak without mdia"))?;
let (mut timescale, mut duration, mut handler) = (0, 0, *b"\0\0\0\0");
let mut stbl = None;
for b in boxes(mdia) {
let (typ, body) = b?;
match &typ {
b"mdhd" => {
let (v, _, mut r) = full_box(body, "mp4 mdhd")?;
if v == 1 {
r.skip(16)?;
timescale = r.u32_be()?;
duration = r.u64_be()?;
} else {
r.skip(8)?;
timescale = r.u32_be()?;
duration = r.u32_be()? as u64;
}
}
b"hdlr" => {
let (_, _, mut r) = full_box(body, "mp4 hdlr")?;
r.skip(4)?;
handler = r.array()?;
}
b"minf" => stbl = child(body, b"stbl")?,
_ => {}
}
}
let track_type = match &handler {
b"vide" => TrackType::Video,
b"soun" => TrackType::Audio,
_ => TrackType::Other(handler),
};
if matches!(track_type, TrackType::Other(_)) {
return Ok(None);
}
let stbl = stbl.ok_or_else(|| Error::malformed("mp4: track without stbl"))?;
if timescale == 0 {
return Err(Error::malformed("mp4: track timescale is zero"));
}
let (entry, descriptions, mut table) = parse_stbl(stbl, track_type)?;
if let Codec::Pcm { .. } = entry.codec {
table.group = PCM_PACKET_SAMPLES;
}
let (total_bytes, reorders) = scan_table(&table);
Ok(Some(Track {
id,
track_type,
timescale,
duration,
fourcc: entry.fourcc,
codec: entry.codec,
sample_descriptions: descriptions,
width: entry.width,
height: entry.height,
sample_rate: entry.sample_rate,
channels: entry.channels,
matrix,
time_offset: edit_offset(&edits, movie_timescale, timescale),
edits,
sample_count: table.sizes.count() as u64,
total_bytes,
reorders,
table,
defaults: Defaults::default(),
}))
}
struct Moov {
tracks: Vec<Track>,
fragmented: bool,
}
fn parse_moov(body: &[u8]) -> Result<Moov> {
let movie_timescale = match child(body, b"mvhd")? {
Some(m) => parse_mvhd(m)?,
None => return Err(Error::malformed("mp4: moov without mvhd")),
};
let mut tracks: Vec<Track> = Vec::new();
let mut fragmented = false;
for b in boxes(body) {
let (typ, body) = b?;
match &typ {
b"trak" => {
if let Some(t) = parse_trak(body, movie_timescale)? {
if tracks.iter().any(|o| o.id == t.id) {
return Err(Error::malformed("mp4: duplicate track ID"));
}
if tracks.len() >= MAX_TRACKS {
return Err(Error::unsupported("mp4: too many tracks"));
}
tracks.push(t);
}
}
b"mvex" => {
fragmented = true;
for b in boxes(body) {
let (typ, body) = b?;
if &typ != b"trex" {
continue;
}
let (_, _, mut r) = full_box(body, "mp4 trex")?;
let id = r.u32_be()?;
r.skip(4)?;
let d = Defaults {
duration: r.u32_be()?,
size: r.u32_be()?,
flags: r.u32_be()?,
};
if let Some(t) = tracks.iter_mut().find(|t| t.id == id) {
t.defaults = d;
}
}
}
_ => {}
}
}
Ok(Moov { tracks, fragmented })
}
const NON_SYNC: u32 = 0x0001_0000;
fn parse_moof(
body: &[u8],
moof_start: u64,
tracks: &[Track],
frag_dts: &mut [i64],
) -> Result<Vec<SampleRef>> {
let mut out = Vec::new();
let mut prev_end = moof_start;
let mut first_traf = true;
for b in boxes(body) {
let (typ, traf) = b?;
if &typ != b"traf" {
continue;
}
let tfhd =
child(traf, b"tfhd")?.ok_or_else(|| Error::malformed("mp4: traf without tfhd"))?;
let (_, tf, mut r) = full_box(tfhd, "mp4 tfhd")?;
let id = r.u32_be()?;
let Some(ti) = tracks.iter().position(|t| t.id == id) else {
continue;
};
let mut d = tracks[ti].defaults;
let base = if tf & 0x01 != 0 {
r.u64_be()?
} else if tf & 0x02_0000 != 0 || first_traf {
moof_start
} else {
prev_end
};
first_traf = false;
if tf & 0x02 != 0 {
r.skip(4)?; }
if tf & 0x08 != 0 {
d.duration = r.u32_be()?;
}
if tf & 0x10 != 0 {
d.size = r.u32_be()?;
}
if tf & 0x20 != 0 {
d.flags = r.u32_be()?;
}
if let Some(tfdt) = child(traf, b"tfdt")? {
let (v, _, mut r) = full_box(tfdt, "mp4 tfdt")?;
let t = if v == 1 {
r.u64_be()?
} else {
r.u32_be()? as u64
};
frag_dts[ti] = i64::try_from(t).map_err(|_| Error::malformed("mp4: tfdt too large"))?;
}
let offset = tracks[ti].time_offset;
let mut data_pos = base;
for b in boxes(traf) {
let (typ, trun) = b?;
if &typ != b"trun" {
continue;
}
let (v, fl, mut r) = full_box(trun, "mp4 trun")?;
let n = r.u32_be()? as usize;
if fl & 0x01 != 0 {
let off = r.i32_be()? as i64;
data_pos = (base as i64)
.checked_add(off)
.and_then(|p| u64::try_from(p).ok())
.ok_or_else(|| r.error("data offset out of range"))?;
}
let first_flags = if fl & 0x04 != 0 {
Some(r.u32_be()?)
} else {
None
};
let per = [0x100, 0x200, 0x400, 0x800]
.iter()
.filter(|&&f| fl & f != 0)
.count()
* 4;
if n.saturating_mul(per) > r.remaining()
|| out.len().saturating_add(n) as u64 > MAX_SAMPLES
{
return Err(r.error("trun sample count exceeds box size"));
}
for i in 0..n {
let duration = if fl & 0x100 != 0 {
r.u32_be()?
} else {
d.duration
};
let size = if fl & 0x200 != 0 { r.u32_be()? } else { d.size };
let flags = if fl & 0x400 != 0 {
r.u32_be()?
} else if i == 0 {
first_flags.unwrap_or(d.flags)
} else {
d.flags
};
let cts = if fl & 0x800 != 0 {
let raw = r.u32_be()?;
if v == 0 {
raw as i64
} else {
raw as i32 as i64
}
} else {
0
};
let dts = frag_dts[ti];
out.push(SampleRef {
track: ti,
dts: dts.saturating_add(offset),
pts: dts.saturating_add(cts).saturating_add(offset),
duration,
keyframe: flags & NON_SYNC == 0,
pos: data_pos,
size,
});
frag_dts[ti] = dts.saturating_add(duration as i64);
data_pos = data_pos.saturating_add(size as u64);
}
}
prev_end = data_pos;
}
Ok(out)
}
fn shifted(mut r: SampleRef, offset: i64) -> SampleRef {
r.dts = r.dts.saturating_add(offset);
r.pts = r.pts.saturating_add(offset);
r
}
#[derive(Debug, Clone, Copy)]
struct BoxHeader {
typ: FourCc,
start: u64,
header: u64,
size: u64,
}
pub struct Demuxer<R> {
input: BufReader<R>,
pos: u64,
len: u64,
tracks: Vec<Track>,
fragmented: bool,
cursors: Vec<Cursor>,
frag_scan: u64,
frag_start: u64,
frag_dts: Vec<i64>,
pending: VecDeque<SampleRef>,
major_brand: Option<FourCc>,
}
impl<R: Read + Seek> Demuxer<R> {
pub fn new(inner: R) -> Result<Demuxer<R>> {
let mut input = BufReader::with_capacity(64 * 1024, inner);
let len = input
.seek(SeekFrom::End(0))
.map_err(|e| Error::io("mp4: seek", e))?;
input
.seek(SeekFrom::Start(0))
.map_err(|e| Error::io("mp4: seek", e))?;
let mut d = Demuxer {
input,
pos: 0,
len,
tracks: Vec::new(),
fragmented: false,
cursors: Vec::new(),
frag_scan: 0,
frag_start: 0,
frag_dts: Vec::new(),
pending: VecDeque::new(),
major_brand: None,
};
let mut at = 0;
let mut first = true;
loop {
let Some(h) = d.header_at(at)? else {
return Err(Error::malformed("mp4: no moov box"));
};
if first {
let known = [
b"ftyp", b"moov", b"mdat", b"free", b"skip", b"wide", b"pnot", b"styp",
b"sidx", b"uuid",
];
if !known.contains(&&h.typ) {
return Err(Error::malformed(format!(
"mp4: unexpected first box {}",
fourcc_str(h.typ)
)));
}
first = false;
}
match &h.typ {
b"ftyp" => {
let body = d.read_body(&h, 4096)?;
if body.len() >= 4 {
d.major_brand = Some([body[0], body[1], body[2], body[3]]);
}
}
b"moov" => {
let body = d.read_body(&h, MAX_MOOV_SIZE)?;
let moov = parse_moov(&body)?;
d.tracks = moov.tracks;
d.fragmented = moov.fragmented;
d.frag_start = h.start + h.size;
break;
}
_ => {}
}
at = h.start + h.size;
}
if d.tracks.is_empty() {
return Err(Error::unsupported("mp4: no audio or video tracks"));
}
d.rewind();
Ok(d)
}
pub fn tracks(&self) -> &[Track] {
&self.tracks
}
pub fn track(&self, id: u32) -> Option<&Track> {
self.tracks.iter().find(|t| t.id == id)
}
pub fn is_fragmented(&self) -> bool {
self.fragmented
}
pub fn major_brand(&self) -> Option<FourCc> {
self.major_brand
}
pub fn rewind(&mut self) {
self.cursors = vec![Cursor::default(); self.tracks.len()];
self.frag_dts = vec![0; self.tracks.len()];
self.frag_scan = self.frag_start;
self.pending.clear();
}
pub fn next_sample(&mut self) -> Result<Option<Sample>> {
let Some(r) = self.next_ref()? else {
return Ok(None);
};
if r.size > MAX_SAMPLE_SIZE {
return Err(Error::malformed(format!(
"mp4: sample of {} bytes is too large",
r.size
)));
}
let data = self.read_at(r.pos, r.size as usize)?;
Ok(Some(Sample {
track_id: self.tracks[r.track].id,
dts: r.dts,
pts: r.pts,
duration: r.duration,
keyframe: r.keyframe,
pos: r.pos,
data,
}))
}
fn next_ref(&mut self) -> Result<Option<SampleRef>> {
let mut best: Option<(usize, SampleRef)> = None;
for i in 0..self.tracks.len() {
let Some(r) = self.peek_table(i) else {
continue;
};
let earlier = best.is_none_or(|(bi, b)| {
let (ta, tb) = (self.tracks[i].timescale, self.tracks[bi].timescale);
let (a, b2) = (r.dts as i128 * tb as i128, b.dts as i128 * ta as i128);
a < b2 || (a == b2 && r.pos < b.pos)
});
if earlier {
best = Some((i, r));
}
}
if let Some((i, _)) = best {
return Ok(self.take_table(i));
}
if !self.fragmented {
return Ok(None);
}
loop {
if let Some(r) = self.pending.pop_front() {
return Ok(Some(r));
}
if !self.next_fragment()? {
return Ok(None);
}
}
}
fn peek_table(&self, i: usize) -> Option<SampleRef> {
let mut c = self.cursors[i].clone();
let off = self.tracks[i].time_offset;
c.next(&self.tracks[i].table, i).map(|r| shifted(r, off))
}
fn take_table(&mut self, i: usize) -> Option<SampleRef> {
let t = &self.tracks[i].table;
let c = &mut self.cursors[i];
let mut r = c.next(t, i)?;
let mut frames = 1;
while frames < t.group && c.continues_chunk(t) {
let mut look = c.clone();
let Some(n) = look.next(t, i) else { break };
let size = r.size as u64 + n.size as u64;
if n.pos != r.pos + r.size as u64 || size > MAX_SAMPLE_SIZE as u64 {
break;
}
*c = look;
r.size = size as u32;
r.duration = r.duration.saturating_add(n.duration);
frames += 1;
}
Some(shifted(r, self.tracks[i].time_offset))
}
fn next_fragment(&mut self) -> Result<bool> {
loop {
let Some(h) = self.header_at(self.frag_scan)? else {
return Ok(false);
};
self.frag_scan = h.start + h.size;
if &h.typ != b"moof" {
continue;
}
let body = self.read_body(&h, MAX_MOOF_SIZE)?;
let mut refs = parse_moof(&body, h.start, &self.tracks, &mut self.frag_dts)?;
let tracks = &self.tracks;
refs.sort_by(|a, b| {
let (ta, tb) = (tracks[a.track].timescale, tracks[b.track].timescale);
(a.dts as i128 * tb as i128)
.cmp(&(b.dts as i128 * ta as i128))
.then(a.pos.cmp(&b.pos))
});
self.pending.extend(refs);
if !self.pending.is_empty() {
return Ok(true);
}
}
}
fn seek_to(&mut self, pos: u64) -> Result<()> {
if pos != self.pos {
let delta = pos as i128 - self.pos as i128;
match i64::try_from(delta) {
Ok(d) => self.input.seek_relative(d),
Err(_) => self.input.seek(SeekFrom::Start(pos)).map(|_| ()),
}
.map_err(|e| Error::io("mp4: seek", e))?;
self.pos = pos;
}
Ok(())
}
fn read_at(&mut self, pos: u64, len: usize) -> Result<Vec<u8>> {
if pos.saturating_add(len as u64) > self.len {
return Err(Error::malformed(format!(
"mp4: data at {pos}+{len} past the end of the file ({})",
self.len
)));
}
self.seek_to(pos)?;
let mut buf = vec![0u8; len];
self.input.read_exact(&mut buf).map_err(|e| {
if e.kind() == io::ErrorKind::UnexpectedEof {
Error::malformed("mp4: truncated file")
} else {
Error::io("mp4: read", e)
}
})?;
self.pos += len as u64;
Ok(buf)
}
fn header_at(&mut self, pos: u64) -> Result<Option<BoxHeader>> {
if pos.saturating_add(8) > self.len {
return Ok(None);
}
let head = self.read_at(pos, 8)?;
let size32 = u32::from_be_bytes([head[0], head[1], head[2], head[3]]);
let typ = [head[4], head[5], head[6], head[7]];
let (size, header) = match size32 {
0 => (self.len - pos, 8),
1 => {
if pos + 16 > self.len {
return Err(Error::malformed("mp4: truncated box header"));
}
let b = self.read_at(pos + 8, 8)?;
let mut a = [0u8; 8];
a.copy_from_slice(&b);
(u64::from_be_bytes(a), 16)
}
n => (n as u64, 8),
};
if size < header {
return Err(Error::malformed(format!(
"mp4: box {} at {pos} smaller than its header",
fourcc_str(typ)
)));
}
if pos.saturating_add(size) > self.len {
if &typ != b"mdat" {
return Err(Error::malformed(format!(
"mp4: box {} at {pos} runs past the end of the file",
fourcc_str(typ)
)));
}
return Ok(Some(BoxHeader {
typ,
start: pos,
header,
size: self.len - pos,
}));
}
Ok(Some(BoxHeader {
typ,
start: pos,
header,
size,
}))
}
fn read_body(&mut self, h: &BoxHeader, limit: u64) -> Result<Vec<u8>> {
let n = h.size - h.header;
if n > limit {
return Err(Error::unsupported(format!(
"mp4: {} box of {n} bytes is too large",
fourcc_str(h.typ)
)));
}
self.read_at(h.start + h.header, n as usize)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::util::vectors;
use std::io::Cursor as IoCursor;
struct Expected {
streams: Vec<Vec<String>>,
packets: Vec<Vec<String>>,
}
fn expected(name: &str) -> (String, Expected) {
let blocks = vectors::load(&format!("mp4/{name}.txt"));
let b = &blocks[0];
let streams: Vec<Vec<String>> = b
.str("streams")
.split(';')
.filter(|s| !s.is_empty())
.map(|s| s.split(',').map(str::to_owned).collect())
.collect();
let mut packets = vec![Vec::new(); streams.len()];
for p in b.str("packets").split(';').filter(|p| !p.is_empty()) {
let f: Vec<&str> = p.split(' ').collect();
let idx: usize = f[0].parse().unwrap();
packets[idx].push(f[1..].join(" "));
}
(b.str("file").to_owned(), Expected { streams, packets })
}
fn check(name: &str) -> Demuxer<IoCursor<Vec<u8>>> {
let (file, exp) = expected(name);
let data = vectors::fixture(&format!("media/{file}"));
let mut d = Demuxer::new(IoCursor::new(data)).unwrap();
assert_eq!(d.tracks().len(), exp.streams.len(), "{name}: tracks");
for (t, s) in d.tracks().iter().zip(&exp.streams) {
let tb = s.last().unwrap();
assert_eq!(tb, &format!("1/{}", t.timescale), "{name}: time base");
match t.track_type {
TrackType::Video => {
assert_eq!(s[2], t.width.to_string(), "{name}: width");
assert_eq!(s[3], t.height.to_string(), "{name}: height");
}
TrackType::Audio => {
assert_eq!(s[2], t.sample_rate.to_string(), "{name}: rate");
assert_eq!(s[3], t.channels.to_string(), "{name}: channels");
}
TrackType::Other(_) => panic!("unexpected track"),
}
}
let ids: Vec<u32> = d.tracks().iter().map(|t| t.id).collect();
let mut got = vec![Vec::new(); ids.len()];
let mut last_dts = vec![i64::MIN; ids.len()];
while let Some(s) = d.next_sample().unwrap() {
let i = ids.iter().position(|&id| id == s.track_id).unwrap();
assert!(s.dts > last_dts[i], "{name}: dts not increasing");
last_dts[i] = s.dts;
got[i].push(format!(
"{} {} {} {} {}",
s.pts,
s.dts,
s.duration,
if s.keyframe { "K" } else { "_" },
s.data.len()
));
}
for (i, (g, e)) in got.iter().zip(&exp.packets).enumerate() {
assert_eq!(g.len(), e.len(), "{name}: stream {i} packet count");
for (k, (g, e)) in g.iter().zip(e).enumerate() {
assert_eq!(g, e, "{name}: stream {i} packet {k}");
}
}
d
}
#[test]
fn progressive_no_bframes() {
let d = check("bf0");
let v = &d.tracks()[0];
assert!(matches!(v.codec, Codec::Avc { avcc: Some(_) }));
assert!(!v.reorders);
assert!(!d.is_fragmented());
assert!(v.matrix_is_identity());
assert_eq!(v.media_edits(), 1);
let a = &d.tracks()[1];
let Codec::Aac { asc } = &a.codec else {
panic!("{:?}", a.codec)
};
assert_eq!(asc[0] >> 3, 2, "AAC-LC");
}
#[test]
fn rotated_track() {
let tmp = std::env::temp_dir().join(format!("tvpipe-mp4-rot-{}.mp4", std::process::id()));
let ok = std::process::Command::new("ffmpeg")
.args([
"-hide_banner",
"-loglevel",
"error",
"-y",
"-display_rotation",
"90",
"-i",
])
.arg(vectors::path("tests/fixtures/media/bf0.mp4"))
.args(["-c", "copy"])
.arg(&tmp)
.status()
.is_ok_and(|s| s.success());
if !ok {
return;
}
let d = Demuxer::new(std::fs::File::open(&tmp).unwrap()).unwrap();
let _ = std::fs::remove_file(&tmp);
assert!(!d.tracks()[0].matrix_is_identity());
assert!(d.tracks()[1].matrix_is_identity());
}
#[test]
fn bframes_faststart() {
let d = check("bframes");
assert!(d.tracks()[0].reorders);
assert_eq!(d.tracks()[0].time_offset, -1024);
assert!(!d.tracks()[1].reorders);
}
#[test]
fn fragmented_opus() {
let d = check("frag");
assert!(d.is_fragmented());
let Codec::Opus { head } = &d.tracks()[1].codec else {
panic!()
};
assert!(head.starts_with(b"OpusHead\x01\x02"));
assert_eq!(head.len(), 19);
assert_eq!(u16::from_le_bytes([head[10], head[11]]), 312);
}
#[test]
fn quicktime_alac() {
let d = check("alac");
assert_eq!(d.major_brand(), Some(*b"qt "));
let Codec::Alac { config } = &d.tracks()[1].codec else {
panic!()
};
assert_eq!(config.len(), 24);
assert_eq!(
u32::from_be_bytes([config[0], config[1], config[2], config[3]]),
4096
);
}
#[test]
fn quicktime_pcm() {
let d = check("pcm");
assert_eq!(
d.tracks()[1].codec,
Codec::Pcm {
bits: 16,
big_endian: false,
float: false
}
);
}
#[test]
fn hevc() {
let d = check("hevc");
assert!(matches!(&d.tracks()[0].codec, Codec::Hevc { hvcc } if hvcc[0] == 1));
}
#[test]
fn rewind_repeats() {
let data = vectors::fixture("media/frag.mp4");
let mut d = Demuxer::new(IoCursor::new(data)).unwrap();
let mut first = Vec::new();
while let Some(s) = d.next_sample().unwrap() {
first.push(s);
}
d.rewind();
let mut second = Vec::new();
while let Some(s) = d.next_sample().unwrap() {
second.push(s);
}
assert_eq!(first, second);
}
#[test]
fn edit_offsets() {
let e = |duration, media_time| Edit {
duration,
media_time,
rate: 1 << 16,
};
assert_eq!(edit_offset(&[], 1000, 90_000), 0);
assert_eq!(edit_offset(&[e(1000, 1024)], 1000, 48_000), -1024);
assert_eq!(edit_offset(&[e(100, -1), e(900, 0)], 1000, 90_000), 9000);
}
#[test]
fn opus_head_from_dops() {
let dops = [0, 2, 0x01, 0x38, 0, 0, 0xbb, 0x80, 0xff, 0xfe, 0];
let head = dops_to_opus_head(&dops).unwrap();
assert_eq!(
head,
[
b'O', b'p', b'u', b's', b'H', b'e', b'a', b'd', 1, 2, 0x38, 0x01, 0x80, 0xbb, 0, 0,
0xfe, 0xff, 0
]
);
assert!(dops_to_opus_head(&[1, 2]).is_err());
assert!(dops_to_opus_head(&[0, 2, 0, 0, 0, 0, 0xbb, 0x80, 0, 0, 1, 1]).is_err());
}
#[test]
fn hostile_input_never_panics() {
for name in ["bf0.mp4", "frag.mp4", "alac.mov"] {
let data = vectors::fixture(&format!("media/{name}"));
let run = |bytes: Vec<u8>| {
if let Ok(mut d) = Demuxer::new(IoCursor::new(bytes)) {
for _ in 0..4000 {
if !matches!(d.next_sample(), Ok(Some(_))) {
break;
}
}
}
};
let step = (data.len() / 300).max(1);
for n in (0..data.len()).step_by(step) {
run(data[..n].to_vec());
}
let moov = data
.windows(4)
.position(|w| w == b"moov")
.unwrap()
.saturating_sub(4);
for i in (moov..data.len().min(moov + 2500)).step_by(3) {
let mut bad = data.clone();
bad[i] ^= 0xa5;
run(bad.clone());
bad[i] = 0xff;
run(bad);
}
}
assert!(Demuxer::new(IoCursor::new(b"not an mp4 file".to_vec())).is_err());
assert!(Demuxer::new(IoCursor::new(Vec::new())).is_err());
let mut stsz = vec![0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 0xff, 0xff];
stsz.extend([0; 8]);
let mut r = Reader::new(&stsz[8..], "t");
assert!(count(&mut r, 4).is_err());
}
}