Skip to main content

threers_codec/
mp4_demux.rs

1//! Reading an MP4 back apart — the mirror of [`mp4`](super::mp4).
2//!
3//! The crate could already write an MP4 and not read one, which meant any
4//! pipeline that wanted a clip *in* had to shell out to ffmpeg for it. That is
5//! the same trade the `videotoolbox` module exists to avoid on the
6//! way out: a process boundary and a raw pipe, 8 MB a frame at 1080p and 99 MB
7//! at 8K, copied twice, to move pictures the system decoder could have handed
8//! over directly.
9//!
10//! This finds the video track's sample table and reports where every coded
11//! frame lives in the file, along with the parameter sets a decoder needs to
12//! start. It does not decode anything — see
13//! `videotoolbox_decode::VideoToolboxDecoder` for that (macOS, `videotoolbox`
14//! feature).
15//!
16//! Only what a video track needs: `stsd` for the codec and its configuration,
17//! `stts` for durations, `stsz`/`stsc`/`stco` for where the samples are, and
18//! `stss` for which of them a decoder can start on. Edit lists, fragmented MP4
19//! (`moof`) and multiple sample descriptions are refused rather than guessed
20//! at.
21
22/// Which codec a track's samples are in. The two VideoToolbox will decode.
23#[derive(Debug, Clone, Copy, PartialEq, Eq)]
24pub enum TrackCodec {
25    /// `avc1` / `avc3`, configured by an `avcC` box.
26    H264,
27    /// `hvc1` / `hev1`, configured by an `hvcC` box.
28    Hevc,
29    /// VP9 (Matroska `V_VP9`; no decoder configuration).
30    Vp9,
31    /// AV1 (`av01` with an `av1C` box, Matroska `V_AV1`, or IVF).
32    Av1,
33}
34
35/// One coded frame: where it is in the file, when it decodes and when it shows.
36#[derive(Debug, Clone, Copy)]
37pub struct Sample {
38    /// Byte offset from the start of the file.
39    pub offset: u64,
40    /// Size in bytes.
41    pub size: u32,
42    /// In the track's timescale.
43    pub duration: u32,
44    /// Decode timestamp: the running sum of the durations before this one.
45    /// Samples are stored in decode order, so this only ever goes up.
46    pub dts: i64,
47    /// Presentation timestamp — `dts` plus the `ctts` composition offset.
48    ///
49    /// **Not the same order as `dts` when the stream has B-frames**, which is
50    /// most streams: a B-frame is coded after the picture it is displayed
51    /// before. Anything showing these frames has to sort by this and not by
52    /// their position in the file.
53    pub pts: i64,
54    /// A sync sample — a decoder can begin here. Everything is one for
55    /// all-intra footage; for anything else this is what `stss` lists.
56    pub sync: bool,
57}
58
59/// A demuxed video track.
60#[derive(Debug, Clone)]
61pub struct VideoTrack {
62    /// Which decoder this track needs.
63    pub codec: TrackCodec,
64    /// Picture width in pixels.
65    pub width: u32,
66    /// Picture height in pixels.
67    pub height: u32,
68    /// The `avcC` or `hvcC` payload, verbatim — what
69    /// `CMVideoFormatDescriptionCreateFromH264ParameterSets` and its HEVC
70    /// twin want, and the only part of the file a decoder needs before the
71    /// first frame.
72    pub config: Vec<u8>,
73    /// Ticks per second for [`Sample::duration`].
74    pub timescale: u32,
75    /// In decode order — the order they sit in the file, which is the order a
76    /// decoder must be fed them in.
77    pub samples: Vec<Sample>,
78}
79
80impl VideoTrack {
81    /// Indices into [`samples`](Self::samples), in the order the frames are
82    /// meant to be *shown*.
83    ///
84    /// The same as decode order for an all-intra or IPPP stream, and not for
85    /// anything with B-frames. A renderer walking `samples` directly gets the
86    /// pictures shuffled — which looks like the clip stuttering in a short
87    /// repeating cycle, one cycle per mini-GOP.
88    pub fn presentation_order(&self) -> Vec<usize> {
89        let mut idx: Vec<usize> = (0..self.samples.len()).collect();
90        idx.sort_by_key(|&i| self.samples[i].pts);
91        idx
92    }
93
94    /// How far a decoder has to read ahead before the next picture to show is
95    /// available — the most any frame is displayed after something coded later.
96    pub fn max_reorder(&self) -> usize {
97        let order = self.presentation_order();
98        order
99            .iter()
100            .enumerate()
101            .map(|(shown, &decoded)| decoded.saturating_sub(shown))
102            .max()
103            .unwrap_or(0)
104    }
105
106    /// Frames per second, averaged over the track. `None` if it has no
107    /// duration to divide by.
108    pub fn fps(&self) -> Option<f64> {
109        let ticks: u64 = self.samples.iter().map(|s| s.duration as u64).sum();
110        if ticks == 0 || self.samples.is_empty() {
111            return None;
112        }
113        Some(self.samples.len() as f64 * self.timescale as f64 / ticks as f64)
114    }
115}
116
117/// Walk the boxes at `data`, calling `f` with the type and body of each.
118///
119/// Handles the two size escapes: `0` means "to the end of the parent", and `1`
120/// means a 64-bit size follows the type. A box that claims to run past its
121/// parent stops the walk rather than panicking on the slice.
122pub(crate) fn boxes<'a>(data: &'a [u8], mut f: impl FnMut(&[u8; 4], &'a [u8])) {
123    let mut off = 0usize;
124    while off + 8 <= data.len() {
125        let size32 = u32::from_be_bytes(data[off..off + 4].try_into().unwrap()) as usize;
126        let ty: [u8; 4] = data[off + 4..off + 8].try_into().unwrap();
127        let (header, size) = match size32 {
128            0 => (8, data.len() - off),
129            1 => {
130                if off + 16 > data.len() {
131                    return;
132                }
133                let s = u64::from_be_bytes(data[off + 8..off + 16].try_into().unwrap()) as usize;
134                (16, s)
135            }
136            n => (8, n),
137        };
138        if size < header || off + size > data.len() {
139            return;
140        }
141        f(&ty, &data[off + header..off + size]);
142        off += size;
143    }
144}
145
146pub(crate) fn find<'a>(data: &'a [u8], want: &[u8; 4]) -> Option<&'a [u8]> {
147    let mut hit = None;
148    boxes(data, |ty, body| {
149        if ty == want && hit.is_none() {
150            hit = Some(body);
151        }
152    });
153    hit
154}
155
156pub(crate) fn u32at(b: &[u8], i: usize) -> u32 {
157    u32::from_be_bytes(b[i..i + 4].try_into().unwrap())
158}
159
160/// Read the first video track out of an MP4.
161///
162/// `data` is the whole file; sample offsets are relative to it.
163pub fn demux_video(data: &[u8]) -> Result<VideoTrack, String> {
164    if find(data, b"moof").is_some() {
165        return Err("fragmented MP4 (moof) is not supported".into());
166    }
167    let moov = find(data, b"moov").ok_or("no moov box — not an MP4?")?;
168
169    // The first trak whose handler says 'vide'.
170    let mut video_stbl: Option<&[u8]> = None;
171    let mut timescale = 0u32;
172    boxes(moov, |ty, trak| {
173        if ty != b"trak" || video_stbl.is_some() {
174            return;
175        }
176        let Some(mdia) = find(trak, b"mdia") else {
177            return;
178        };
179        let is_video = find(mdia, b"hdlr")
180            .map(|h| h.len() >= 12 && &h[8..12] == b"vide")
181            .unwrap_or(false);
182        if !is_video {
183            return;
184        }
185        if let Some(mdhd) = find(mdia, b"mdhd") {
186            // version 0: creation(4) modification(4) timescale(4); version 1
187            // widens the first two to 8 bytes each.
188            timescale = match mdhd.first() {
189                Some(1) if mdhd.len() >= 24 => u32at(mdhd, 20),
190                _ if mdhd.len() >= 16 => u32at(mdhd, 12),
191                _ => 0,
192            };
193        }
194        video_stbl = find(mdia, b"minf").and_then(|m| find(m, b"stbl"));
195    });
196    let stbl = video_stbl.ok_or("no video track")?;
197    if timescale == 0 {
198        return Err("video track has no timescale".into());
199    }
200
201    // --- stsd: codec, size, and the decoder configuration ------------------
202    let stsd = find(stbl, b"stsd").ok_or("no stsd")?;
203    if stsd.len() < 8 {
204        return Err("stsd too short".into());
205    }
206    if u32at(stsd, 4) != 1 {
207        return Err("more than one sample description is not supported".into());
208    }
209    let entry = &stsd[8..];
210    if entry.len() < 86 {
211        return Err("sample entry too short".into());
212    }
213    let format: [u8; 4] = entry[4..8].try_into().unwrap();
214    let codec = match &format {
215        b"avc1" | b"avc3" => TrackCodec::H264,
216        b"hvc1" | b"hev1" => TrackCodec::Hevc,
217        b"vp09" => TrackCodec::Vp9,
218        b"av01" => TrackCodec::Av1,
219        other => {
220            return Err(format!(
221                "unsupported sample format {:?} — only H.264, HEVC, VP9 and AV1",
222                String::from_utf8_lossy(other)
223            ))
224        }
225    };
226    let width = u16::from_be_bytes(entry[32..34].try_into().unwrap()) as u32;
227    let height = u16::from_be_bytes(entry[34..36].try_into().unwrap()) as u32;
228    // Extensions follow the 78-byte visual sample entry body.
229    let want_cfg: &[u8; 4] = match codec {
230        TrackCodec::H264 => b"avcC",
231        TrackCodec::Hevc => b"hvcC",
232        TrackCodec::Vp9 => b"vpcC",
233        TrackCodec::Av1 => b"av1C",
234    };
235    let config = find(&entry[86..], want_cfg)
236        .ok_or_else(|| {
237            format!(
238                "no {} in the sample entry",
239                String::from_utf8_lossy(want_cfg)
240            )
241        })?
242        .to_vec();
243
244    // --- stts: one duration per sample -------------------------------------
245    let stts = find(stbl, b"stts").ok_or("no stts")?;
246    let mut durations: Vec<u32> = Vec::new();
247    if stts.len() >= 8 {
248        let n = u32at(stts, 4) as usize;
249        for i in 0..n {
250            let o = 8 + i * 8;
251            if o + 8 > stts.len() {
252                break;
253            }
254            let count = u32at(stts, o);
255            let delta = u32at(stts, o + 4);
256            durations.extend(std::iter::repeat_n(delta, count as usize));
257        }
258    }
259
260    // --- ctts: how far each sample's display time is from its decode time ---
261    //
262    // Optional, and its absence means they are the same. Version 1 makes the
263    // offset signed, which is how a stream expresses a picture shown *before*
264    // the frame it was coded after.
265    let mut offsets: Vec<i32> = Vec::new();
266    if let Some(ctts) = find(stbl, b"ctts")
267        && ctts.len() >= 8
268    {
269        let n = u32at(ctts, 4) as usize;
270        for i in 0..n {
271            let o = 8 + i * 8;
272            if o + 8 > ctts.len() {
273                break;
274            }
275            let count = u32at(ctts, o) as usize;
276            let off = u32at(ctts, o + 4) as i32;
277            offsets.extend(std::iter::repeat_n(off, count));
278        }
279    }
280
281    // --- stsz: one size per sample -----------------------------------------
282    let stsz = find(stbl, b"stsz").ok_or("no stsz")?;
283    if stsz.len() < 12 {
284        return Err("stsz too short".into());
285    }
286    let uniform = u32at(stsz, 4);
287    let count = u32at(stsz, 8) as usize;
288    let sizes: Vec<u32> = if uniform != 0 {
289        vec![uniform; count]
290    } else {
291        (0..count)
292            .map(|i| {
293                let o = 12 + i * 4;
294                if o + 4 <= stsz.len() {
295                    u32at(stsz, o)
296                } else {
297                    0
298                }
299            })
300            .collect()
301    };
302
303    // --- stsc + stco: where each sample sits --------------------------------
304    let stsc = find(stbl, b"stsc").ok_or("no stsc")?;
305    let mut runs: Vec<(u32, u32)> = Vec::new(); // (first_chunk, samples_per_chunk)
306    if stsc.len() >= 8 {
307        let n = u32at(stsc, 4) as usize;
308        for i in 0..n {
309            let o = 8 + i * 12;
310            if o + 12 > stsc.len() {
311                break;
312            }
313            runs.push((u32at(stsc, o), u32at(stsc, o + 4)));
314        }
315    }
316    let chunk_offsets: Vec<u64> = if let Some(co64) = find(stbl, b"co64") {
317        let n = u32at(co64, 4) as usize;
318        (0..n)
319            .map(|i| {
320                let o = 8 + i * 8;
321                if o + 8 <= co64.len() {
322                    u64::from_be_bytes(co64[o..o + 8].try_into().unwrap())
323                } else {
324                    0
325                }
326            })
327            .collect()
328    } else {
329        let stco = find(stbl, b"stco").ok_or("no stco or co64")?;
330        let n = u32at(stco, 4) as usize;
331        (0..n)
332            .map(|i| {
333                let o = 8 + i * 4;
334                if o + 4 <= stco.len() {
335                    u32at(stco, o) as u64
336                } else {
337                    0
338                }
339            })
340            .collect()
341    };
342
343    // --- stss: which samples a decoder can start on -------------------------
344    let sync: Option<std::collections::HashSet<u32>> = find(stbl, b"stss").map(|stss| {
345        let n = if stss.len() >= 8 {
346            u32at(stss, 4) as usize
347        } else {
348            0
349        };
350        (0..n)
351            .filter_map(|i| {
352                let o = 8 + i * 4;
353                (o + 4 <= stss.len()).then(|| u32at(stss, o))
354            })
355            .collect()
356    });
357
358    // Walk the chunks, laying samples out consecutively within each.
359    let mut samples = Vec::with_capacity(sizes.len());
360    let mut idx = 0usize;
361    let mut dts = 0i64;
362    for (ci, &chunk_off) in chunk_offsets.iter().enumerate() {
363        let chunk = ci as u32 + 1;
364        // The last run whose first_chunk has been reached governs this chunk.
365        let per = runs
366            .iter()
367            .rev()
368            .find(|(first, _)| *first <= chunk)
369            .map(|(_, n)| *n)
370            .unwrap_or(0);
371        let mut at = chunk_off;
372        for _ in 0..per {
373            if idx >= sizes.len() {
374                break;
375            }
376            let duration = durations.get(idx).copied().unwrap_or(0);
377            samples.push(Sample {
378                offset: at,
379                size: sizes[idx],
380                duration,
381                dts,
382                pts: dts + offsets.get(idx).copied().unwrap_or(0) as i64,
383                // No stss means every sample is a sync sample.
384                sync: sync
385                    .as_ref()
386                    .map(|s| s.contains(&(idx as u32 + 1)))
387                    .unwrap_or(true),
388            });
389            at += sizes[idx] as u64;
390            dts += duration as i64;
391            idx += 1;
392        }
393    }
394    if samples.is_empty() {
395        return Err("the video track has no samples".into());
396    }
397
398    Ok(VideoTrack {
399        codec,
400        width,
401        height,
402        config,
403        timescale,
404        samples,
405    })
406}