Skip to main content

threers_codec/decode/
webcodecs.rs

1//! What a platform decoder needs from a [`Demuxed`] track that the container
2//! does not spell out: the browser's WebCodecs `VideoDecoder` takes a codec
3//! string (`avc1.640028`, `hvc1.1.6.L120.90`, `av01.0.08M.08`,
4//! `vp09.00.41.08`) and the configuration record as its `description`, and
5//! each chunk marked key or delta.
6//!
7//! All of it is read from the configuration record and the coded samples, so
8//! it is plain Rust, tested natively, and used by threers's wasm player to
9//! hand the browser's hardware decoder the same files the native decoders
10//! read.
11//!
12//! ```no_run
13//! use threers_codec::decode::{demux, webcodecs};
14//!
15//! let track = demux(std::fs::read("clip.mp4").unwrap()).unwrap();
16//! let codec = webcodecs::codec_string(&track).unwrap();
17//! let first_is_key = webcodecs::is_keyframe(&track, 0);
18//! # let _ = (codec, first_is_key);
19//! ```
20
21use super::reader::{vp9_record, Demuxed};
22use crate::mp4_demux::TrackCodec;
23
24/// The RFC 6381 / WebCodecs codec string for `track`, from its configuration
25/// record (or, for VP9 without one, its first frame). `None` when there is
26/// not enough to say: AV1 in IVF, a damaged record.
27pub fn codec_string(track: &Demuxed) -> Option<String> {
28    let c = &track.config;
29    match track.codec {
30        TrackCodec::H264 => {
31            // avcC: version, profile, compatibility, level.
32            let b = c.get(1..4)?;
33            Some(format!("avc1.{:02x}{:02x}{:02x}", b[0], b[1], b[2]))
34        }
35        TrackCodec::Hevc => hevc_string(c),
36        TrackCodec::Av1 if c.len() < 4 => {
37            // No av1C (IVF, some Matroska): the sequence header the first
38            // temporal unit carries says the same.
39            let seq = av1_sequence_header(track.samples.first().map(|_| track.sample(0))?)?;
40            let op = seq.operating_points.first()?;
41            let level = (op.major_level.saturating_sub(2)) * 4 + op.minor_level;
42            let tier = if op.tier != 0 { 'H' } else { 'M' };
43            let depth = [8, 10, 12][seq.hbd.min(2) as usize];
44            Some(format!("av01.{}.{level:02}{tier}.{depth:02}", seq.profile))
45        }
46        TrackCodec::Av1 => {
47            // av1C: marker and version, then seq_profile (3) and
48            // seq_level_idx_0 (5), then tier, high_bitdepth, twelve_bit.
49            let (a, b) = (*c.get(1)?, *c.get(2)?);
50            let (profile, level) = (a >> 5, a & 0x1f);
51            let tier = if b & 0x80 != 0 { 'H' } else { 'M' };
52            let depth = if b & 0x40 == 0 {
53                8
54            } else if b & 0x20 != 0 {
55                12
56            } else {
57                10
58            };
59            Some(format!("av01.{profile}.{level:02}{tier}.{depth:02}"))
60        }
61        TrackCodec::Vp9 => {
62            let record = if c.len() >= 7 {
63                c.clone()
64            } else {
65                vp9_record(
66                    track.samples.first().map(|_| track.sample(0))?,
67                    track.width,
68                    track.height,
69                )?
70            };
71            // vpcC: version and flags, profile, level, bit depth (4 bits).
72            Some(format!(
73                "vp09.{:02}.{:02}.{:02}",
74                record[4],
75                record[5],
76                record[6] >> 4
77            ))
78        }
79    }
80}
81
82/// `hvc1.<space><profile>.<compatibility>.<tier><level>.<constraints>`, as
83/// ISO/IEC 14496-15 Annex E writes it.
84fn hevc_string(c: &[u8]) -> Option<String> {
85    let b = c.get(1..13)?;
86    let space = ["", "A", "B", "C"][(b[0] >> 6) as usize];
87    let tier = if b[0] & 0x20 != 0 { 'H' } else { 'L' };
88    let profile = b[0] & 0x1f;
89    // The 32 compatibility flags, in reverse bit order, as hex.
90    let compat = u32::from_be_bytes([b[1], b[2], b[3], b[4]]).reverse_bits();
91    let level = b[11];
92    let mut s = format!("hvc1.{space}{profile}.{compat:x}.{tier}{level}");
93    // The six constraint bytes, trailing zero bytes left off.
94    let constraints = &b[5..11];
95    let used = constraints
96        .iter()
97        .rposition(|&x| x != 0)
98        .map_or(0, |i| i + 1);
99    for x in &constraints[..used] {
100        s.push_str(&format!(".{x:02x}"));
101    }
102    Some(s)
103}
104
105/// The `description` a WebCodecs `VideoDecoder` is configured with: the
106/// `avcC` or `hvcC` record for H.264 and HEVC, which arrive length-prefixed
107/// and need it; the `av1C` for AV1 where there is one; nothing for VP9.
108pub fn description(track: &Demuxed) -> Option<&[u8]> {
109    match track.codec {
110        TrackCodec::H264 | TrackCodec::Hevc | TrackCodec::Av1 if !track.config.is_empty() => {
111            Some(&track.config)
112        }
113        _ => None,
114    }
115}
116
117/// Whether sample `i` can be decoded on its own: an IDR picture (H.264), an
118/// IRAP one (HEVC), a key frame (VP9), or a temporal unit opening with a
119/// sequence header or a key frame (AV1). WebCodecs wants the first chunk after
120/// a configure or reset to be one, and a decoder can start at any.
121pub fn is_keyframe(track: &Demuxed, i: usize) -> bool {
122    let Some(sample) = track.samples.get(i).map(|_| track.sample(i)) else {
123        return false;
124    };
125    match track.codec {
126        TrackCodec::H264 => {
127            let len = track.config.get(4).map_or(4, |b| (b & 3) as usize + 1);
128            nal_types(sample, len).any(|h| h & 0x1f == 5)
129        }
130        TrackCodec::Hevc => {
131            let len = track.config.get(21).map_or(4, |b| (b & 3) as usize + 1);
132            nal_types(sample, len).any(|h| (16..=23).contains(&((h >> 1) & 0x3f)))
133        }
134        TrackCodec::Vp9 => vp9_key(sample),
135        TrackCodec::Av1 => av1_key(sample),
136    }
137}
138
139/// The first byte of each length-prefixed NAL unit in `sample`.
140fn nal_types(sample: &[u8], len: usize) -> impl Iterator<Item = u8> + '_ {
141    let mut at = 0usize;
142    std::iter::from_fn(move || {
143        let size = sample
144            .get(at..at + len)?
145            .iter()
146            .fold(0usize, |a, &b| a << 8 | b as usize);
147        let header = *sample.get(at + len)?;
148        at += len + size;
149        Some(header)
150    })
151}
152
153/// VP9's uncompressed header: marker, profile, then `show_existing_frame`
154/// and `frame_type` (0 is a key frame).
155fn vp9_key(frame: &[u8]) -> bool {
156    let Some(&b) = frame.first() else {
157        return false;
158    };
159    if b >> 6 != 0b10 {
160        return false;
161    }
162    let profile = ((b >> 5) & 1) | ((b >> 4) & 1) << 1;
163    // Profile 3 has a reserved bit before the rest.
164    let at = if profile == 3 { 3 } else { 2 };
165    let bit = |n: u32| (b >> (5 - at as u32 - n)) & 1;
166    bit(0) == 0 && bit(1) == 0
167}
168
169/// The OBUs of an AV1 temporal unit: `(type, payload)`.
170fn obus(unit: &[u8]) -> impl Iterator<Item = (u8, &[u8])> + '_ {
171    let mut at = 0usize;
172    std::iter::from_fn(move || {
173        let header = *unit.get(at)?;
174        let kind = (header >> 3) & 0xf;
175        at += 1 + (header & 0x04 != 0) as usize;
176        let size = if header & 0x02 != 0 {
177            let mut value = 0usize;
178            let mut shift = 0;
179            loop {
180                let byte = *unit.get(at)?;
181                at += 1;
182                value |= ((byte & 0x7f) as usize) << shift;
183                shift += 7;
184                if byte & 0x80 == 0 || shift > 56 {
185                    break;
186                }
187            }
188            value
189        } else {
190            unit.len().saturating_sub(at)
191        };
192        let payload = unit.get(at..at.checked_add(size)?)?;
193        at += size;
194        Some((kind, payload))
195    })
196}
197
198/// The sequence header in an AV1 temporal unit, parsed by the decoder's own
199/// parser.
200fn av1_sequence_header(unit: &[u8]) -> Option<super::av1::obu::SeqHeader> {
201    let (_, payload) = obus(unit).find(|(kind, _)| *kind == 1)?;
202    let mut bits = super::av1::obu::GetBits::new(payload);
203    super::av1::obu::parse_seq_hdr(&mut bits).ok()
204}
205
206/// An AV1 temporal unit starts a sequence when it carries a sequence header,
207/// which encoders put before every key frame, or a frame header saying
208/// `KEY_FRAME`.
209fn av1_key(unit: &[u8]) -> bool {
210    for (kind, payload) in obus(unit) {
211        match kind {
212            // OBU_SEQUENCE_HEADER.
213            1 => return true,
214            // OBU_FRAME_HEADER, OBU_FRAME: show_existing_frame, frame_type.
215            3 | 6 => {
216                let Some(&b) = payload.first() else {
217                    return false;
218                };
219                return b & 0x80 == 0 && (b >> 5) & 0b11 == 0;
220            }
221            _ => {}
222        }
223    }
224    false
225}
226
227#[cfg(test)]
228mod tests {
229    use super::*;
230
231    fn track(codec: TrackCodec, config: Vec<u8>, samples: &[&[u8]]) -> Demuxed {
232        let mut bytes = Vec::new();
233        let mut list = Vec::new();
234        for (i, s) in samples.iter().enumerate() {
235            let start = bytes.len();
236            bytes.extend_from_slice(s);
237            list.push(super::super::reader::CodedSample {
238                start,
239                end: bytes.len(),
240                pts: i as i64,
241                dts: i as i64,
242                duration: 1,
243                seconds: i as f64 / 30.0,
244            });
245        }
246        Demuxed {
247            codec,
248            config,
249            width: 1920,
250            height: 1080,
251            bytes: bytes.into(),
252            samples: list,
253        }
254    }
255
256    #[test]
257    fn codec_strings_are_read_from_the_records() {
258        // High profile, level 4.0.
259        let avc = track(TrackCodec::H264, vec![1, 0x64, 0x00, 0x28, 0xff], &[]);
260        assert_eq!(codec_string(&avc).unwrap(), "avc1.640028");
261        // Main, compatible with Main and Main10, main tier, level 4 (120),
262        // the progressive-source constraint (0x90 = progressive + frame only).
263        let mut hvcc = vec![1, 0x01, 0x60, 0, 0, 0, 0x90, 0, 0, 0, 0, 0, 120];
264        hvcc.resize(23, 0);
265        let hevc = track(TrackCodec::Hevc, hvcc, &[]);
266        assert_eq!(codec_string(&hevc).unwrap(), "hvc1.1.6.L120.90");
267        // Main profile, level 3.0 (8), main tier, 8-bit; and 10-bit high tier.
268        let av1 = track(TrackCodec::Av1, vec![0x81, 0x08, 0x0c, 0], &[]);
269        assert_eq!(codec_string(&av1).unwrap(), "av01.0.08M.08");
270        let av1 = track(TrackCodec::Av1, vec![0x81, 0x2d, 0xc0, 0], &[]);
271        assert_eq!(codec_string(&av1).unwrap(), "av01.1.13H.10");
272        // VP9 from a vpcC, and from a profile 0 key frame without one.
273        let vp9 = track(
274            TrackCodec::Vp9,
275            vec![1, 0, 0, 0, 2, 31, 0xa2, 2, 2, 2, 0, 0],
276            &[],
277        );
278        assert_eq!(codec_string(&vp9).unwrap(), "vp09.02.31.10");
279        let vp9 = track(TrackCodec::Vp9, Vec::new(), &[&[0x82, 0x49, 0x83, 0x42]]);
280        assert_eq!(codec_string(&vp9).unwrap(), "vp09.00.41.08");
281        // AV1 with no record has nothing to go on.
282        assert!(codec_string(&track(TrackCodec::Av1, Vec::new(), &[])).is_none());
283    }
284
285    #[test]
286    fn key_frames_are_found_in_each_codec() {
287        let avcc = vec![1, 0x64, 0, 0x28, 0xff];
288        // An SEI then an IDR slice; then a non-IDR slice.
289        let h264 = track(
290            TrackCodec::H264,
291            avcc,
292            &[
293                &[0, 0, 0, 2, 0x06, 0x05, 0, 0, 0, 2, 0x65, 0x88],
294                &[0, 0, 0, 2, 0x41, 0x9a],
295            ],
296        );
297        assert!(is_keyframe(&h264, 0));
298        assert!(!is_keyframe(&h264, 1));
299
300        let mut hvcc = vec![0u8; 23];
301        hvcc[21] = 0xff;
302        // IDR_W_RADL (19), then TRAIL_R (1).
303        let hevc = track(
304            TrackCodec::Hevc,
305            hvcc,
306            &[&[0, 0, 0, 2, 19 << 1, 1], &[0, 0, 0, 2, 1 << 1, 1]],
307        );
308        assert!(is_keyframe(&hevc, 0));
309        assert!(!is_keyframe(&hevc, 1));
310
311        // Profile 0: marker 10, profile 00, show_existing 0, frame_type 0/1.
312        let vp9 = track(
313            TrackCodec::Vp9,
314            Vec::new(),
315            &[&[0b1000_0000], &[0b1000_0100], &[0b1000_1000]],
316        );
317        assert!(is_keyframe(&vp9, 0));
318        assert!(!is_keyframe(&vp9, 1), "inter frame");
319        assert!(!is_keyframe(&vp9, 2), "show existing frame");
320
321        // A temporal delimiter then a sequence header; then a temporal
322        // delimiter and an inter OBU_FRAME (frame_type 1).
323        let av1 = track(
324            TrackCodec::Av1,
325            Vec::new(),
326            &[
327                &[0x12, 0x00, 0x0a, 0x01, 0x00],
328                &[0x12, 0x00, 0x32, 0x01, 0b0010_0000],
329            ],
330        );
331        assert!(is_keyframe(&av1, 0));
332        assert!(!is_keyframe(&av1, 1));
333    }
334}