Skip to main content

threers_codec/decode/
mod.rs

1//! Native, dependency-free H.264, HEVC, VP9 and AV1 *decoding* — the software
2//! twin of `videotoolbox_decode::VideoToolboxDecoder`.
3//!
4//! VideoToolbox only exists on Apple platforms, and on the web there is no
5//! system decoder a `wasm32` build can reach synchronously at all. This is
6//! the same job done in pure Rust: no OS calls, no threads, no I/O, so it
7//! builds and runs everywhere the rest of [`crate`] does.
8//!
9//! The decoders need the `decode` feature; an app that only encodes does not
10//! compile them. The bit reader and the H.264 and
11//! HEVC parameter-set parsers (`bits`, `h264::params`, `hevc::params`) are
12//! here either way, because the MP4 muxer reads a stream's SPS to describe
13//! its colour.
14//!
15//! [`NativeDecoder`] has the same shape as the VideoToolbox session, so code
16//! written against one works with the other: open it from a demuxed track,
17//! push samples in decode order, and take pictures back by presentation
18//! time.
19//!
20//! ```no_run
21//! # #[cfg(feature = "decode")] {
22//! use threers_codec::decode::NativeDecoder;
23//! use threers_codec::mp4_demux::demux_video;
24//!
25//! let file = std::fs::read("clip.mp4").unwrap();
26//! let track = demux_video(&file).unwrap();
27//! let mut dec = NativeDecoder::new(&track).unwrap();
28//! for s in &track.samples {
29//!     let bytes = &file[s.offset as usize..][..s.size as usize];
30//!     dec.push(bytes, s.pts, s.dts, s.duration as i64).unwrap();
31//!     if let Some(bgra) = dec.take(s.pts) {
32//!         let _ = bgra; // width * height * 4
33//!     }
34//! }
35//! # }
36//! ```
37//!
38//! [`NativeDecoder::take_frame`] returns the planes instead of BGRA, at the
39//! stream's own bit depth — what a renderer uploading YUV textures wants,
40//! and what the conformance tests compare against a reference decoder
41//! sample for sample.
42//!
43//! What each decoder covers is in its module docs: [`h264`], [`hevc`], `vp9`
44//! and `av1`.
45//! A stream using something outside that is refused with an error naming
46//! the feature, not decoded into garbage.
47
48// The decoders follow the specifications' own notation: loops over an index
49// that addresses several parallel arrays at once (`bs[dir][e][s]`, the two
50// reference lists, the 16 4×4 blocks). Iterator chains would hide that
51// correspondence for no gain.
52#![allow(clippy::needless_range_loop)]
53
54pub mod bits;
55#[cfg(feature = "decode")]
56pub mod cabac;
57pub mod h264;
58pub mod hevc;
59#[cfg(feature = "decode")]
60pub mod av1;
61#[cfg(feature = "decode")]
62pub mod vp9;
63pub mod vp9_packet;
64#[cfg(feature = "decode")]
65pub mod picture;
66#[cfg(feature = "decode")]
67pub mod platform;
68
69#[cfg(feature = "decode")]
70pub use h264::H264Decoder;
71#[cfg(feature = "decode")]
72pub use hevc::HevcDecoder;
73#[cfg(feature = "decode")]
74pub use picture::{ChromaFormat, ColorInfo, DecodedFrame, Matrix, Plane};
75#[cfg(feature = "decode")]
76pub use platform::{Backend, Decoder};
77
78#[cfg(feature = "decode")]
79use crate::mp4_demux::{TrackCodec, VideoTrack};
80
81#[cfg(feature = "decode")]
82enum Codec {
83    H264(Box<H264Decoder>),
84    Hevc(Box<HevcDecoder>),
85    Vp9(Box<vp9::Vp9Decoder>),
86    Av1(Box<av1::Av1Decoder>),
87}
88
89#[cfg(feature = "decode")]
90impl Codec {
91    fn decode_nal(&mut self, nal: &[u8]) -> Result<(), String> {
92        match self {
93            Codec::H264(d) => d.decode_nal(nal),
94            Codec::Hevc(d) => d.decode_nal(nal),
95            Codec::Vp9(_) => Err("VP9 has no NAL units".into()),
96            Codec::Av1(_) => Err("AV1 has no NAL units".into()),
97        }
98    }
99
100    /// End of an access unit: finish the picture in progress and return
101    /// everything completed, in decode order.
102    fn flush(&mut self) -> Vec<DecodedFrame> {
103        match self {
104            Codec::H264(d) => {
105                d.finish_picture();
106                d.take_pictures().into_iter().map(|p| p.frame).collect()
107            }
108            Codec::Hevc(d) => {
109                d.finish_picture();
110                d.take_pictures().into_iter().map(|p| p.frame).collect()
111            }
112            Codec::Vp9(_) | Codec::Av1(_) => Vec::new(),
113        }
114    }
115}
116
117#[cfg(feature = "decode")]
118/// Split an `avcC` payload into `(parameter set NALs, NAL length size)`.
119pub(crate) fn split_avcc(cfg: &[u8]) -> Result<(Vec<&[u8]>, usize), String> {
120    if cfg.len() < 7 {
121        return Err("avcC too short".into());
122    }
123    let nal_len = (cfg[4] & 3) as usize + 1;
124    let mut sets = Vec::new();
125    let mut at = 5;
126    for pass in 0..2 {
127        let count = if pass == 0 {
128            (cfg[at] & 31) as usize
129        } else {
130            cfg.get(at).copied().unwrap_or(0) as usize
131        };
132        at += 1;
133        for _ in 0..count {
134            if at + 2 > cfg.len() {
135                return Err("avcC ended inside a parameter set".into());
136            }
137            let len = u16::from_be_bytes([cfg[at], cfg[at + 1]]) as usize;
138            at += 2;
139            if at + len > cfg.len() {
140                return Err("avcC parameter set runs past the box".into());
141            }
142            sets.push(&cfg[at..at + len]);
143            at += len;
144        }
145        if at >= cfg.len() {
146            break;
147        }
148    }
149    Ok((sets, nal_len))
150}
151
152#[cfg(feature = "decode")]
153/// Split an `hvcC` payload into `(parameter set NALs, NAL length size)`.
154pub(crate) fn split_hvcc(cfg: &[u8]) -> Result<(Vec<&[u8]>, usize), String> {
155    if cfg.len() < 23 {
156        return Err("hvcC too short".into());
157    }
158    let nal_len = (cfg[21] & 3) as usize + 1;
159    let arrays = cfg[22] as usize;
160    let mut sets = Vec::new();
161    let mut at = 23;
162    for _ in 0..arrays {
163        if at + 3 > cfg.len() {
164            return Err("hvcC ended inside an array header".into());
165        }
166        let count = u16::from_be_bytes([cfg[at + 1], cfg[at + 2]]) as usize;
167        at += 3;
168        for _ in 0..count {
169            if at + 2 > cfg.len() {
170                return Err("hvcC ended inside a parameter set".into());
171            }
172            let len = u16::from_be_bytes([cfg[at], cfg[at + 1]]) as usize;
173            at += 2;
174            if at + len > cfg.len() {
175                return Err("hvcC parameter set runs past the box".into());
176            }
177            sets.push(&cfg[at..at + len]);
178            at += len;
179        }
180    }
181    Ok((sets, nal_len))
182}
183
184#[cfg(feature = "decode")]
185/// A software decoder, fed one coded frame at a time — the same interface as
186/// `videotoolbox_decode::VideoToolboxDecoder`.
187pub struct NativeDecoder {
188    codec: Codec,
189    nal_len: usize,
190    width: u32,
191    height: u32,
192    ready: Vec<(i64, DecodedFrame)>,
193}
194
195#[cfg(feature = "decode")]
196impl NativeDecoder {
197    /// Open a decoder for `track`'s codec and configuration.
198    pub fn new(track: &VideoTrack) -> Result<Self, String> {
199        Self::from_config(track.codec, &track.config, track.width, track.height)
200    }
201
202    /// Open a decoder from a decoder configuration alone — an `avcC` or
203    /// `hvcC` payload — for streams that do not come from
204    /// [`demux_video`](crate::mp4_demux::demux_video): RTSP, a
205    /// container of your own. `width` and `height` are what
206    /// [`size`](Self::size) reports.
207    pub fn from_config(codec: TrackCodec, config: &[u8], width: u32, height: u32) -> Result<Self, String> {
208        let (mut codec, (sets, nal_len)) = match codec {
209            TrackCodec::H264 => (
210                Codec::H264(Box::default()),
211                split_avcc(config)?,
212            ),
213            TrackCodec::Hevc => (
214                Codec::Hevc(Box::default()),
215                split_hvcc(config)?,
216            ),
217            TrackCodec::Vp9 => (Codec::Vp9(Box::default()), (Vec::new(), 0)),
218            TrackCodec::Av1 => (Codec::Av1(Box::default()), (Vec::new(), 0)),
219        };
220        for nal in sets {
221            codec.decode_nal(nal)?;
222        }
223        Ok(NativeDecoder {
224            codec,
225            nal_len,
226            width,
227            height,
228            ready: Vec::new(),
229        })
230    }
231
232    /// Size of the track, as the container states it.
233    pub fn size(&self) -> (u32, u32) {
234        (self.width, self.height)
235    }
236
237    /// Hand one coded frame to the decoder, in **decode order**: the bytes
238    /// the demuxer pointed at, length-prefixed NAL units as they sit in the
239    /// file. The picture it carries is decoded before this returns and
240    /// queued under `pts`.
241    pub fn push(&mut self, sample: &[u8], pts: i64, _dts: i64, _duration: i64) -> Result<(), String> {
242        if let Codec::Av1(d) = &mut self.codec {
243            // A temporal unit shows at most one picture.
244            for (i, f) in d.decode(sample)?.into_iter().enumerate() {
245                self.ready.push((pts + i as i64, f.to_decoded()));
246            }
247            return Ok(());
248        }
249        if let Codec::Vp9(d) = &mut self.codec {
250            // A packet (superframe) shows at most one picture.
251            for (i, f) in d.decode(sample)?.into_iter().enumerate() {
252                self.ready.push((pts + i as i64, f.to_decoded()?));
253            }
254            return Ok(());
255        }
256        for nal in bits::split_length_prefixed(sample, self.nal_len)? {
257            self.codec.decode_nal(nal)?;
258        }
259        for (i, frame) in self.codec.flush().into_iter().enumerate() {
260            // One picture per sample is the rule for frame-coded streams; a
261            // second would be a muxing error, and is kept just after it.
262            self.ready.push((pts + i as i64, frame));
263        }
264        Ok(())
265    }
266
267    /// Take the picture for presentation time `pts` as tightly packed BGRA,
268    /// `width * height * 4`, if it has been decoded. Frames before `pts` are
269    /// dropped on the way past: they have been shown.
270    pub fn take(&mut self, pts: i64) -> Option<Vec<u8>> {
271        self.take_frame(pts).map(|f| f.to_bgra())
272    }
273
274    /// [`take`](Self::take), but the decoded planes rather than BGRA.
275    pub fn take_frame(&mut self, pts: i64) -> Option<DecodedFrame> {
276        let hit = self.ready.iter().position(|(p, _)| *p == pts)?;
277        let (_, frame) = self.ready.remove(hit);
278        self.ready.retain(|(p, _)| *p > pts);
279        Some(frame)
280    }
281
282    /// How many decoded pictures are waiting to be taken.
283    pub fn queued(&self) -> usize {
284        self.ready.len()
285    }
286}
287
288#[cfg(feature = "decode")]
289/// Decode a whole Annex B elementary stream (start-code delimited, as `.264`
290/// and `.265` files are) and return its pictures in display order.
291pub fn decode_annexb(codec: TrackCodec, stream: &[u8]) -> Result<Vec<DecodedFrame>, String> {
292    // (starts a new POC period, POC, frame), in decode order.
293    let mut pics: Vec<(bool, i32, DecodedFrame)> = Vec::new();
294    match codec {
295        TrackCodec::H264 => {
296            let mut d = H264Decoder::new();
297            for nal in bits::split_annexb(stream) {
298                d.decode_nal(nal)?;
299            }
300            d.finish_picture();
301            pics.extend(d.take_pictures().into_iter().map(|p| (p.idr, p.poc, p.frame)));
302        }
303        TrackCodec::Vp9 => return Err("VP9 is not an Annex B stream".into()),
304        TrackCodec::Av1 => return Err("AV1 is not an Annex B H.26x stream".into()),
305        TrackCodec::Hevc => {
306            // HEVC defines its output order normatively (§C.5.2), including
307            // pictures it discards; follow it rather than sorting by POC.
308            let mut d = HevcDecoder::new();
309            for nal in bits::split_annexb(stream) {
310                d.decode_nal(nal)?;
311            }
312            d.flush_output();
313            let mut by_id: std::collections::HashMap<u64, DecodedFrame> =
314                d.take_pictures().into_iter().map(|p| (p.id, p.frame)).collect();
315            return Ok(d
316                .take_output_order()
317                .into_iter()
318                .filter_map(|id| by_id.remove(&id))
319                .collect());
320        }
321    }
322    let mut period = 0u32;
323    let mut keyed: Vec<((u32, i32), DecodedFrame)> = Vec::with_capacity(pics.len());
324    for (i, (starts, poc, frame)) in pics.into_iter().enumerate() {
325        if starts && i > 0 {
326            period += 1;
327        }
328        keyed.push(((period, poc), frame));
329    }
330    keyed.sort_by_key(|(k, _)| *k);
331    Ok(keyed.into_iter().map(|(_, f)| f).collect())
332}