Skip to main content

threers_codec/decode/
platform.rs

1//! The best decoder this build has: VideoToolbox on macOS (with the
2//! `videotoolbox` feature), [`NativeDecoder`] everywhere else.
3//!
4//! [`Decoder`] has the interface both of them share — push samples in decode
5//! order, take pictures back by presentation time — so code written against
6//! it runs on every platform and gets hardware decoding where there is some.
7//! When VideoToolbox will not open a stream, [`Decoder::new`] falls back to
8//! the native decoder rather than failing.
9//!
10//! ```no_run
11//! use threers_codec::decode::Decoder;
12//! use threers_codec::mp4_demux::demux_video;
13//!
14//! let file = std::fs::read("clip.mp4").unwrap();
15//! let track = demux_video(&file).unwrap();
16//! let mut dec = Decoder::new(&track).unwrap();
17//! println!("decoding with {:?}", dec.backend());
18//! for s in &track.samples {
19//!     dec.push(&file[s.offset as usize..][..s.size as usize], s.pts, s.dts, s.duration as i64).unwrap();
20//!     let _bgra = dec.take(s.pts);
21//! }
22//! ```
23
24use super::NativeDecoder;
25use crate::mp4_demux::{TrackCodec, VideoTrack};
26
27#[cfg(all(
28    feature = "nvidia",
29    any(target_os = "linux", target_os = "windows"),
30    target_pointer_width = "64"
31))]
32use crate::nvidia::NvdecDecoder;
33#[cfg(all(feature = "vaapi", target_os = "linux"))]
34use crate::vaapi::VaapiDecoder;
35#[cfg(all(feature = "videotoolbox", target_os = "macos"))]
36use crate::videotoolbox_decode::VideoToolboxDecoder;
37
38/// Which decoder a [`Decoder`] is using.
39#[derive(Debug, Clone, Copy, PartialEq, Eq)]
40#[non_exhaustive]
41pub enum Backend {
42    /// Apple's, in hardware where the machine has it. macOS with the
43    /// `videotoolbox` feature.
44    VideoToolbox,
45    /// An NVIDIA GPU's NVDEC engine. Linux and Windows with the `nvidia`
46    /// feature, on a machine with the driver.
47    Nvdec,
48    /// A VA driver (AMD, Intel, ...). Linux with the `vaapi` feature, on a
49    /// machine with libva and a driver for the GPU.
50    Vaapi,
51    /// [`NativeDecoder`]: pure Rust, every platform.
52    Native,
53}
54
55impl Backend {
56    /// Whether this build, on this machine, can decode with `self`. For
57    /// NVDEC that means loading the driver, which happens once.
58    pub fn available(self) -> bool {
59        match self {
60            Backend::VideoToolbox => cfg!(all(feature = "videotoolbox", target_os = "macos")),
61            #[cfg(all(
62                feature = "nvidia",
63                any(target_os = "linux", target_os = "windows"),
64                target_pointer_width = "64"
65            ))]
66            Backend::Nvdec => crate::nvidia::available(),
67            #[cfg(not(all(
68                feature = "nvidia",
69                any(target_os = "linux", target_os = "windows"),
70                target_pointer_width = "64"
71            )))]
72            Backend::Nvdec => false,
73            #[cfg(all(feature = "vaapi", target_os = "linux"))]
74            Backend::Vaapi => crate::vaapi::available(),
75            #[cfg(not(all(feature = "vaapi", target_os = "linux")))]
76            Backend::Vaapi => false,
77            Backend::Native => true,
78        }
79    }
80
81    /// The hardware backends, in the order [`Decoder::new`] tries them.
82    pub const HARDWARE: [Backend; 3] = [Backend::VideoToolbox, Backend::Nvdec, Backend::Vaapi];
83
84    /// The backend [`Decoder::new`] tries first.
85    pub fn preferred() -> Backend {
86        Self::HARDWARE
87            .into_iter()
88            .find(|b| b.available())
89            .unwrap_or(Backend::Native)
90    }
91}
92
93enum Inner {
94    #[cfg(all(feature = "videotoolbox", target_os = "macos"))]
95    VideoToolbox(VideoToolboxDecoder),
96    #[cfg(all(
97        feature = "nvidia",
98        any(target_os = "linux", target_os = "windows"),
99        target_pointer_width = "64"
100    ))]
101    Nvdec(Box<NvdecDecoder>),
102    #[cfg(all(feature = "vaapi", target_os = "linux"))]
103    Vaapi(Box<VaapiDecoder>),
104    Native(Box<NativeDecoder>),
105}
106
107/// A video decoder on whichever [`Backend`] suits the platform.
108pub struct Decoder {
109    inner: Inner,
110}
111
112impl Decoder {
113    /// The first [hardware backend](Backend::HARDWARE) available that takes
114    /// `track`, or the native decoder when none does.
115    pub fn new(track: &VideoTrack) -> Result<Self, String> {
116        for b in Backend::HARDWARE.into_iter().filter(|b| b.available()) {
117            if let Ok(d) = Self::with_backend(track, b) {
118                return Ok(d);
119            }
120        }
121        // On failure, the native decoder says why.
122        Self::with_backend(track, Backend::Native)
123    }
124
125    /// [`new`](Self::new) from a decoder configuration alone — an `avcC` or
126    /// `hvcC` payload — for streams that do not come from a demuxed file.
127    /// `timescale` is the unit of the timestamps given to [`push`](Self::push).
128    pub fn from_config(
129        codec: TrackCodec,
130        config: &[u8],
131        width: u32,
132        height: u32,
133        timescale: u32,
134    ) -> Result<Self, String> {
135        Self::new(&config_track(codec, config, width, height, timescale))
136    }
137
138    /// Exactly `backend`, without falling back: an error when this build does
139    /// not have it or it will not take the stream.
140    pub fn with_backend(track: &VideoTrack, backend: Backend) -> Result<Self, String> {
141        let inner = match backend {
142            #[cfg(all(feature = "videotoolbox", target_os = "macos"))]
143            Backend::VideoToolbox => Inner::VideoToolbox(VideoToolboxDecoder::new(track)?),
144            #[cfg(not(all(feature = "videotoolbox", target_os = "macos")))]
145            Backend::VideoToolbox => {
146                return Err("VideoToolbox needs macOS and threers' `videotoolbox` feature".into())
147            }
148            #[cfg(all(
149                feature = "nvidia",
150                any(target_os = "linux", target_os = "windows"),
151                target_pointer_width = "64"
152            ))]
153            Backend::Nvdec => Inner::Nvdec(Box::new(NvdecDecoder::new(track)?)),
154            #[cfg(not(all(
155                feature = "nvidia",
156                any(target_os = "linux", target_os = "windows"),
157                target_pointer_width = "64"
158            )))]
159            Backend::Nvdec => {
160                return Err("NVDEC needs Linux or Windows and the `nvidia` feature".into())
161            }
162            #[cfg(all(feature = "vaapi", target_os = "linux"))]
163            Backend::Vaapi => Inner::Vaapi(Box::new(VaapiDecoder::new(track)?)),
164            #[cfg(not(all(feature = "vaapi", target_os = "linux")))]
165            Backend::Vaapi => return Err("VAAPI needs Linux and the `vaapi` feature".into()),
166            Backend::Native => Inner::Native(Box::new(NativeDecoder::new(track)?)),
167        };
168        Ok(Decoder { inner })
169    }
170
171    /// Which decoder is doing the work.
172    pub fn backend(&self) -> Backend {
173        match self.inner {
174            #[cfg(all(feature = "videotoolbox", target_os = "macos"))]
175            Inner::VideoToolbox(_) => Backend::VideoToolbox,
176            #[cfg(all(
177                feature = "nvidia",
178                any(target_os = "linux", target_os = "windows"),
179                target_pointer_width = "64"
180            ))]
181            Inner::Nvdec(_) => Backend::Nvdec,
182            #[cfg(all(feature = "vaapi", target_os = "linux"))]
183            Inner::Vaapi(_) => Backend::Vaapi,
184            Inner::Native(_) => Backend::Native,
185        }
186    }
187
188    /// Size of the pictures, as the stream's configuration states it.
189    pub fn size(&self) -> (u32, u32) {
190        match &self.inner {
191            #[cfg(all(feature = "videotoolbox", target_os = "macos"))]
192            Inner::VideoToolbox(d) => d.size(),
193            #[cfg(all(
194                feature = "nvidia",
195                any(target_os = "linux", target_os = "windows"),
196                target_pointer_width = "64"
197            ))]
198            Inner::Nvdec(d) => d.size(),
199            #[cfg(all(feature = "vaapi", target_os = "linux"))]
200            Inner::Vaapi(d) => d.size(),
201            Inner::Native(d) => d.size(),
202        }
203    }
204
205    /// Hand one coded frame to the decoder, in **decode order**:
206    /// length-prefixed NAL units, as they sit in an MP4 sample. Its picture
207    /// can be [taken](Self::take) as soon as this returns.
208    pub fn push(&mut self, sample: &[u8], pts: i64, dts: i64, duration: i64) -> Result<(), String> {
209        match &mut self.inner {
210            #[cfg(all(feature = "videotoolbox", target_os = "macos"))]
211            Inner::VideoToolbox(d) => d.push(sample, pts, dts, duration),
212            #[cfg(all(
213                feature = "nvidia",
214                any(target_os = "linux", target_os = "windows"),
215                target_pointer_width = "64"
216            ))]
217            Inner::Nvdec(d) => d.push(sample, pts, dts, duration),
218            #[cfg(all(feature = "vaapi", target_os = "linux"))]
219            Inner::Vaapi(d) => d.push(sample, pts, dts, duration),
220            Inner::Native(d) => d.push(sample, pts, dts, duration),
221        }
222    }
223
224    /// The picture for presentation time `pts` as tightly packed BGRA,
225    /// `width * height * 4`, if it has been decoded.
226    pub fn take(&mut self, pts: i64) -> Option<Vec<u8>> {
227        match &mut self.inner {
228            #[cfg(all(feature = "videotoolbox", target_os = "macos"))]
229            Inner::VideoToolbox(d) => d.take(pts),
230            #[cfg(all(
231                feature = "nvidia",
232                any(target_os = "linux", target_os = "windows"),
233                target_pointer_width = "64"
234            ))]
235            Inner::Nvdec(d) => d.take(pts),
236            #[cfg(all(feature = "vaapi", target_os = "linux"))]
237            Inner::Vaapi(d) => d.take(pts),
238            Inner::Native(d) => d.take(pts),
239        }
240    }
241}
242
243fn config_track(
244    codec: TrackCodec,
245    config: &[u8],
246    width: u32,
247    height: u32,
248    timescale: u32,
249) -> VideoTrack {
250    VideoTrack {
251        codec,
252        width,
253        height,
254        config: config.to_vec(),
255        timescale,
256        samples: Vec::new(),
257    }
258}
259
260#[cfg(test)]
261mod tests {
262    use super::*;
263    use crate::h264::{CompressedEncoder, Yuv420Frame};
264
265    #[test]
266    fn decodes_on_every_backend_this_build_has() {
267        let (w, h) = (64u32, 32u32);
268        let rgba: Vec<u8> = (0..w * h)
269            .flat_map(|i| [(i % 64 * 4) as u8, 120, 200, 255])
270            .collect();
271        let enc = CompressedEncoder::new(w, h, 20);
272        let sample = enc.encode_sample(&Yuv420Frame::from_rgba(w, h, &rgba));
273        let track = config_track(TrackCodec::H264, &enc.avcc(), w, h, 90_000);
274        for backend in [
275            Backend::VideoToolbox,
276            Backend::Nvdec,
277            Backend::Vaapi,
278            Backend::Native,
279        ] {
280            let mut dec = match Decoder::with_backend(&track, backend) {
281                Ok(d) => d,
282                // Hardware may refuse a stream (this one is tiny, below some
283                // GPUs' minimum size); `Decoder::new` then decodes natively.
284                Err(e) if backend != Backend::Native => {
285                    eprintln!("{backend:?}: {e}");
286                    continue;
287                }
288                Err(e) => panic!("the native decoder would not open: {e}"),
289            };
290            assert_eq!(dec.backend(), backend);
291            dec.push(&sample, 0, 0, 3600).unwrap();
292            let bgra = dec.take(0).expect("a picture");
293            assert_eq!(bgra.len(), (w * h * 4) as usize);
294            // Mid-frame, the blue channel is near the 200 put in.
295            let px = &bgra[((16 * w + 32) * 4) as usize..][..4];
296            assert!((px[0] as i32 - 200).abs() < 12, "{backend:?}: {px:?}");
297        }
298        let auto = Decoder::from_config(TrackCodec::H264, &enc.avcc(), w, h, 90_000).unwrap();
299        // The preferred backend, unless it refused this stream.
300        assert!([Backend::preferred(), Backend::Native].contains(&auto.backend()));
301    }
302}