Skip to main content

threers_codec/decode/
platform.rs

1//! The best decoder this build has: VideoToolbox on macOS (with the
2//! `videotoolbox` feature), [`NativeDecoder`] everywhere else.
3//!
4//! [`Decoder`] has the interface both of them share — push samples in decode
5//! order, take pictures back by presentation time — so code written against
6//! it runs on every platform and gets hardware decoding where there is some.
7//! When VideoToolbox will not open a stream, [`Decoder::new`] falls back to
8//! the native decoder rather than failing.
9//!
10//! ```no_run
11//! use threers_codec::decode::Decoder;
12//! use threers_codec::mp4_demux::demux_video;
13//!
14//! let file = std::fs::read("clip.mp4").unwrap();
15//! let track = demux_video(&file).unwrap();
16//! let mut dec = Decoder::new(&track).unwrap();
17//! println!("decoding with {:?}", dec.backend());
18//! for s in &track.samples {
19//!     dec.push(&file[s.offset as usize..][..s.size as usize], s.pts, s.dts, s.duration as i64).unwrap();
20//!     let _bgra = dec.take(s.pts);
21//! }
22//! ```
23
24use super::NativeDecoder;
25use crate::mp4_demux::{TrackCodec, VideoTrack};
26
27#[cfg(all(feature = "videotoolbox", target_os = "macos"))]
28use crate::videotoolbox_decode::VideoToolboxDecoder;
29#[cfg(all(feature = "nvidia", any(target_os = "linux", target_os = "windows"), target_pointer_width = "64"))]
30use crate::nvidia::NvdecDecoder;
31#[cfg(all(feature = "vaapi", target_os = "linux"))]
32use crate::vaapi::VaapiDecoder;
33
34/// Which decoder a [`Decoder`] is using.
35#[derive(Debug, Clone, Copy, PartialEq, Eq)]
36#[non_exhaustive]
37pub enum Backend {
38    /// Apple's, in hardware where the machine has it. macOS with the
39    /// `videotoolbox` feature.
40    VideoToolbox,
41    /// An NVIDIA GPU's NVDEC engine. Linux and Windows with the `nvidia`
42    /// feature, on a machine with the driver.
43    Nvdec,
44    /// A VA driver (AMD, Intel, ...). Linux with the `vaapi` feature, on a
45    /// machine with libva and a driver for the GPU.
46    Vaapi,
47    /// [`NativeDecoder`]: pure Rust, every platform.
48    Native,
49}
50
51impl Backend {
52    /// Whether this build, on this machine, can decode with `self`. For
53    /// NVDEC that means loading the driver, which happens once.
54    pub fn available(self) -> bool {
55        match self {
56            Backend::VideoToolbox => cfg!(all(feature = "videotoolbox", target_os = "macos")),
57            #[cfg(all(feature = "nvidia", any(target_os = "linux", target_os = "windows"), target_pointer_width = "64"))]
58            Backend::Nvdec => crate::nvidia::available(),
59            #[cfg(not(all(feature = "nvidia", any(target_os = "linux", target_os = "windows"), target_pointer_width = "64")))]
60            Backend::Nvdec => false,
61            #[cfg(all(feature = "vaapi", target_os = "linux"))]
62            Backend::Vaapi => crate::vaapi::available(),
63            #[cfg(not(all(feature = "vaapi", target_os = "linux")))]
64            Backend::Vaapi => false,
65            Backend::Native => true,
66        }
67    }
68
69    /// The hardware backends, in the order [`Decoder::new`] tries them.
70    pub const HARDWARE: [Backend; 3] = [Backend::VideoToolbox, Backend::Nvdec, Backend::Vaapi];
71
72    /// The backend [`Decoder::new`] tries first.
73    pub fn preferred() -> Backend {
74        Self::HARDWARE.into_iter().find(|b| b.available()).unwrap_or(Backend::Native)
75    }
76}
77
78enum Inner {
79    #[cfg(all(feature = "videotoolbox", target_os = "macos"))]
80    VideoToolbox(VideoToolboxDecoder),
81    #[cfg(all(feature = "nvidia", any(target_os = "linux", target_os = "windows"), target_pointer_width = "64"))]
82    Nvdec(Box<NvdecDecoder>),
83    #[cfg(all(feature = "vaapi", target_os = "linux"))]
84    Vaapi(Box<VaapiDecoder>),
85    Native(Box<NativeDecoder>),
86}
87
88/// A video decoder on whichever [`Backend`] suits the platform.
89pub struct Decoder {
90    inner: Inner,
91}
92
93impl Decoder {
94    /// The first [hardware backend](Backend::HARDWARE) available that takes
95    /// `track`, or the native decoder when none does.
96    pub fn new(track: &VideoTrack) -> Result<Self, String> {
97        for b in Backend::HARDWARE.into_iter().filter(|b| b.available()) {
98            if let Ok(d) = Self::with_backend(track, b) {
99                return Ok(d);
100            }
101        }
102        // On failure, the native decoder says why.
103        Self::with_backend(track, Backend::Native)
104    }
105
106    /// [`new`](Self::new) from a decoder configuration alone — an `avcC` or
107    /// `hvcC` payload — for streams that do not come from a demuxed file.
108    /// `timescale` is the unit of the timestamps given to [`push`](Self::push).
109    pub fn from_config(codec: TrackCodec, config: &[u8], width: u32, height: u32, timescale: u32) -> Result<Self, String> {
110        Self::new(&config_track(codec, config, width, height, timescale))
111    }
112
113    /// Exactly `backend`, without falling back: an error when this build does
114    /// not have it or it will not take the stream.
115    pub fn with_backend(track: &VideoTrack, backend: Backend) -> Result<Self, String> {
116        let inner = match backend {
117            #[cfg(all(feature = "videotoolbox", target_os = "macos"))]
118            Backend::VideoToolbox => Inner::VideoToolbox(VideoToolboxDecoder::new(track)?),
119            #[cfg(not(all(feature = "videotoolbox", target_os = "macos")))]
120            Backend::VideoToolbox => {
121                return Err("VideoToolbox needs macOS and threers' `videotoolbox` feature".into())
122            }
123            #[cfg(all(feature = "nvidia", any(target_os = "linux", target_os = "windows"), target_pointer_width = "64"))]
124            Backend::Nvdec => Inner::Nvdec(Box::new(NvdecDecoder::new(track)?)),
125            #[cfg(not(all(feature = "nvidia", any(target_os = "linux", target_os = "windows"), target_pointer_width = "64")))]
126            Backend::Nvdec => {
127                return Err("NVDEC needs Linux or Windows and the `nvidia` feature".into())
128            }
129            #[cfg(all(feature = "vaapi", target_os = "linux"))]
130            Backend::Vaapi => Inner::Vaapi(Box::new(VaapiDecoder::new(track)?)),
131            #[cfg(not(all(feature = "vaapi", target_os = "linux")))]
132            Backend::Vaapi => return Err("VAAPI needs Linux and the `vaapi` feature".into()),
133            Backend::Native => Inner::Native(Box::new(NativeDecoder::new(track)?)),
134        };
135        Ok(Decoder { inner })
136    }
137
138    /// Which decoder is doing the work.
139    pub fn backend(&self) -> Backend {
140        match self.inner {
141            #[cfg(all(feature = "videotoolbox", target_os = "macos"))]
142            Inner::VideoToolbox(_) => Backend::VideoToolbox,
143            #[cfg(all(feature = "nvidia", any(target_os = "linux", target_os = "windows"), target_pointer_width = "64"))]
144            Inner::Nvdec(_) => Backend::Nvdec,
145            #[cfg(all(feature = "vaapi", target_os = "linux"))]
146            Inner::Vaapi(_) => Backend::Vaapi,
147            Inner::Native(_) => Backend::Native,
148        }
149    }
150
151    /// Size of the pictures, as the stream's configuration states it.
152    pub fn size(&self) -> (u32, u32) {
153        match &self.inner {
154            #[cfg(all(feature = "videotoolbox", target_os = "macos"))]
155            Inner::VideoToolbox(d) => d.size(),
156            #[cfg(all(feature = "nvidia", any(target_os = "linux", target_os = "windows"), target_pointer_width = "64"))]
157            Inner::Nvdec(d) => d.size(),
158            #[cfg(all(feature = "vaapi", target_os = "linux"))]
159            Inner::Vaapi(d) => d.size(),
160            Inner::Native(d) => d.size(),
161        }
162    }
163
164    /// Hand one coded frame to the decoder, in **decode order**:
165    /// length-prefixed NAL units, as they sit in an MP4 sample. Its picture
166    /// can be [taken](Self::take) as soon as this returns.
167    pub fn push(&mut self, sample: &[u8], pts: i64, dts: i64, duration: i64) -> Result<(), String> {
168        match &mut self.inner {
169            #[cfg(all(feature = "videotoolbox", target_os = "macos"))]
170            Inner::VideoToolbox(d) => d.push(sample, pts, dts, duration),
171            #[cfg(all(feature = "nvidia", any(target_os = "linux", target_os = "windows"), target_pointer_width = "64"))]
172            Inner::Nvdec(d) => d.push(sample, pts, dts, duration),
173            #[cfg(all(feature = "vaapi", target_os = "linux"))]
174            Inner::Vaapi(d) => d.push(sample, pts, dts, duration),
175            Inner::Native(d) => d.push(sample, pts, dts, duration),
176        }
177    }
178
179    /// The picture for presentation time `pts` as tightly packed BGRA,
180    /// `width * height * 4`, if it has been decoded.
181    pub fn take(&mut self, pts: i64) -> Option<Vec<u8>> {
182        match &mut self.inner {
183            #[cfg(all(feature = "videotoolbox", target_os = "macos"))]
184            Inner::VideoToolbox(d) => d.take(pts),
185            #[cfg(all(feature = "nvidia", any(target_os = "linux", target_os = "windows"), target_pointer_width = "64"))]
186            Inner::Nvdec(d) => d.take(pts),
187            #[cfg(all(feature = "vaapi", target_os = "linux"))]
188            Inner::Vaapi(d) => d.take(pts),
189            Inner::Native(d) => d.take(pts),
190        }
191    }
192}
193
194fn config_track(codec: TrackCodec, config: &[u8], width: u32, height: u32, timescale: u32) -> VideoTrack {
195    VideoTrack { codec, width, height, config: config.to_vec(), timescale, samples: Vec::new() }
196}
197
198#[cfg(test)]
199mod tests {
200    use super::*;
201    use crate::h264::{CompressedEncoder, Yuv420Frame};
202
203    #[test]
204    fn decodes_on_every_backend_this_build_has() {
205        let (w, h) = (64u32, 32u32);
206        let rgba: Vec<u8> = (0..w * h).flat_map(|i| [(i % 64 * 4) as u8, 120, 200, 255]).collect();
207        let enc = CompressedEncoder::new(w, h, 20);
208        let sample = enc.encode_sample(&Yuv420Frame::from_rgba(w, h, &rgba));
209        let track = config_track(TrackCodec::H264, &enc.avcc(), w, h, 90_000);
210        for backend in [Backend::VideoToolbox, Backend::Nvdec, Backend::Vaapi, Backend::Native] {
211            let mut dec = match Decoder::with_backend(&track, backend) {
212                Ok(d) => d,
213                // Hardware may refuse a stream (this one is tiny, below some
214                // GPUs' minimum size); `Decoder::new` then decodes natively.
215                Err(e) if backend != Backend::Native => {
216                    eprintln!("{backend:?}: {e}");
217                    continue;
218                }
219                Err(e) => panic!("the native decoder would not open: {e}"),
220            };
221            assert_eq!(dec.backend(), backend);
222            dec.push(&sample, 0, 0, 3600).unwrap();
223            let bgra = dec.take(0).expect("a picture");
224            assert_eq!(bgra.len(), (w * h * 4) as usize);
225            // Mid-frame, the blue channel is near the 200 put in.
226            let px = &bgra[((16 * w + 32) * 4) as usize..][..4];
227            assert!((px[0] as i32 - 200).abs() < 12, "{backend:?}: {px:?}");
228        }
229        let auto = Decoder::from_config(TrackCodec::H264, &enc.avcc(), w, h, 90_000).unwrap();
230        // The preferred backend, unless it refused this stream.
231        assert!([Backend::preferred(), Backend::Native].contains(&auto.backend()));
232    }
233}