moq_video/decode/
mod.rs

1//! Subscribe to an H.264, H.265, or AV1 track and decode it to raw frames.
2//!
3//! The decode counterpart to [`encode`](crate::encode), and the mirror of
4//! `moq_audio::decode::Consumer`. [`Consumer`] subscribes to a moq-mux video
5//! track and hands back decoded [`Frame`]s; a native backend does the work
6//! (VideoToolbox on macOS, Media Foundation / DXVA on Windows, NVDEC on Linux,
7//! openh264 everywhere as the software fallback for H.264).
8//!
9//! H.264 and H.265 are supported, symmetric with what [`encode`](crate::encode)
10//! produces. AV1 is decode-only on NVDEC. H.265 and AV1 are hardware-only (no
11//! software fallback). Any other codec yields
12//! [`Error::UnsupportedCodec`](crate::Error).
13
14use moq_net::Timestamp;
15
16use crate::{Error, Size, Surface};
17
18// Crate-visible so the NVENC encode backend's round-trip test can decode its
19// output with the software decoder (an in-crate, ffmpeg-free encode->decode
20// check that catches input-pitch corruption).
21pub(crate) mod backend;
22mod consumer;
23mod decoder;
24
25pub use consumer::Consumer;
26pub use decoder::{Config, Decoder, Kind};
27
28/// A decoded raw video frame: CPU I420, or a GPU frame when a hardware decoder
29/// produced one (NVDEC on Linux).
30///
31/// A GPU frame stays on the GPU until something needs bytes: feeding it to
32/// [`encode::Encoder::encode`](crate::encode::Encoder::encode) keeps it there
33/// (the zero-copy transcode path), while [`Surface::into_i420`]
34/// downloads it.
35pub struct Frame {
36	/// Presentation timestamp, carried through from the container. It rides out of
37	/// the decoder with each picture, so a reordered frame (B-frames) keeps its own
38	/// time rather than the input access unit's.
39	pub timestamp: Timestamp,
40	/// The decoded resolution, which is [`Config::resize`] when the backend
41	/// honored it and the stream's native size otherwise.
42	pub size: Size,
43	/// Where the pixels live. Match on it for a zero-copy path, or call
44	/// [`Surface::into_i420`] for the universal one.
45	pub surface: Surface,
46}
47
48impl Frame {
49	/// A copy of this frame scaled to `size` (both dimensions even and non-zero),
50	/// preserving the timestamp. A CUDA frame scales on the GPU (a box filter,
51	/// correct at any downscale factor) and stays there, so resize ->
52	/// [`encode`](crate::encode::Encoder::encode) never touches the CPU. Every
53	/// other frame scales on the CPU, which for a `CVPixelBuffer` means a
54	/// download first. When one output size is enough, prefer decoding
55	/// straight to it ([`Config::resize`]), which is free on decoders with a
56	/// hardware scaler; this method is for fanning one decoded stream out to
57	/// several sizes.
58	pub fn resize(&self, size: Size) -> Result<Frame, Error> {
59		size.validate("resize to")?;
60		let Size { width, height } = size;
61
62		let surface = match &self.surface {
63			Surface::I420(i420) => Surface::I420(i420.resize(width, height)?),
64			#[cfg(all(target_os = "linux", feature = "nvdec"))]
65			Surface::Cuda(cuda) => match cuda.resize(width, height) {
66				Ok(scaled) => Surface::Cuda(scaled),
67				// E.g. the driver rejected the vendored PTX: degrade to a CPU
68				// resize (download once) instead of killing the stream.
69				Err(err) => {
70					static WARN_ONCE: std::sync::Once = std::sync::Once::new();
71					WARN_ONCE.call_once(|| tracing::warn!(%err, "GPU resize failed; falling back to the CPU"));
72					Surface::I420(cuda.download_i420()?.resize(width, height)?)
73				}
74			},
75			// CVPixelBuffer (the VideoToolbox decoder's output) and D3D11 textures
76			// have no GPU scaler wired up yet: download, then scale on the CPU.
77			#[allow(unreachable_patterns)]
78			other => Surface::I420(other.to_i420()?.into_owned().resize(width, height)?),
79		};
80
81		Ok(Frame {
82			timestamp: self.timestamp,
83			size,
84			surface,
85		})
86	}
87}
88
89#[cfg(test)]
90mod tests {
91	/// Callers (libmoq, moq-transcode) hold these across `.await`s in spawned
92	/// tasks and share frames via `Arc` (the transcode fanout), so both must
93	/// stay `Send` and `Frame` also `Sync` even when a platform's frame wraps
94	/// a GPU handle. Compile-time check; fails per-platform if a variant
95	/// regresses.
96	#[test]
97	fn frame_and_consumer_are_thread_safe() {
98		fn assert_send<T: Send>() {}
99		fn assert_sync<T: Sync>() {}
100		assert_send::<super::Frame>();
101		assert_sync::<super::Frame>();
102		assert_send::<super::Consumer>();
103	}
104
105	/// `into_pixel_buffer` is total: a CPU frame uploads rather than failing, so a
106	/// renderer never has to write the upload itself. Software-decoded frames take
107	/// this path.
108	#[cfg(target_os = "macos")]
109	#[test]
110	fn into_pixel_buffer_uploads_a_cpu_frame() {
111		use objc2_core_video::{CVPixelBufferGetHeight, CVPixelBufferGetWidth};
112
113		let frame = super::Frame {
114			timestamp: moq_net::Timestamp::from_micros(0).unwrap(),
115			size: crate::Size::new(64, 32),
116			surface: crate::Surface::I420(crate::I420 {
117				width: 64,
118				height: 32,
119				data: vec![0x80; crate::I420::len(64, 32)],
120			}),
121		};
122
123		let buffer = frame.surface.into_pixel_buffer().expect("upload a CPU frame");
124		assert_eq!(CVPixelBufferGetWidth(&buffer), 64);
125		assert_eq!(CVPixelBufferGetHeight(&buffer), 32);
126	}
127}
moq_video/decode/mod.rs

moq_video/decode/
mod.rs
