media_pp/core/buffer.rs
1//! What travels between elements.
2//!
3//! [`MediaBuffer`] is an enum rather than one opaque buffer type, because
4//! `ffmpeg-next` already hands back strongly-typed packets and frames and
5//! collapsing them would only mean unwrapping again downstream. Its own
6//! documentation covers why each payload is shared rather than copied, and
7//! why a video frame arrives through a pool reference.
8
9use std::sync::Arc;
10
11use ffmpeg_next::{self as ffmpeg, ffi};
12
13use crate::pool::UnboundObjectPoolRef;
14
15/// The unit of data that flows between elements.
16///
17/// Compressed and uncompressed data are kept as distinct variants (rather
18/// than a single opaque `Buffer` type like GStreamer) because ffmpeg-next
19/// already gives us strongly-typed `Packet`/`Frame` types — collapsing them
20/// into one type would just mean unwrapping again downstream.
21///
22/// Payloads are `Arc`-wrapped so `MediaBuffer` is cheaply `Clone` —
23/// duplicating a buffer (e.g. [`crate::elements::Tee`] fanning packets out
24/// to a decode branch and a remux branch) is a refcount bump, never a copy
25/// of the encoded/decoded data.
26///
27/// `Video` specifically wraps an [`UnboundObjectPoolRef`], not a plain
28/// `ffmpeg::frame::Video` — that's what lets whichever element produced it
29/// (see [`crate::pool::UnboundObjectPool`], owned as that element's own
30/// struct field) get the underlying buffer back automatically once every
31/// `Arc` clone downstream has been dropped, instead of it just being freed.
32#[derive(Clone)]
33pub enum MediaBuffer {
34 /// Encoded media packet produced by a demuxer or encoder.
35 ///
36 /// Its PTS, DTS, duration, stream index, and time base remain part of the
37 /// packet contract while it travels through packet-level elements.
38 Packet(Arc<ffmpeg::Packet>),
39
40 /// Decoded video frame whose backing storage returns to its producer's
41 /// [`crate::pool::UnboundObjectPool`] after the last `Arc` clone drops.
42 ///
43 /// Treat the published frame as immutable. A transforming element creates
44 /// a replacement frame and preserves PTS, duration, and color metadata
45 /// unless it intentionally establishes a new timeline.
46 Video(Arc<UnboundObjectPoolRef<ffmpeg::frame::Video>>),
47
48 /// Decoded audio frame shared immutably between downstream branches.
49 ///
50 /// Sample format, sample rate, channel layout, PTS, and duration describe
51 /// the audio contract a transforming element must either preserve or
52 /// deliberately replace.
53 Audio(Arc<ffmpeg::frame::Audio>),
54
55 /// Ordered end-of-stream marker.
56 ///
57 /// Stateful elements flush delayed output before forwarding it, and
58 /// muxers finalize their output after receiving it. Unlike
59 /// [`crate::control::ControlMsg::Stop`], this requests natural completion
60 /// rather than abandoning buffered work.
61 Eos,
62}
63
64impl MediaBuffer {
65 /// Returns whether this buffer is the ordered [`MediaBuffer::Eos`] marker.
66 pub fn is_eos(&self) -> bool {
67 matches!(self, MediaBuffer::Eos)
68 }
69
70 /// Stable, human-readable variant name for diagnostics emitted when
71 /// elements are wired to an incompatible media type.
72 pub fn kind(&self) -> &'static str {
73 match self {
74 MediaBuffer::Packet(_) => "Packet",
75 MediaBuffer::Video(_) => "Video",
76 MediaBuffer::Audio(_) => "Audio",
77 MediaBuffer::Eos => "Eos",
78 }
79 }
80}
81
82/// Which buffer a video frame's pixels live in.
83///
84/// Not the frame: a producer with nothing new to show re-emits a fresh
85/// `AVFrame` referencing the same picture every tick — a screen capture of a
86/// still desktop does exactly that — so comparing frames, or the `Arc`s
87/// around them, answers "changed" every time while the pixels have not
88/// moved. The plane pointers do not.
89///
90/// The first two planes are enough for every layout this crate carries:
91/// packed formats use one, and the semi-planar and planar ones this crate
92/// composites in differ in the first two whenever they differ at all.
93///
94/// Only sound as an identity while the frame it came from is still
95/// referenced. A picture whose buffer has been released can be handed out
96/// again at the same address, so every caller here holds that reference for
97/// as long as it holds the identity.
98pub(crate) fn picture_id(frame: &ffmpeg::frame::Video) -> (usize, usize) {
99 // SAFETY: `as_ptr` is a live `AVFrame`. Only the values of the first two
100 // plane pointers are read; nothing dereferences them, which for GPU
101 // memory would not be valid from the host anyway.
102 unsafe {
103 let ptr = frame.as_ptr();
104 ((*ptr).data[0] as usize, (*ptr).data[1] as usize)
105 }
106}
107
108/// Lets go of the picture a pooled wrapper was pointing at, as it returns to
109/// its pool.
110///
111/// The `release` an [`UnboundObjectPool`](crate::pool::UnboundObjectPool) of
112/// *wrappers* wants: an empty frame is given a picture with `av_frame_ref` on
113/// every checkout, and without this it would keep that reference until its
114/// next one. Nothing reads those pixels in the meantime, but everything that
115/// asks whether a picture is still in use — [`picture_is_referenced`] — would
116/// go on answering yes, so a producer would keep a buffer, or a screen-sized
117/// texture, alive for an idle wrapper.
118///
119/// Only for a pool whose items own no picture of their own. A pool of real
120/// frames composited or decoded into would be emptied by this.
121pub(crate) fn release_picture(frame: &mut ffmpeg::frame::Video) {
122 // SAFETY: `as_mut_ptr` is this frame's own live `AVFrame`, and the pool
123 // has taken it back, so nothing else refers to it.
124 unsafe { ffi::av_frame_unref(frame.as_mut_ptr()) }
125}
126
127/// Whether anything besides this frame itself still points at its picture.
128///
129/// The companion to [`picture_id`], for an element that offers an unchanged
130/// picture again by pointing an empty wrapper at it with `av_frame_ref`.
131/// Such a wrapper shares the picture's *buffer*, not the pool slot the frame
132/// came from, so an [`UnboundObjectPoolRef`] that has gone back to its pool
133/// says nothing about whether a wrapper downstream is still showing those
134/// pixels — the buffer's own reference count is the only record of it, and
135/// a producer that recycles its frames has to keep one out of the pool until
136/// this reads false for it.
137///
138/// Only the first buffer is examined, for the same reason [`picture_id`]
139/// reads only the first plane pointers: a wrapper references either all of a
140/// frame's buffers or none of them.
141pub(crate) fn picture_is_referenced(frame: &ffmpeg::frame::Video) -> bool {
142 // SAFETY: `as_ptr` is a live `AVFrame`. `buf[0]` is either null (a frame
143 // that owns no picture) or a live `AVBufferRef`, and reading its count
144 // does not touch the picture itself — which for GPU memory would not be
145 // valid from the host anyway.
146 unsafe {
147 let buf = (*frame.as_ptr()).buf[0];
148 !buf.is_null() && ffi::av_buffer_get_ref_count(buf) > 1
149 }
150}
151
152#[cfg(test)]
153mod tests {
154 use super::*;
155
156 #[test]
157 fn kind_reports_each_variant() {
158 assert_eq!(
159 MediaBuffer::Packet(Arc::new(ffmpeg::Packet::empty())).kind(),
160 "Packet"
161 );
162 assert_eq!(
163 MediaBuffer::Audio(Arc::new(ffmpeg::frame::Audio::empty())).kind(),
164 "Audio"
165 );
166 assert_eq!(MediaBuffer::Eos.kind(), "Eos");
167 }
168}