ff_render/compositor/mod.rs
1//! Multi-layer GPU compositing: [`FrameLayer`] stack in, one texture out.
2//!
3//! # Alpha convention
4//!
5//! The canvas starts transparent (its texture is zero-initialised) and each
6//! layer is composited onto it in `z_order`. Two rules, which
7//! `shaders/blend.wgsl` and [`nodes::BlendModeNode`](crate::BlendModeNode)
8//! implement identically:
9//!
10//! - **Colour** is composited against an **opaque black backdrop**, so the blend
11//! result is not reweighted by the backdrop alpha the way W3C's
12//! `Cs' = (1 - ab) * Cs + ab * B(Cb, Cs)` would. That matches the CPU
13//! compositor's `color=c=#000000` canvas, which ADR-0007 keeps as the
14//! correctness reference.
15//! - **Alpha** accumulates as src-over **coverage**, `ao = as + ab * (1 - as)`:
16//! zero where no layer has drawn (letterbox bands, the transparent regions a
17//! transform leaves behind), rising toward one as opaque layers cover it
18//! (#1750).
19//! - **RGB is premultiplied** by that alpha. A white layer composited at opacity
20//! `0.5` over the empty canvas reads back `(128, 128, 128, 128)`, not
21//! `(255, 255, 255, 128)`. A consumer that wants straight alpha divides by
22//! `ao`, guarding `ao == 0`.
23//!
24//! Consumers that flatten to an opaque format (the export path converts rgba to
25//! yuv420p) ignore the alpha and read the premultiplied RGB directly, which is
26//! the same thing as compositing over black; consumers that composite further,
27//! or that need to know what a layer covered, read the alpha.
28
29#[cfg(feature = "wgpu")]
30mod compositor_inner;
31
32use ff_format::VideoFrame;
33
34use crate::nodes::{BlendMode, CompositeOp};
35
36// LayerTransform
37
38/// 2D affine transform parameters for a compositor layer.
39///
40/// All values use UV-space coordinates where 0.0 is no change. The default
41/// (identity) transform leaves the layer centred and unscaled.
42#[derive(Debug, Clone)]
43pub struct LayerTransform {
44 /// Horizontal UV-space offset (positive = shift right). Default: `0.0`.
45 pub x: f32,
46 /// Vertical UV-space offset (positive = shift down). Default: `0.0`.
47 pub y: f32,
48 /// Horizontal scale factor (`1.0` = no change). Default: `1.0`.
49 pub scale_x: f32,
50 /// Vertical scale factor (`1.0` = no change). Default: `1.0`.
51 pub scale_y: f32,
52 /// Counter-clockwise rotation in radians. Default: `0.0`.
53 pub rotation: f32,
54}
55
56impl Default for LayerTransform {
57 fn default() -> Self {
58 Self {
59 x: 0.0,
60 y: 0.0,
61 scale_x: 1.0,
62 scale_y: 1.0,
63 rotation: 0.0,
64 }
65 }
66}
67
68impl LayerTransform {
69 /// Returns `true` when this transform is the identity (no visual change).
70 #[must_use]
71 pub fn is_identity(&self) -> bool {
72 self.x.abs() < 1e-6
73 && self.y.abs() < 1e-6
74 && (self.scale_x - 1.0).abs() < 1e-6
75 && (self.scale_y - 1.0).abs() < 1e-6
76 && self.rotation.abs() < 1e-6
77 }
78}
79
80// FrameLayer
81
82/// A single layer in the composition stack.
83pub struct FrameLayer {
84 /// Source video frame (uploaded to GPU by [`Compositor`]).
85 pub frame: VideoFrame,
86 /// 2D affine transform applied before compositing.
87 pub transform: LayerTransform,
88 /// Blend mode used when compositing this layer over layers below.
89 ///
90 /// Only meaningful with [`CompositeOp::Over`]: the editing model does not
91 /// combine the two, so `avio::gpu::map_scene` emits `Normal` for a layer
92 /// whose composite operator is anything else.
93 pub blend_mode: BlendMode,
94 /// Porter-Duff operator deciding how much of this layer and the canvas
95 /// below survive. Defaults to [`CompositeOp::Over`].
96 pub composite_op: CompositeOp,
97 /// Layer opacity (`0.0` = transparent, `1.0` = fully opaque).
98 pub opacity: f32,
99 /// Z-order — lower values are further back. Layers are sorted ascending
100 /// by this field before compositing.
101 pub z_order: i32,
102}
103
104// Compositor
105
106/// Stateful high-level multi-layer GPU compositor.
107///
108/// Accepts a list of [`FrameLayer`]s, sorts them by [`FrameLayer::z_order`],
109/// uploads each frame to the GPU, applies per-layer transforms and blend modes,
110/// and returns the composited [`wgpu::Texture`].
111///
112/// The wgpu render pipeline is built on the first call to
113/// [`composite`](Self::composite) and reused across frames. It is rebuilt only
114/// when the number of layers changes.
115///
116/// # Thread safety
117///
118/// `Compositor` is [`Send`] and can be moved to a background thread. When
119/// multiple threads need to share a compositor, wrap it in
120/// `Arc<Mutex<Compositor>>`.
121///
122/// Requires the `wgpu` feature.
123#[cfg(feature = "wgpu")]
124pub struct Compositor {
125 ctx: std::sync::Arc<crate::context::RenderContext>,
126 width: u32,
127 height: u32,
128 graph: Option<compositor_inner::CompositorGraph>,
129 last_layer_count: usize,
130}
131
132#[cfg(feature = "wgpu")]
133impl Compositor {
134 /// Create a compositor targeting the given output resolution.
135 #[must_use]
136 pub fn new(
137 ctx: std::sync::Arc<crate::context::RenderContext>,
138 width: u32,
139 height: u32,
140 ) -> Self {
141 Self {
142 ctx,
143 width,
144 height,
145 graph: None,
146 last_layer_count: 0,
147 }
148 }
149
150 /// Composite `layers` into a single [`wgpu::Texture`].
151 ///
152 /// Layers are sorted by [`FrameLayer::z_order`] before compositing
153 /// (ascending — lowest `z_order` is the bottom layer).
154 ///
155 /// The wgpu pipeline is built on the first call and cached; it is rebuilt
156 /// only when `layers.len()` changes between calls.
157 ///
158 /// # Errors
159 ///
160 /// Returns [`RenderError`](crate::error::RenderError) on GPU texture
161 /// creation failure, unsupported pixel format, or render failure.
162 pub fn composite(
163 &mut self,
164 layers: &mut [FrameLayer],
165 ) -> Result<wgpu::Texture, crate::error::RenderError> {
166 layers.sort_unstable_by_key(|l| l.z_order);
167
168 let need_rebuild = self.graph.is_none() || self.last_layer_count != layers.len();
169 if need_rebuild {
170 self.graph = Some(compositor_inner::CompositorGraph::build(
171 &self.ctx,
172 layers.len(),
173 self.width,
174 self.height,
175 ));
176 self.last_layer_count = layers.len();
177 }
178
179 let Some(graph) = self.graph.as_mut() else {
180 return Err(crate::error::RenderError::Composite {
181 message: "compositor graph not initialized".to_string(),
182 });
183 };
184 graph.composite(&self.ctx, layers, self.width, self.height)
185 }
186
187 /// Composite `layers` and read the result back to a dense `rgba` buffer
188 /// (`width * height * 4` bytes, `Rgba8Unorm`), returning `(rgba, width, height)`.
189 ///
190 /// A convenience over [`composite`](Self::composite) for callers that need the
191 /// pixels on the CPU (e.g. handing a frame to an encoder or a CPU frame sink).
192 /// Prefer [`composite`](Self::composite) plus the zero-copy display path when a
193 /// GPU-resident texture is enough.
194 ///
195 /// # Errors
196 ///
197 /// Returns [`RenderError`](crate::error::RenderError) on composite failure or if
198 /// the GPU-to-CPU readback fails.
199 pub fn composite_to_rgba(
200 &mut self,
201 layers: &mut [FrameLayer],
202 ) -> Result<(Vec<u8>, u32, u32), crate::error::RenderError> {
203 let texture = self.composite(layers)?;
204 let rgba =
205 compositor_inner::read_texture_rgba(&self.ctx, &texture, self.width, self.height)?;
206 Ok((rgba, self.width, self.height))
207 }
208}
209
210#[cfg(test)]
211mod tests {
212 use super::*;
213 use ff_format::{PixelFormat, VideoFrame};
214
215 fn make_frame() -> VideoFrame {
216 VideoFrame::empty(2, 2, PixelFormat::Rgba).expect("test frame")
217 }
218
219 #[test]
220 fn layer_transform_default_should_be_identity() {
221 let t = LayerTransform::default();
222 assert!(
223 t.is_identity(),
224 "default LayerTransform must be the identity"
225 );
226 }
227
228 #[test]
229 fn layer_transform_nonzero_x_should_not_be_identity() {
230 let t = LayerTransform {
231 x: 0.1,
232 ..Default::default()
233 };
234 assert!(
235 !t.is_identity(),
236 "LayerTransform with non-zero x must not be identity"
237 );
238 }
239
240 #[test]
241 fn frame_layer_should_construct_with_defaults() {
242 let layer = FrameLayer {
243 frame: make_frame(),
244 transform: LayerTransform::default(),
245 blend_mode: BlendMode::Normal,
246 composite_op: CompositeOp::Over,
247 opacity: 1.0,
248 z_order: 0,
249 };
250 assert_eq!(layer.z_order, 0);
251 assert!((layer.opacity - 1.0).abs() < 1e-6);
252 }
253
254 #[test]
255 fn compositor_layers_should_sort_by_z_order() {
256 let mut layers = vec![
257 FrameLayer {
258 frame: make_frame(),
259 transform: LayerTransform::default(),
260 blend_mode: BlendMode::Normal,
261 composite_op: CompositeOp::Over,
262 opacity: 1.0,
263 z_order: 3,
264 },
265 FrameLayer {
266 frame: make_frame(),
267 transform: LayerTransform::default(),
268 blend_mode: BlendMode::Normal,
269 composite_op: CompositeOp::Over,
270 opacity: 1.0,
271 z_order: 1,
272 },
273 FrameLayer {
274 frame: make_frame(),
275 transform: LayerTransform::default(),
276 blend_mode: BlendMode::Normal,
277 composite_op: CompositeOp::Over,
278 opacity: 1.0,
279 z_order: 2,
280 },
281 ];
282 layers.sort_unstable_by_key(|l| l.z_order);
283 let z_orders: Vec<i32> = layers.iter().map(|l| l.z_order).collect();
284 assert_eq!(
285 z_orders,
286 vec![1, 2, 3],
287 "layers must sort ascending by z_order"
288 );
289 }
290
291 #[cfg(feature = "wgpu")]
292 #[test]
293 fn compositor_should_be_send() {
294 fn assert_send<T: Send>() {}
295 assert_send::<Compositor>();
296 }
297}