Skip to main content

alpha2_render/
readback.rs

1use std::sync::atomic::{AtomicBool, Ordering};
2#[cfg(not(target_arch = "wasm32"))]
3use std::sync::mpsc;
4
5use crate::canary::{CANARY_BYTES, Canary};
6use crate::{
7    ReadbackFailure, RenderError, RenderSize, RenderedTexture, Renderer,
8    pipeline::{CachedPipeline, CachedYuvReadbackPipeline},
9    pool::{ReadbackBufferLease, TextureLease},
10    uniform::{UniformArenaLease, f32_bytes},
11};
12use alpha2_gpu::GpuContext;
13use alpha2_media::{ChromaLocation, ColorMatrix, PixelLayout, YuvEncoding};
14use alpha2_shaders::{READBACK_PACK_WGSL, RGB_TO_YUV420_WGSL};
15
16const RGBA8_BYTES_PER_PIXEL: u32 = 4;
17const RGBA16F_BYTES_PER_PIXEL: u32 = 8;
18const YUV420_PLANE_COUNT: u64 = 3;
19
20/// D3D12 texture-to-buffer copies must start on this; wgpu's DX12 workaround for an unaligned
21/// start pads rows and overwrites whatever follows.
22const PLANE_PLACEMENT_ALIGNMENT: u64 = 512;
23
24/// Where each plane of one I420 readback sits in its staging buffer.
25///
26/// Each plane starts on [`PLANE_PLACEMENT_ALIGNMENT`] so DX12's padded-copy workaround cannot
27/// overwrite the next plane.
28#[derive(Copy, Clone, Debug, Eq, PartialEq)]
29pub(crate) struct Yuv420StagingLayout {
30    luma_pitch: u32,
31    chroma_pitch: u32,
32    u_offset: u64,
33    v_offset: u64,
34    bytes: u64,
35}
36
37impl Yuv420StagingLayout {
38    pub(crate) fn new(size: RenderSize) -> Result<Self, RenderError> {
39        let luma_pitch = aligned_copy_row(size.width)?;
40        let chroma_pitch = aligned_copy_row(size.width / 2)?;
41        let region = |pitch: u32, rows: u32| u64::from(pitch).checked_mul(u64::from(rows));
42        let place = |end: u64| end.checked_next_multiple_of(PLANE_PLACEMENT_ALIGNMENT);
43        let chroma_bytes = region(chroma_pitch, size.height / 2)
44            .ok_or(RenderError::TextureReadback(ReadbackFailure::Layout))?;
45        let u_offset = region(luma_pitch, size.height)
46            .and_then(place)
47            .ok_or(RenderError::TextureReadback(ReadbackFailure::Layout))?;
48        let v_offset = u_offset
49            .checked_add(chroma_bytes)
50            .and_then(place)
51            .ok_or(RenderError::TextureReadback(ReadbackFailure::Layout))?;
52        let bytes = v_offset
53            .checked_add(chroma_bytes)
54            .ok_or(RenderError::TextureReadback(ReadbackFailure::Layout))?;
55        Ok(Self {
56            luma_pitch,
57            chroma_pitch,
58            u_offset,
59            v_offset,
60            bytes,
61        })
62    }
63
64    /// `(offset, pitch, width, rows)` in Y, U, V order.
65    fn planes(self, size: RenderSize) -> [(u64, u32, u32, u32); 3] {
66        let (chroma_width, chroma_rows) = (size.width / 2, size.height / 2);
67        [
68            (0, self.luma_pitch, size.width, size.height),
69            (self.u_offset, self.chroma_pitch, chroma_width, chroma_rows),
70            (self.v_offset, self.chroma_pitch, chroma_width, chroma_rows),
71        ]
72    }
73}
74
75/// Staging bytes of one planar YUV420 readback of `size`, including row padding, plane
76/// alignment and the 16-byte canary. A budget must reserve this per in-flight readback.
77///
78/// # Errors
79/// Returns [`RenderError::TextureReadback`] when the layout overflows.
80pub fn yuv420p_staging_bytes(size: RenderSize) -> Result<u64, RenderError> {
81    Yuv420StagingLayout::new(size).map(|layout| layout.bytes + CANARY_BYTES)
82}
83
84/// The mapped buffer a layout readback ([`Renderer::begin_layout_readback`]) of a `size` frame in
85/// `format` takes, canary included: one aligned row of texels per row.
86///
87/// # Errors
88/// [`RenderError::TextureReadback`] for a format the readback does not copy or an overflowing size.
89pub fn layout_staging_bytes(
90    size: RenderSize,
91    format: wgpu::TextureFormat,
92) -> Result<u64, RenderError> {
93    let texel_bytes = match format {
94        wgpu::TextureFormat::Rgba16Float => 8,
95        wgpu::TextureFormat::Rgba8Unorm => 4,
96        _ => {
97            return Err(RenderError::TextureReadback(
98                ReadbackFailure::UnsupportedTarget,
99            ));
100        }
101    };
102    texel_staging(size, texel_bytes).map(|(_, bytes)| bytes)
103}
104
105/// The row pitch and the buffer size, canary included, of a texel readback.
106fn texel_staging(size: RenderSize, texel_bytes: u32) -> Result<(u32, u64), RenderError> {
107    let row_bytes = size
108        .width
109        .checked_mul(texel_bytes)
110        .ok_or(RenderError::TextureReadback(ReadbackFailure::Layout))?;
111    let pitch = aligned_copy_row(row_bytes)?;
112    let bytes = u64::from(pitch)
113        .checked_mul(u64::from(size.height))
114        .and_then(|image| image.checked_add(CANARY_BYTES))
115        .ok_or(RenderError::TextureReadback(ReadbackFailure::Layout))?;
116    Ok((pitch, bytes))
117}
118
119fn plane_rows(
120    mapped: &[u8],
121    (offset, pitch, width, rows): (u64, u32, u32, u32),
122) -> Result<std::iter::Take<std::slice::ChunksExact<'_, u8>>, RenderError> {
123    let offset = usize::try_from(offset)
124        .map_err(|_| RenderError::TextureReadback(ReadbackFailure::Layout))?;
125    let pitch = usize::try_from(pitch)
126        .map_err(|_| RenderError::TextureReadback(ReadbackFailure::Layout))?;
127    let width = usize::try_from(width)
128        .map_err(|_| RenderError::TextureReadback(ReadbackFailure::Layout))?;
129    let rows =
130        usize::try_from(rows).map_err(|_| RenderError::TextureReadback(ReadbackFailure::Layout))?;
131    let region = pitch
132        .checked_mul(rows)
133        .and_then(|length| length.checked_add(offset))
134        .and_then(|end| mapped.get(offset..end))
135        .filter(|_| width <= pitch)
136        .ok_or(RenderError::TextureReadback(ReadbackFailure::Layout))?;
137    Ok(region.chunks_exact(pitch).take(rows))
138}
139
140pub(crate) struct EncodedYuv420pReadback {
141    buffer: ReadbackBufferLease,
142    size: RenderSize,
143    layout: Yuv420StagingLayout,
144    staging_bytes: u64,
145    canary: Canary,
146    y_texture: TextureLease,
147    u_texture: TextureLease,
148    v_texture: TextureLease,
149    uniforms: UniformArenaLease,
150}
151
152impl EncodedYuv420pReadback {
153    pub(crate) fn flush_uniforms(&self, queue: &wgpu::Queue) {
154        self.uniforms.flush(queue);
155    }
156}
157
158impl Renderer {
159    /// Uploads tightly packed premultiplied sRGB RGBA8 pixels into a pooled upload texture.
160    ///
161    /// # Errors
162    /// [`RenderError::InvalidImageData`] when `pixels` is not four bytes per pixel.
163    pub fn upload_rgba8(
164        &self,
165        size: RenderSize,
166        pixels: &[u8],
167    ) -> Result<crate::UploadTexture, RenderError> {
168        self.write_rgba8(
169            size,
170            crate::SourcePlane {
171                bytes: pixels,
172                stride: size
173                    .width
174                    .checked_mul(RGBA8_BYTES_PER_PIXEL)
175                    .ok_or(RenderError::InvalidImageData)?,
176            },
177        )
178    }
179
180    /// Uploads tightly packed premultiplied linear RGBA half floats (BT.709, `1.0` = SDR white) for
181    /// [`Renderer::prepare_linear_presented`].
182    ///
183    /// # Errors
184    /// [`RenderError::InvalidImageData`] when `pixels` is not four halves per pixel.
185    pub fn upload_linear_rgba16f(
186        &self,
187        size: RenderSize,
188        pixels: &[u16],
189    ) -> Result<crate::UploadTexture, RenderError> {
190        // Avoids a frame-sized copy per frame on little-endian targets.
191        let bytes: std::borrow::Cow<'_, [u8]> = if cfg!(target_endian = "little") {
192            std::borrow::Cow::Borrowed(bytemuck::cast_slice(pixels))
193        } else {
194            std::borrow::Cow::Owned(pixels.iter().flat_map(|half| half.to_le_bytes()).collect())
195        };
196        self.write_linear_rgba16f(
197            size,
198            crate::SourcePlane {
199                bytes: &bytes,
200                stride: size
201                    .width
202                    .checked_mul(RGBA16F_BYTES_PER_PIXEL)
203                    .ok_or(RenderError::InvalidImageData)?,
204            },
205        )
206    }
207
208    /// Writes strided little-endian RGBA half-float rows into a pooled upload texture.
209    ///
210    /// # Errors
211    /// [`RenderError::InvalidImageData`] for a plane shorter than `size`.
212    pub fn write_linear_rgba16f(
213        &self,
214        size: RenderSize,
215        plane: crate::SourcePlane<'_>,
216    ) -> Result<crate::UploadTexture, RenderError> {
217        self.validate_size(size)?;
218        let texture = self.upload_textures.acquire(
219            self.gpu.device(),
220            size,
221            wgpu::TextureFormat::Rgba16Float,
222            "alpha2 linear RGBA upload texture",
223        );
224        self.write_plane(&texture, size, plane, RGBA16F_BYTES_PER_PIXEL)?;
225        Ok(texture)
226    }
227
228    pub(crate) fn write_rgba8(
229        &self,
230        size: RenderSize,
231        plane: crate::SourcePlane<'_>,
232    ) -> Result<crate::UploadTexture, RenderError> {
233        let texture = self.acquire_rgba8_upload_texture(size)?;
234        self.write_plane(&texture, size, plane, RGBA8_BYTES_PER_PIXEL)?;
235        Ok(texture)
236    }
237
238    fn write_plane(
239        &self,
240        texture: &crate::UploadTexture,
241        size: RenderSize,
242        plane: crate::SourcePlane<'_>,
243        bytes_per_pixel: u32,
244    ) -> Result<(), RenderError> {
245        let row_bytes = size
246            .width
247            .checked_mul(bytes_per_pixel)
248            .ok_or(RenderError::InvalidImageData)?;
249        let required = usize::try_from(plane.stride)
250            .ok()
251            .zip(usize::try_from(size.height).ok())
252            .and_then(|(stride, rows)| stride.checked_mul(rows));
253        if plane.stride < row_bytes || required != Some(plane.bytes.len()) {
254            return Err(RenderError::InvalidImageData);
255        }
256        self.gpu.queue().write_texture(
257            wgpu::TexelCopyTextureInfo {
258                texture: texture.texture(),
259                mip_level: 0,
260                origin: wgpu::Origin3d::ZERO,
261                aspect: wgpu::TextureAspect::All,
262            },
263            plane.bytes,
264            wgpu::TexelCopyBufferLayout {
265                offset: 0,
266                bytes_per_row: Some(plane.stride),
267                rows_per_image: Some(size.height),
268            },
269            wgpu::Extent3d {
270                width: size.width,
271                height: size.height,
272                depth_or_array_layers: 1,
273            },
274        );
275        Ok(())
276    }
277
278    /// Copies one RGBA8 render result into tightly packed premultiplied host memory.
279    ///
280    /// Blocks on the GPU, so native and test only; callers that cannot block use
281    /// [`Renderer::begin_rgba8_readback`]. Half-float intermediates are quantized to RGBA8 on the
282    /// GPU first.
283    ///
284    /// # Errors
285    /// [`RenderError::TextureReadback`] for another target format or a failed mapping.
286    #[cfg(not(target_arch = "wasm32"))]
287    pub fn read_rgba8(&self, target: &RenderedTexture) -> Result<Vec<u8>, RenderError> {
288        self.begin_rgba8_readback(target)?.wait()
289    }
290
291    /// Submits a readback copy and returns an unmapped handle.
292    ///
293    /// The copy is queued on return, so mapping can resolve on an event loop via
294    /// [`Rgba8Readback::map_async`]. Used for stills, thumbnails and goldens; exports use YUV or
295    /// NV12 readbacks instead.
296    ///
297    /// # Errors
298    /// [`RenderError::TextureReadback`] for an unsupported format or unrepresentable row layout.
299    pub fn begin_rgba8_readback(
300        &self,
301        target: &RenderedTexture,
302    ) -> Result<Rgba8Readback, RenderError> {
303        self.ensure_gpu_runs_work()?;
304        self.validate_size(target.size)?;
305        if !matches!(
306            target.format,
307            wgpu::TextureFormat::Rgba8Unorm | wgpu::TextureFormat::Rgba16Float
308        ) {
309            return Err(RenderError::TextureReadback(
310                ReadbackFailure::UnsupportedTarget,
311            ));
312        }
313
314        let unpadded_bytes_per_row = target
315            .size
316            .width
317            .checked_mul(RGBA8_BYTES_PER_PIXEL)
318            .ok_or(RenderError::TextureReadback(ReadbackFailure::Layout))?;
319        let alignment = wgpu::COPY_BYTES_PER_ROW_ALIGNMENT;
320        let padded_bytes_per_row = unpadded_bytes_per_row
321            .checked_add(alignment - 1)
322            .map(|value| value / alignment)
323            .and_then(|rows| rows.checked_mul(alignment))
324            .ok_or(RenderError::TextureReadback(ReadbackFailure::Layout))?;
325        let image_bytes = u64::from(padded_bytes_per_row)
326            .checked_mul(u64::from(target.size.height))
327            .ok_or(RenderError::TextureReadback(ReadbackFailure::Layout))?;
328        // Rows, then the canary.
329        let buffer_size = image_bytes
330            .checked_add(CANARY_BYTES)
331            .ok_or(RenderError::TextureReadback(ReadbackFailure::Layout))?;
332        let mut encoder = self.command_encoder("alpha2 RGBA8 readback encoder");
333        let buffer =
334            self.readback_buffers
335                .acquire(self.gpu.device(), buffer_size, "alpha2 RGBA8 readback");
336        let packed_texture = self.encode_readback_pack(target, &mut encoder)?;
337        let copy_source = packed_texture
338            .as_ref()
339            .map_or(&target.texture, TextureLease::texture);
340        encoder.copy_texture_to_buffer(
341            wgpu::TexelCopyTextureInfo {
342                texture: copy_source,
343                mip_level: 0,
344                origin: wgpu::Origin3d::ZERO,
345                aspect: wgpu::TextureAspect::All,
346            },
347            wgpu::TexelCopyBufferInfo {
348                buffer: buffer.buffer(),
349                layout: wgpu::TexelCopyBufferLayout {
350                    offset: 0,
351                    bytes_per_row: Some(padded_bytes_per_row),
352                    rows_per_image: Some(target.size.height),
353                },
354            },
355            wgpu::Extent3d {
356                width: target.size.width,
357                height: target.size.height,
358                depth_or_array_layers: 1,
359            },
360        );
361        // Same encoder, after the copy, so it lands only if the copy ran.
362        let canary = self
363            .canaries
364            .encode(&self.gpu, &mut encoder, buffer.buffer(), image_bytes)?;
365        let command_buffer = encoder.finish();
366        #[cfg(not(target_arch = "wasm32"))]
367        let submission = self.submit([command_buffer])?;
368        #[cfg(target_arch = "wasm32")]
369        self.submit([command_buffer])?;
370        let render_passes = u64::from(packed_texture.is_some());
371        let copy_bytes =
372            u64::from(unpadded_bytes_per_row).saturating_mul(u64::from(target.size.height));
373        crate::diagnostics::record_readback_work(
374            &self.diagnostics,
375            render_passes,
376            1,
377            copy_bytes,
378            true,
379        );
380        Ok(Rgba8Readback {
381            buffer,
382            gpu: self.gpu.clone(),
383            size: target.size,
384            padded_bytes_per_row,
385            unpadded_bytes_per_row,
386            #[cfg(not(target_arch = "wasm32"))]
387            submission,
388            #[cfg(not(target_arch = "wasm32"))]
389            timeout: self.gpu_wait_timeout,
390            staging_bytes: buffer_size,
391            image_bytes,
392            canary,
393            mapping_requested: AtomicBool::new(false),
394        })
395    }
396
397    /// Converts a render result to 8-bit planar YUV420 on the GPU and submits an async readback.
398    ///
399    /// The size must be even. Chroma is filtered for the siting `encoding.chroma` names
400    /// ([`alpha2_media::VideoCodec::chroma_location`]).
401    ///
402    /// # Errors
403    /// [`RenderError::TextureReadback`] for unsupported formats, odd dimensions or unrepresentable
404    /// layouts.
405    pub fn begin_yuv420p_readback(
406        &self,
407        target: &RenderedTexture,
408        encoding: YuvEncoding,
409    ) -> Result<Yuv420pReadback, RenderError> {
410        let mut command_encoder = self.command_encoder("alpha2 YUV420P readback encoder");
411        let readback_work = self.encode_yuv420p_readback(target, encoding, &mut command_encoder)?;
412        readback_work.flush_uniforms(self.gpu.queue());
413        #[cfg(not(target_arch = "wasm32"))]
414        let submission = self.submit([command_encoder.finish()])?;
415        #[cfg(target_arch = "wasm32")]
416        self.submit([command_encoder.finish()])?;
417        Ok(self.complete_yuv420p_readback(
418            readback_work,
419            #[cfg(not(target_arch = "wasm32"))]
420            submission,
421            true,
422        ))
423    }
424
425    /// Reads a render result back as stored and packs it into `layout` on the CPU
426    /// ([`Yuv420pReadback::finish_into_planes`]), for outputs other than 8-bit 4:2:0.
427    ///
428    /// 10-bit output requires the `rgba16float` intermediate.
429    ///
430    /// # Errors
431    /// [`RenderError::TextureReadback`] for an unsupported format, a 10-bit layout over an 8-bit
432    /// target, or an unrepresentable row layout.
433    pub fn begin_layout_readback(
434        &self,
435        target: &RenderedTexture,
436        encoding: YuvEncoding,
437        layout: PixelLayout,
438    ) -> Result<Yuv420pReadback, RenderError> {
439        self.ensure_gpu_runs_work()?;
440        self.validate_size(target.size)?;
441        let texel_bytes = match target.format {
442            wgpu::TextureFormat::Rgba16Float => 8,
443            wgpu::TextureFormat::Rgba8Unorm
444                if layout.bit_depth == alpha2_media::BitDepth::Eight =>
445            {
446                4
447            }
448            _ => {
449                return Err(RenderError::TextureReadback(
450                    ReadbackFailure::UnsupportedTarget,
451                ));
452            }
453        };
454        let (pitch, buffer_size) = texel_staging(target.size, texel_bytes)?;
455        let image_bytes = buffer_size - CANARY_BYTES;
456        let row_bytes = target.size.width * texel_bytes;
457        let mut encoder = self.command_encoder("alpha2 texel readback encoder");
458        let buffer =
459            self.readback_buffers
460                .acquire(self.gpu.device(), buffer_size, "alpha2 texel readback");
461        encoder.copy_texture_to_buffer(
462            wgpu::TexelCopyTextureInfo {
463                texture: &target.texture,
464                mip_level: 0,
465                origin: wgpu::Origin3d::ZERO,
466                aspect: wgpu::TextureAspect::All,
467            },
468            wgpu::TexelCopyBufferInfo {
469                buffer: buffer.buffer(),
470                layout: wgpu::TexelCopyBufferLayout {
471                    offset: 0,
472                    bytes_per_row: Some(pitch),
473                    rows_per_image: Some(target.size.height),
474                },
475            },
476            wgpu::Extent3d {
477                width: target.size.width,
478                height: target.size.height,
479                depth_or_array_layers: 1,
480            },
481        );
482        let canary = self
483            .canaries
484            .encode(&self.gpu, &mut encoder, buffer.buffer(), image_bytes)?;
485        #[cfg(not(target_arch = "wasm32"))]
486        let submission = self.submit([encoder.finish()])?;
487        #[cfg(target_arch = "wasm32")]
488        self.submit([encoder.finish()])?;
489        crate::diagnostics::record_readback_work(
490            &self.diagnostics,
491            0,
492            1,
493            u64::from(row_bytes).saturating_mul(u64::from(target.size.height)),
494            true,
495        );
496        Ok(Yuv420pReadback {
497            buffer,
498            gpu: self.gpu.clone(),
499            size: target.size,
500            layout: Yuv420StagingLayout {
501                luma_pitch: 0,
502                chroma_pitch: 0,
503                u_offset: 0,
504                v_offset: 0,
505                bytes: 0,
506            },
507            texels: Some(Box::new(TexelStaging {
508                format: target.format,
509                pitch,
510                bytes: image_bytes,
511                layout,
512                encoding,
513            })),
514            #[cfg(not(target_arch = "wasm32"))]
515            submission,
516            #[cfg(not(target_arch = "wasm32"))]
517            timeout: self.gpu_wait_timeout,
518            staging_bytes: buffer_size,
519            canary,
520            mapping_requested: AtomicBool::new(false),
521        })
522    }
523
524    pub(crate) fn encode_yuv420p_readback(
525        &self,
526        target: &RenderedTexture,
527        encoding: YuvEncoding,
528        encoder: &mut wgpu::CommandEncoder,
529    ) -> Result<EncodedYuv420pReadback, RenderError> {
530        self.ensure_gpu_runs_work()?;
531        self.validate_size(target.size)?;
532        if !matches!(
533            target.format,
534            wgpu::TextureFormat::Rgba8Unorm | wgpu::TextureFormat::Rgba16Float
535        ) || !target.size.width.is_multiple_of(2)
536            || !target.size.height.is_multiple_of(2)
537        {
538            return Err(RenderError::TextureReadback(
539                ReadbackFailure::UnsupportedTarget,
540            ));
541        }
542        let layout = Yuv420StagingLayout::new(target.size)?;
543        let uv_rows = target.size.height / 2;
544        let chroma_width = target.size.width / 2;
545        let buffer_size = layout.bytes + CANARY_BYTES;
546        let buffer = self.readback_buffers.acquire(
547            self.gpu.device(),
548            buffer_size,
549            "alpha2 YUV420P readback",
550        );
551        let y_texture = self.textures.acquire(
552            self.gpu.device(),
553            target.size,
554            wgpu::TextureFormat::R8Unorm,
555            "alpha2 GPU Y readback texture",
556        );
557        let uv_size = RenderSize {
558            width: chroma_width,
559            height: uv_rows,
560        };
561        let u_texture = self.textures.acquire(
562            self.gpu.device(),
563            uv_size,
564            wgpu::TextureFormat::R8Unorm,
565            "alpha2 GPU U readback texture",
566        );
567        let v_texture = self.textures.acquire(
568            self.gpu.device(),
569            uv_size,
570            wgpu::TextureFormat::R8Unorm,
571            "alpha2 GPU V readback texture",
572        );
573        let uniform_bytes = f32_bytes(yuv_uniform(encoding));
574        let mut uniforms = self.uniform_buffers.acquire(
575            self.gpu.device(),
576            u64::try_from(uniform_bytes.len()).unwrap_or(u64::MAX),
577        );
578        let uniform = uniforms.write(&uniform_bytes);
579        self.encode_yuv420_pack(
580            target,
581            [y_texture.view(), u_texture.view(), v_texture.view()],
582            encoder,
583            &uniforms,
584            uniform,
585        )?;
586        encode_plane_copy(
587            encoder,
588            y_texture.texture(),
589            buffer.buffer(),
590            0,
591            layout.luma_pitch,
592            target.size,
593        );
594        encode_plane_copy(
595            encoder,
596            u_texture.texture(),
597            buffer.buffer(),
598            layout.u_offset,
599            layout.chroma_pitch,
600            uv_size,
601        );
602        encode_plane_copy(
603            encoder,
604            v_texture.texture(),
605            buffer.buffer(),
606            layout.v_offset,
607            layout.chroma_pitch,
608            uv_size,
609        );
610        // Same encoder, after the plane copies. Not counted as a copy in diagnostics.
611        let canary = self
612            .canaries
613            .encode(&self.gpu, encoder, buffer.buffer(), layout.bytes)?;
614        Ok(EncodedYuv420pReadback {
615            buffer,
616            size: target.size,
617            layout,
618            staging_bytes: buffer_size,
619            canary,
620            y_texture,
621            u_texture,
622            v_texture,
623            uniforms,
624        })
625    }
626
627    pub(crate) fn complete_yuv420p_readback(
628        &self,
629        encoded: EncodedYuv420pReadback,
630        #[cfg(not(target_arch = "wasm32"))] submission: wgpu::SubmissionIndex,
631        standalone_submission: bool,
632    ) -> Yuv420pReadback {
633        let EncodedYuv420pReadback {
634            buffer,
635            size,
636            layout,
637            staging_bytes,
638            canary,
639            y_texture,
640            u_texture,
641            v_texture,
642            uniforms,
643        } = encoded;
644        uniforms.recycle();
645        drop((y_texture, u_texture, v_texture));
646        let texture_bytes = yuv420_tight_bytes(size).expect("validated YUV420 dimensions");
647        crate::diagnostics::record_readback_work(
648            &self.diagnostics,
649            2,
650            3,
651            texture_bytes,
652            standalone_submission,
653        );
654        Yuv420pReadback {
655            buffer,
656            gpu: self.gpu.clone(),
657            size,
658            layout,
659            texels: None,
660            #[cfg(not(target_arch = "wasm32"))]
661            submission,
662            #[cfg(not(target_arch = "wasm32"))]
663            timeout: self.gpu_wait_timeout,
664            staging_bytes,
665            canary,
666            mapping_requested: AtomicBool::new(false),
667        }
668    }
669
670    /// Packs one rendered frame into caller-owned NV12 plane textures.
671    ///
672    /// Same conversion as [`Self::begin_yuv420p_readback`] without the readback. wgpu allows only
673    /// `TEXTURE_BINDING` on `TextureFormat::NV12`, so the native layer copies these `R8Unorm` and
674    /// `Rg8Unorm` targets into the encoder's image afterwards.
675    ///
676    /// # Errors
677    /// [`RenderError::TextureReadback`] for an unaccepted target format or size,
678    /// [`RenderError::InvalidPlaneTarget`] for mismatched plane textures, and
679    /// [`RenderError::GpuWorkDropped`] once the device was found dropping work.
680    #[cfg_attr(not(feature = "wgpu-interop"), doc(hidden))]
681    pub fn encode_nv12_pack(
682        &self,
683        target: &RenderedTexture,
684        planes: Nv12PlaneTargets<'_>,
685        encoding: YuvEncoding,
686        encoder: &mut wgpu::CommandEncoder,
687    ) -> Result<EncodedNv12Pack, RenderError> {
688        // A pack has no canary of its own; an earlier failed readback canary applies.
689        self.ensure_gpu_runs_work()?;
690        let encode = self.open_encode();
691        self.validate_size(target.size)?;
692        if !matches!(
693            target.format,
694            wgpu::TextureFormat::Rgba8Unorm | wgpu::TextureFormat::Rgba16Float
695        ) || !target.size.width.is_multiple_of(2)
696            || !target.size.height.is_multiple_of(2)
697        {
698            return Err(RenderError::TextureReadback(
699                ReadbackFailure::UnsupportedTarget,
700            ));
701        }
702        let chroma_size = RenderSize {
703            width: target.size.width / 2,
704            height: target.size.height / 2,
705        };
706        // Planes may belong to a hardware encoder, so check them here rather than hit a validation
707        // error.
708        let attachment = |texture: &wgpu::Texture| {
709            texture
710                .usage()
711                .contains(wgpu::TextureUsages::RENDER_ATTACHMENT)
712                && texture.mip_level_count() == 1
713                && texture.sample_count() == 1
714                && texture.depth_or_array_layers() == 1
715        };
716        if planes.luma.format() != wgpu::TextureFormat::R8Unorm
717            || planes.chroma.format() != wgpu::TextureFormat::Rg8Unorm
718            || planes.luma.width() != target.size.width
719            || planes.luma.height() != target.size.height
720            || planes.chroma.width() != chroma_size.width
721            || planes.chroma.height() != chroma_size.height
722            || !attachment(planes.luma)
723            || !attachment(planes.chroma)
724        {
725            return Err(RenderError::InvalidPlaneTarget);
726        }
727        let uniform_bytes = f32_bytes(yuv_uniform(encoding));
728        let mut uniforms = self.uniform_buffers.acquire(
729            self.gpu.device(),
730            u64::try_from(uniform_bytes.len()).unwrap_or(u64::MAX),
731        );
732        let uniform = uniforms.write(&uniform_bytes);
733        let cached = self.yuv_readback_pipeline()?;
734        let bind_group = self.yuv_pack_bind_group(target, cached, &uniforms, uniform);
735        let luma_view = crate::pool::default_view(planes.luma);
736        encode_yuv_render_pass(
737            encoder,
738            "alpha2 GPU NV12 Y pack pass",
739            &cached.y_pipeline,
740            &bind_group,
741            &luma_view,
742        );
743        let chroma_view = crate::pool::default_view(planes.chroma);
744        encode_yuv_render_pass(
745            encoder,
746            "alpha2 GPU NV12 UV pack pass",
747            &cached.uv_nv12_pipeline,
748            &bind_group,
749            &chroma_view,
750        );
751        // `write` fills only the host copy; the shader reads stale data without the flush.
752        uniforms.flush(self.gpu.queue());
753        // Keep the uniforms until the caller submits `encoder`, or the next call could overwrite
754        // them.
755        Ok(EncodedNv12Pack {
756            uniforms: Some(uniforms),
757            _encode: encode,
758        })
759    }
760
761    fn yuv_pack_bind_group(
762        &self,
763        target: &RenderedTexture,
764        cached: &CachedYuvReadbackPipeline,
765        uniforms: &crate::uniform::UniformArenaLease,
766        uniform: crate::uniform::UniformSlice,
767    ) -> wgpu::BindGroup {
768        self.gpu
769            .device()
770            .create_bind_group(&wgpu::BindGroupDescriptor {
771                label: Some("alpha2 RGB to YUV bind group"),
772                layout: &cached.bind_group_layout,
773                entries: &[
774                    wgpu::BindGroupEntry {
775                        binding: 0,
776                        resource: wgpu::BindingResource::TextureView(target.view()),
777                    },
778                    wgpu::BindGroupEntry {
779                        binding: 1,
780                        resource: uniforms.binding(uniform),
781                    },
782                ],
783            })
784    }
785
786    fn encode_yuv420_pack(
787        &self,
788        target: &RenderedTexture,
789        plane_views: [&wgpu::TextureView; 3],
790        encoder: &mut wgpu::CommandEncoder,
791        uniforms: &crate::uniform::UniformArenaLease,
792        uniform: crate::uniform::UniformSlice,
793    ) -> Result<(), RenderError> {
794        let cached = self.yuv_readback_pipeline()?;
795        let bind_group = self.yuv_pack_bind_group(target, cached, uniforms, uniform);
796        let [y_view, u_view, v_view] = plane_views;
797        encode_yuv_render_pass(
798            encoder,
799            "alpha2 GPU Y pack pass",
800            &cached.y_pipeline,
801            &bind_group,
802            y_view,
803        );
804        encode_yuv_render_pass_pair(
805            encoder,
806            "alpha2 GPU UV pack pass",
807            &cached.uv_pipeline,
808            &bind_group,
809            [u_view, v_view],
810        );
811        Ok(())
812    }
813
814    fn yuv_readback_pipeline(&self) -> Result<&CachedYuvReadbackPipeline, RenderError> {
815        crate::gpu_errors::checked_cell(&self.pipelines.yuv_readback, self.gpu.device(), || {
816            let bind_group_layout =
817                self.gpu
818                    .device()
819                    .create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
820                        label: Some("alpha2 RGB to YUV layout"),
821                        entries: &[
822                            wgpu::BindGroupLayoutEntry {
823                                binding: 0,
824                                visibility: wgpu::ShaderStages::FRAGMENT,
825                                ty: wgpu::BindingType::Texture {
826                                    sample_type: wgpu::TextureSampleType::Float {
827                                        filterable: false,
828                                    },
829                                    view_dimension: wgpu::TextureViewDimension::D2,
830                                    multisampled: false,
831                                },
832                                count: None,
833                            },
834                            wgpu::BindGroupLayoutEntry {
835                                binding: 1,
836                                visibility: wgpu::ShaderStages::FRAGMENT,
837                                ty: wgpu::BindingType::Buffer {
838                                    ty: wgpu::BufferBindingType::Uniform,
839                                    has_dynamic_offset: false,
840                                    min_binding_size: None,
841                                },
842                                count: None,
843                            },
844                        ],
845                    });
846            let shader = self
847                .gpu
848                .device()
849                .create_shader_module(wgpu::ShaderModuleDescriptor {
850                    label: Some("alpha2 RGB to YUV shader"),
851                    source: wgpu::ShaderSource::Wgsl(RGB_TO_YUV420_WGSL.into()),
852                });
853            let layout =
854                self.gpu
855                    .device()
856                    .create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
857                        label: Some("alpha2 RGB to YUV pipeline layout"),
858                        bind_group_layouts: &[Some(&bind_group_layout)],
859                        immediate_size: 0,
860                    });
861            CachedYuvReadbackPipeline {
862                y_pipeline: yuv_render_pipeline(
863                    self.gpu.device(),
864                    &layout,
865                    &shader,
866                    "y_fs",
867                    &[wgpu::TextureFormat::R8Unorm],
868                ),
869                uv_pipeline: yuv_render_pipeline(
870                    self.gpu.device(),
871                    &layout,
872                    &shader,
873                    "uv_fs",
874                    &[wgpu::TextureFormat::R8Unorm, wgpu::TextureFormat::R8Unorm],
875                ),
876                uv_nv12_pipeline: yuv_render_pipeline(
877                    self.gpu.device(),
878                    &layout,
879                    &shader,
880                    "uv_nv12_fs",
881                    &[wgpu::TextureFormat::Rg8Unorm],
882                ),
883                bind_group_layout,
884            }
885        })
886    }
887
888    fn encode_readback_pack(
889        &self,
890        target: &RenderedTexture,
891        encoder: &mut wgpu::CommandEncoder,
892    ) -> Result<Option<TextureLease>, RenderError> {
893        if target.format != wgpu::TextureFormat::Rgba16Float {
894            return Ok(None);
895        }
896        let texture = self.textures.acquire(
897            self.gpu.device(),
898            target.size,
899            wgpu::TextureFormat::Rgba8Unorm,
900            "alpha2 GPU-packed RGBA8 readback texture",
901        );
902        let cached = self.readback_pack_pipeline()?;
903        let bind_group = self
904            .gpu
905            .device()
906            .create_bind_group(&wgpu::BindGroupDescriptor {
907                label: Some("alpha2 readback pack bind group"),
908                layout: &cached.bind_group_layout,
909                entries: &[wgpu::BindGroupEntry {
910                    binding: 0,
911                    resource: wgpu::BindingResource::TextureView(target.view()),
912                }],
913            });
914        {
915            let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
916                label: Some("alpha2 GPU RGBA8 readback pack pass"),
917                color_attachments: &[Some(wgpu::RenderPassColorAttachment {
918                    view: texture.view(),
919                    depth_slice: None,
920                    resolve_target: None,
921                    ops: wgpu::Operations {
922                        load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT),
923                        store: wgpu::StoreOp::Store,
924                    },
925                })],
926                depth_stencil_attachment: None,
927                timestamp_writes: None,
928                occlusion_query_set: None,
929                multiview_mask: None,
930            });
931            pass.set_pipeline(&cached.pipeline);
932            pass.set_bind_group(0, &bind_group, &[]);
933            pass.draw(0..3, 0..1);
934        }
935        Ok(Some(texture))
936    }
937
938    fn readback_pack_pipeline(&self) -> Result<&CachedPipeline, RenderError> {
939        crate::gpu_errors::checked_cell(&self.pipelines.readback_pack, self.gpu.device(), || {
940            let bind_group_layout =
941                self.gpu
942                    .device()
943                    .create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
944                        label: Some("alpha2 readback pack layout"),
945                        entries: &[wgpu::BindGroupLayoutEntry {
946                            binding: 0,
947                            visibility: wgpu::ShaderStages::FRAGMENT,
948                            ty: wgpu::BindingType::Texture {
949                                sample_type: wgpu::TextureSampleType::Float { filterable: false },
950                                view_dimension: wgpu::TextureViewDimension::D2,
951                                multisampled: false,
952                            },
953                            count: None,
954                        }],
955                    });
956            let shader = self
957                .gpu
958                .device()
959                .create_shader_module(wgpu::ShaderModuleDescriptor {
960                    label: Some("alpha2 readback pack shader"),
961                    source: wgpu::ShaderSource::Wgsl(READBACK_PACK_WGSL.into()),
962                });
963            let layout =
964                self.gpu
965                    .device()
966                    .create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
967                        label: Some("alpha2 readback pack pipeline layout"),
968                        bind_group_layouts: &[Some(&bind_group_layout)],
969                        immediate_size: 0,
970                    });
971            let pipeline =
972                self.gpu
973                    .device()
974                    .create_render_pipeline(&wgpu::RenderPipelineDescriptor {
975                        label: Some("alpha2 readback pack pipeline"),
976                        layout: Some(&layout),
977                        vertex: wgpu::VertexState {
978                            module: &shader,
979                            entry_point: Some("vs"),
980                            buffers: &[],
981                            compilation_options: wgpu::PipelineCompilationOptions::default(),
982                        },
983                        fragment: Some(wgpu::FragmentState {
984                            module: &shader,
985                            entry_point: Some("fs"),
986                            targets: &[Some(wgpu::TextureFormat::Rgba8Unorm.into())],
987                            compilation_options: wgpu::PipelineCompilationOptions::default(),
988                        }),
989                        primitive: wgpu::PrimitiveState::default(),
990                        depth_stencil: None,
991                        multisample: wgpu::MultisampleState::default(),
992                        multiview_mask: None,
993                        cache: None,
994                    });
995            CachedPipeline {
996                bind_group_layout,
997                pipeline,
998            }
999        })
1000    }
1001}
1002
1003/// A submitted texture readback whose buffer has not been mapped yet.
1004///
1005/// Owns its staging buffer, so it can move into an async browser task.
1006#[derive(Debug)]
1007pub struct Rgba8Readback {
1008    buffer: ReadbackBufferLease,
1009    gpu: GpuContext,
1010    size: RenderSize,
1011    padded_bytes_per_row: u32,
1012    unpadded_bytes_per_row: u32,
1013    #[cfg(not(target_arch = "wasm32"))]
1014    submission: wgpu::SubmissionIndex,
1015    #[cfg(not(target_arch = "wasm32"))]
1016    timeout: std::time::Duration,
1017    staging_bytes: u64,
1018    image_bytes: u64,
1019    canary: Canary,
1020    mapping_requested: AtomicBool,
1021}
1022
1023impl Rgba8Readback {
1024    /// Test-only: expect a different canary, simulating a stale staging buffer.
1025    #[cfg(test)]
1026    pub(crate) fn expecting(mut self, canary: Canary) -> Self {
1027        self.canary = canary;
1028        self
1029    }
1030
1031    /// The pixel dimensions of the result.
1032    #[must_use]
1033    pub const fn size(&self) -> RenderSize {
1034        self.size
1035    }
1036
1037    /// Bytes held by the staging buffer.
1038    #[must_use]
1039    pub const fn staging_bytes(&self) -> u64 {
1040        self.staging_bytes
1041    }
1042
1043    /// Blocks a native caller until this readback is mapped, without draining the whole queue.
1044    ///
1045    /// # Errors
1046    /// [`RenderError::TextureReadback`] when mapping or device polling fails.
1047    #[cfg(not(target_arch = "wasm32"))]
1048    pub fn wait_ready(&self) -> Result<(), RenderError> {
1049        let (sender, receiver) = mpsc::channel();
1050        self.map_async(move |mapped| {
1051            let _ = sender.send(mapped);
1052        });
1053        self.gpu
1054            .device()
1055            .poll(wgpu::PollType::Wait {
1056                submission_index: Some(self.submission.clone()),
1057                timeout: Some(self.timeout),
1058            })
1059            .map_err(crate::error::poll_error)?;
1060        receive_mapped(&self.gpu, &receiver, self.timeout)?
1061    }
1062
1063    /// Waits for this readback and returns tightly packed RGBA8 pixels.
1064    ///
1065    /// # Errors
1066    /// [`RenderError::TextureReadback`] when mapping, polling or row packing fails.
1067    #[cfg(not(target_arch = "wasm32"))]
1068    pub fn wait(self) -> Result<Vec<u8>, RenderError> {
1069        self.wait_ready()?;
1070        self.finish()
1071    }
1072
1073    /// Requests buffer mapping and reports readiness exactly once.
1074    ///
1075    /// A refused or repeated request reports a failure, which callers must treat as a failed frame.
1076    pub fn map_async(&self, on_ready: impl FnOnce(Result<(), RenderError>) + Send + 'static) {
1077        map_staging(&self.gpu, &self.buffer, &self.mapping_requested, on_ready);
1078    }
1079
1080    /// Polls backends that need it to resolve a mapping; browser `WebGPU` ignores this.
1081    pub fn poll(&self) {
1082        let _ = self.gpu.device().poll(wgpu::PollType::Poll);
1083    }
1084
1085    /// Packs the mapped rows into tightly packed premultiplied RGBA8.
1086    ///
1087    /// # Errors
1088    /// [`RenderError::TextureReadback`] when the buffer is not mapped, and
1089    /// [`RenderError::GpuWorkDropped`] when the GPU did not run the copy; the buffer is then not
1090    /// recycled and the renderer refuses further work.
1091    pub fn finish(self) -> Result<Vec<u8>, RenderError> {
1092        let Ok(view) = self.buffer.buffer().slice(..).get_mapped_range() else {
1093            // Unmapping an unmapped buffer is a validation error. Dropping the lease destroys the
1094            // buffer and ends any pending mapping.
1095            return Err(RenderError::TextureReadback(ReadbackFailure::NotMapped));
1096        };
1097        let image_bytes = usize::try_from(self.image_bytes)
1098            .map_err(|_| RenderError::TextureReadback(ReadbackFailure::Layout))?;
1099        let checked = view
1100            .get(image_bytes..)
1101            .ok_or(RenderError::TextureReadback(ReadbackFailure::Layout))
1102            .and_then(|found| self.canary.check(found));
1103        if let Err(error) = checked {
1104            drop(view);
1105            self.buffer.buffer().unmap();
1106            return Err(error);
1107        }
1108        let mapped = &view[..image_bytes];
1109        let padded_row = usize::try_from(self.padded_bytes_per_row)
1110            .map_err(|_| RenderError::TextureReadback(ReadbackFailure::Layout))?;
1111        let unpadded_row = usize::try_from(self.unpadded_bytes_per_row)
1112            .map_err(|_| RenderError::TextureReadback(ReadbackFailure::Layout))?;
1113        let packed = if padded_row == unpadded_row {
1114            mapped.to_vec()
1115        } else {
1116            let capacity = unpadded_row
1117                .checked_mul(
1118                    usize::try_from(self.size.height)
1119                        .map_err(|_| RenderError::TextureReadback(ReadbackFailure::Layout))?,
1120                )
1121                .ok_or(RenderError::TextureReadback(ReadbackFailure::Layout))?;
1122            let mut packed = Vec::with_capacity(capacity);
1123            for row in mapped.chunks_exact(padded_row) {
1124                packed.extend_from_slice(&row[..unpadded_row]);
1125            }
1126            packed
1127        };
1128        drop(view);
1129        self.buffer.buffer().unmap();
1130        self.buffer.recycle();
1131        Ok(packed)
1132    }
1133}
1134
1135/// GPU state an NV12 pack recorded into a caller's encoder still needs.
1136///
1137/// Drop it only after that encoder is submitted; dropping earlier lets a later renderer call
1138/// overwrite the pack's uniforms.
1139#[must_use = "hold the pack until its encoder is submitted"]
1140#[derive(Debug)]
1141#[cfg_attr(not(feature = "wgpu-interop"), doc(hidden))]
1142pub struct EncodedNv12Pack {
1143    uniforms: Option<crate::uniform::UniformArenaLease>,
1144    /// Defers destruction of released pool resources until the pack is dropped.
1145    _encode: crate::retire::EncodeScope,
1146}
1147
1148impl Drop for EncodedNv12Pack {
1149    fn drop(&mut self) {
1150        if let Some(uniforms) = self.uniforms.take() {
1151            uniforms.recycle();
1152        }
1153    }
1154}
1155
1156/// Caller-owned NV12 plane textures for a readback-free pack.
1157///
1158/// `luma` is `R8Unorm` at the output size; `chroma` is `Rg8Unorm` at half size with U in red and
1159/// V in green.
1160#[derive(Clone, Copy, Debug)]
1161#[cfg_attr(not(feature = "wgpu-interop"), doc(hidden))]
1162pub struct Nv12PlaneTargets<'a> {
1163    /// Full-size single-channel luma target.
1164    pub luma: &'a wgpu::Texture,
1165    /// Half-size two-channel interleaved chroma target.
1166    pub chroma: &'a wgpu::Texture,
1167}
1168
1169/// A caller-owned destination for one packed plane.
1170#[derive(Debug)]
1171pub struct PlaneTarget<'a> {
1172    pub(crate) bytes: &'a mut [u8],
1173    pub(crate) stride: usize,
1174}
1175
1176impl<'a> PlaneTarget<'a> {
1177    /// Describes plane memory whose rows start `stride` bytes apart.
1178    #[must_use]
1179    pub fn new(bytes: &'a mut [u8], stride: usize) -> Self {
1180        Self { bytes, stride }
1181    }
1182
1183    /// The plane memory.
1184    #[must_use]
1185    pub fn bytes(&self) -> &[u8] {
1186        self.bytes
1187    }
1188
1189    /// Reborrows the plane memory.
1190    pub fn reborrow(&mut self) -> PlaneTarget<'_> {
1191        PlaneTarget {
1192            bytes: self.bytes,
1193            stride: self.stride,
1194        }
1195    }
1196
1197    pub(crate) fn fits(&self, width: usize, height: usize) -> Result<(), RenderError> {
1198        if self.stride < width {
1199            return Err(RenderError::TextureReadback(
1200                ReadbackFailure::DestinationTooSmall,
1201            ));
1202        }
1203        // The last row needs only its visible width, as `FFmpeg` sizes a frame's final plane.
1204        let needed = self
1205            .stride
1206            .checked_mul(height.saturating_sub(1))
1207            .and_then(|rows| rows.checked_add(width))
1208            .ok_or(RenderError::TextureReadback(
1209                ReadbackFailure::DestinationTooSmall,
1210            ))?;
1211        if self.bytes.len() < needed {
1212            return Err(RenderError::TextureReadback(
1213                ReadbackFailure::DestinationTooSmall,
1214            ));
1215        }
1216        Ok(())
1217    }
1218
1219    fn row(&mut self, index: usize, width: usize) -> Result<&mut [u8], RenderError> {
1220        let start = index
1221            .checked_mul(self.stride)
1222            .ok_or(RenderError::TextureReadback(
1223                ReadbackFailure::DestinationTooSmall,
1224            ))?;
1225        let end = start
1226            .checked_add(width)
1227            .ok_or(RenderError::TextureReadback(
1228                ReadbackFailure::DestinationTooSmall,
1229            ))?;
1230        self.bytes
1231            .get_mut(start..end)
1232            .ok_or(RenderError::TextureReadback(
1233                ReadbackFailure::DestinationTooSmall,
1234            ))
1235    }
1236}
1237
1238/// Destinations for the three planes of one packed frame.
1239#[derive(Debug)]
1240pub struct Yuv420pTargets<'a> {
1241    /// Full-resolution luma.
1242    pub y: PlaneTarget<'a>,
1243    /// Half-resolution blue-difference chroma.
1244    pub u: PlaneTarget<'a>,
1245    /// Half-resolution red-difference chroma.
1246    pub v: PlaneTarget<'a>,
1247}
1248
1249/// Tightly packed planar YUV420 data produced by a GPU readback.
1250#[derive(Clone, Debug, Eq, PartialEq)]
1251pub struct Yuv420pPlanes {
1252    /// Full-resolution luma samples.
1253    pub y: Vec<u8>,
1254    /// Half-width, half-height blue-difference chroma samples.
1255    pub u: Vec<u8>,
1256    /// Half-width, half-height red-difference chroma samples.
1257    pub v: Vec<u8>,
1258}
1259
1260/// A submitted GPU RGB-to-YUV420 conversion whose buffer is not mapped yet.
1261#[derive(Debug)]
1262pub struct Yuv420pReadback {
1263    buffer: ReadbackBufferLease,
1264    gpu: GpuContext,
1265    size: RenderSize,
1266    layout: Yuv420StagingLayout,
1267    /// Set for a texel readback packed on the CPU ([`Renderer::begin_layout_readback`]); `layout`
1268    /// is unused then.
1269    texels: Option<Box<TexelStaging>>,
1270    #[cfg(not(target_arch = "wasm32"))]
1271    submission: wgpu::SubmissionIndex,
1272    #[cfg(not(target_arch = "wasm32"))]
1273    timeout: std::time::Duration,
1274    staging_bytes: u64,
1275    canary: Canary,
1276    mapping_requested: AtomicBool,
1277}
1278
1279impl Yuv420pReadback {
1280    /// The luma-plane dimensions.
1281    #[must_use]
1282    pub const fn size(&self) -> RenderSize {
1283        self.size
1284    }
1285
1286    /// Bytes held by the staging buffer.
1287    #[must_use]
1288    pub const fn staging_bytes(&self) -> u64 {
1289        self.staging_bytes
1290    }
1291
1292    /// Blocks a native caller until this readback is mapped.
1293    ///
1294    /// # Errors
1295    /// Returns [`RenderError::TextureReadback`] when mapping or polling fails.
1296    #[cfg(not(target_arch = "wasm32"))]
1297    pub fn wait_ready(&self) -> Result<(), RenderError> {
1298        let (sender, receiver) = mpsc::channel();
1299        self.map_async(move |mapped| {
1300            let _ = sender.send(mapped);
1301        });
1302        self.wait_for_submission()?;
1303        self.wait_for_mapped(&receiver)?
1304    }
1305
1306    /// Waits for what a [`Self::map_async`] callback sends to `receiver`, after
1307    /// [`Self::wait_for_submission`], polling the device, for at most the renderer's wait timeout.
1308    ///
1309    /// # Errors
1310    /// [`RenderError::GpuTimeout`] when nothing arrives in time, and
1311    /// [`ReadbackFailure::CallbackDropped`] when the callback was dropped without sending.
1312    #[cfg(not(target_arch = "wasm32"))]
1313    pub fn wait_for_mapped<T>(&self, receiver: &mpsc::Receiver<T>) -> Result<T, RenderError> {
1314        receive_mapped(&self.gpu, receiver, self.timeout)
1315    }
1316
1317    /// Blocks until this readback's submission completes; may follow an earlier
1318    /// [`Self::map_async`].
1319    ///
1320    /// # Errors
1321    /// [`RenderError::GpuTimeout`] past the renderer's wait timeout, or
1322    /// [`RenderError::TextureReadback`] when device polling fails.
1323    #[cfg(not(target_arch = "wasm32"))]
1324    pub fn wait_for_submission(&self) -> Result<(), RenderError> {
1325        self.gpu
1326            .device()
1327            .poll(wgpu::PollType::Wait {
1328                submission_index: Some(self.submission.clone()),
1329                timeout: Some(self.timeout),
1330            })
1331            .map_err(crate::error::poll_error)?;
1332        Ok(())
1333    }
1334
1335    /// Waits for this readback and returns tightly packed planes.
1336    ///
1337    /// # Errors
1338    /// [`RenderError::TextureReadback`] when mapping, polling or row packing fails.
1339    #[cfg(not(target_arch = "wasm32"))]
1340    pub fn wait(self) -> Result<Yuv420pPlanes, RenderError> {
1341        self.wait_ready()?;
1342        self.finish()
1343    }
1344
1345    /// Requests buffer mapping and reports readiness exactly once.
1346    pub fn map_async(&self, on_ready: impl FnOnce(Result<(), RenderError>) + Send + 'static) {
1347        map_staging(&self.gpu, &self.buffer, &self.mapping_requested, on_ready);
1348    }
1349
1350    /// Polls backends that need it to resolve a mapping.
1351    pub fn poll(&self) {
1352        let _ = self.gpu.device().poll(wgpu::PollType::Poll);
1353    }
1354
1355    /// Packs mapped planes into caller-owned, possibly strided, plane buffers, avoiding the
1356    /// allocation and copy of [`Yuv420pReadback::finish`].
1357    ///
1358    /// # Errors [`RenderError::TextureReadback`] when the buffer is not mapped, the layout is
1359    /// unrepresentable or a target is too small, and [`RenderError::GpuWorkDropped`] when the GPU
1360    /// did not run the copy (the buffer is not recycled and the renderer refuses further work).
1361    pub fn finish_into(self, targets: &mut Yuv420pTargets<'_>) -> Result<(), RenderError> {
1362        if self.texels.is_some() {
1363            return Err(RenderError::TextureReadback(ReadbackFailure::Layout));
1364        }
1365        let Ok(mapped) = self.buffer.buffer().slice(..).get_mapped_range() else {
1366            // Not mapped, so nothing to unmap.
1367            return Err(RenderError::TextureReadback(ReadbackFailure::NotMapped));
1368        };
1369        let result = (|| {
1370            self.check_canary(&mapped)?;
1371            let planes = self.layout.planes(self.size);
1372            let destinations = [&mut targets.y, &mut targets.u, &mut targets.v];
1373            for (plane, target) in planes.into_iter().zip(destinations) {
1374                let (_, _, width, rows) = plane;
1375                let width = usize::try_from(width)
1376                    .map_err(|_| RenderError::TextureReadback(ReadbackFailure::Layout))?;
1377                let rows = usize::try_from(rows)
1378                    .map_err(|_| RenderError::TextureReadback(ReadbackFailure::Layout))?;
1379                target.fits(width, rows)?;
1380                for (index, row) in plane_rows(&mapped, plane)?.enumerate() {
1381                    target.row(index, width)?.copy_from_slice(&row[..width]);
1382                }
1383            }
1384            Ok(())
1385        })();
1386        drop(mapped);
1387        self.buffer.buffer().unmap();
1388        if result.is_ok() {
1389            self.buffer.recycle();
1390        }
1391        result
1392    }
1393
1394    /// Packs mapped planes into three tightly packed planes.
1395    ///
1396    /// # Errors
1397    /// As [`Self::finish_into`].
1398    pub fn finish(self) -> Result<Yuv420pPlanes, RenderError> {
1399        let planes = self.unpack(|[y, u, v]| Yuv420pPlanes { y, u, v })?;
1400        Ok(planes)
1401    }
1402
1403    /// Plane offsets and strides in the bytes [`Self::finish_mapped`] hands over.
1404    #[must_use]
1405    #[cfg_attr(not(feature = "wgpu-interop"), doc(hidden))]
1406    pub fn plane_layout(&self) -> [StagingPlane; 3] {
1407        self.layout
1408            .planes(self.size)
1409            .map(|(offset, stride, _, _)| StagingPlane { offset, stride })
1410    }
1411
1412    /// Hands the mapped staging buffer to `read`, then unmaps it.
1413    ///
1414    /// Planes are at the offsets and strides [`Self::plane_layout`] reports, rows padded to the
1415    /// copy alignment. A strided consumer such as a `WebCodecs` `VideoFrame` can copy straight from
1416    /// the mapping; in a browser that copy can stay in JavaScript (`BufferView::as_uint8array`).
1417    ///
1418    /// # Errors
1419    /// As [`Self::finish_into`]; `read` is not called then.
1420    #[cfg_attr(not(feature = "wgpu-interop"), doc(hidden))]
1421    pub fn finish_mapped<T>(
1422        self,
1423        read: impl FnOnce(&wgpu::BufferView) -> T,
1424    ) -> Result<T, RenderError> {
1425        let Ok(canary) = self
1426            .buffer
1427            .buffer()
1428            .slice(self.image_bytes()..)
1429            .get_mapped_range()
1430        else {
1431            // Not mapped, so nothing to unmap.
1432            return Err(RenderError::TextureReadback(ReadbackFailure::NotMapped));
1433        };
1434        let checked = self.canary.check(&canary);
1435        drop(canary);
1436        if let Err(error) = checked {
1437            self.buffer.buffer().unmap();
1438            return Err(error);
1439        }
1440        let Ok(mapped) = self
1441            .buffer
1442            .buffer()
1443            .slice(..self.image_bytes())
1444            .get_mapped_range()
1445        else {
1446            self.buffer.buffer().unmap();
1447            return Err(RenderError::TextureReadback(ReadbackFailure::NotMapped));
1448        };
1449        let result = read(&mapped);
1450        drop(mapped);
1451        self.buffer.buffer().unmap();
1452        self.buffer.recycle();
1453        Ok(result)
1454    }
1455
1456    /// Copies the three planes out tightly packed and hands them to `pack`.
1457    fn unpack<T>(self, pack: impl FnOnce([Vec<u8>; 3]) -> T) -> Result<T, RenderError> {
1458        if self.texels.is_some() {
1459            return Err(RenderError::TextureReadback(ReadbackFailure::Layout));
1460        }
1461        let Ok(mapped) = self.buffer.buffer().slice(..).get_mapped_range() else {
1462            // Not mapped, so nothing to unmap.
1463            return Err(RenderError::TextureReadback(ReadbackFailure::NotMapped));
1464        };
1465        if let Err(error) = self.check_canary(&mapped) {
1466            drop(mapped);
1467            self.buffer.buffer().unmap();
1468            return Err(error);
1469        }
1470        let planes = self.layout.planes(self.size).map(|plane| {
1471            let (_, _, width, rows) = plane;
1472            let width = usize::try_from(width)
1473                .map_err(|_| RenderError::TextureReadback(ReadbackFailure::Layout))?;
1474            let rows = usize::try_from(rows)
1475                .map_err(|_| RenderError::TextureReadback(ReadbackFailure::Layout))?;
1476            plane_rows(&mapped, plane).map(|source| {
1477                let mut tight = Vec::with_capacity(width * rows);
1478                for row in source {
1479                    tight.extend_from_slice(&row[..width]);
1480                }
1481                tight
1482            })
1483        });
1484        drop(mapped);
1485        self.buffer.buffer().unmap();
1486        let [y, u, v] = planes;
1487        let planes = [y?, u?, v?];
1488        self.buffer.recycle();
1489        Ok(pack(planes))
1490    }
1491
1492    /// Bytes before the canary.
1493    fn image_bytes(&self) -> u64 {
1494        self.texels
1495            .as_ref()
1496            .map_or(self.layout.bytes, |texels| texels.bytes)
1497    }
1498
1499    /// The layout a texel readback packs into, `None` for a GPU-packed 8-bit 4:2:0 readback.
1500    #[must_use]
1501    pub fn pixel_layout(&self) -> Option<PixelLayout> {
1502        self.texels.as_ref().map(|texels| texels.layout)
1503    }
1504
1505    /// Packs the mapped frame into one target per plane of its layout, in storage order
1506    /// ([`PixelLayout::planes`]).
1507    ///
1508    /// # Errors
1509    /// As [`Self::finish_into`], and [`RenderError::TextureReadback`] for a wrong target count.
1510    pub fn finish_into_planes(self, targets: &mut [PlaneTarget<'_>]) -> Result<(), RenderError> {
1511        let Some(texels) = self.texels.as_deref().cloned() else {
1512            let [y, u, v] = targets else {
1513                return Err(RenderError::TextureReadback(ReadbackFailure::Layout));
1514            };
1515            let mut planes = Yuv420pTargets {
1516                y: PlaneTarget::new(y.bytes, y.stride),
1517                u: PlaneTarget::new(u.bytes, u.stride),
1518                v: PlaneTarget::new(v.bytes, v.stride),
1519            };
1520            return self.finish_into(&mut planes);
1521        };
1522        let Ok(mapped) = self.buffer.buffer().slice(..).get_mapped_range() else {
1523            // Not mapped, so nothing to unmap.
1524            return Err(RenderError::TextureReadback(ReadbackFailure::NotMapped));
1525        };
1526        let result = (|| {
1527            self.check_canary(&mapped)?;
1528            let (width, height) = (self.size.width as usize, self.size.height as usize);
1529            let pitch = texels.pitch as usize;
1530            let rows = mapped
1531                .get(..pitch * height)
1532                .ok_or(RenderError::TextureReadback(ReadbackFailure::Layout))?;
1533            let row = |y: usize, out: &mut [[f32; 4]]| {
1534                let texels_row = &rows[y * pitch..];
1535                decode_texels(texels.format, texels_row, &mut out[..width]);
1536            };
1537            let picture = crate::layout_pack::Picture {
1538                width,
1539                height,
1540                row: &row,
1541            };
1542            crate::layout_pack::pack(&picture, texels.encoding, texels.layout, targets)
1543        })();
1544        drop(mapped);
1545        self.buffer.buffer().unmap();
1546        if result.is_ok() {
1547            self.buffer.recycle();
1548        }
1549        result
1550    }
1551
1552    /// Checks that the mapped buffer ends in this readback's canary; a failing buffer is not
1553    /// recycled.
1554    fn check_canary(&self, mapped: &[u8]) -> Result<(), RenderError> {
1555        let start = usize::try_from(self.image_bytes())
1556            .map_err(|_| RenderError::TextureReadback(ReadbackFailure::Layout))?;
1557        let found = mapped
1558            .get(start..)
1559            .ok_or(RenderError::TextureReadback(ReadbackFailure::Layout))?;
1560        self.canary.check(found)
1561    }
1562}
1563
1564/// A map callback runs on whichever thread's poll collected the mapping, possibly after this
1565/// thread's own wait returned.
1566#[cfg(not(target_arch = "wasm32"))]
1567fn receive_mapped<T>(
1568    gpu: &GpuContext,
1569    receiver: &mpsc::Receiver<T>,
1570    timeout: std::time::Duration,
1571) -> Result<T, RenderError> {
1572    crate::settle::receive_polling(receiver, timeout, || {
1573        let _ = gpu.device().poll(wgpu::PollType::Poll);
1574    })
1575}
1576
1577/// How the next [`map_staging`] on this thread delivers its callback in tests.
1578#[cfg(test)]
1579#[derive(Clone, Copy, Debug)]
1580pub(crate) enum MapReport {
1581    /// From another thread, after this delay.
1582    Elsewhere(std::time::Duration),
1583    Never,
1584}
1585
1586#[cfg(test)]
1587thread_local! {
1588    pub(crate) static MAP_REPORT: std::cell::Cell<Option<MapReport>> =
1589        const { std::cell::Cell::new(None) };
1590}
1591
1592fn map_staging(
1593    gpu: &GpuContext,
1594    buffer: &ReadbackBufferLease,
1595    requested: &AtomicBool,
1596    on_ready: impl FnOnce(Result<(), RenderError>) + Send + 'static,
1597) {
1598    #[cfg(test)]
1599    let on_ready = {
1600        let report = MAP_REPORT.with(std::cell::Cell::take);
1601        move |result: Result<(), RenderError>| match report {
1602            None => on_ready(result),
1603            Some(MapReport::Elsewhere(delay)) => {
1604                std::thread::spawn(move || {
1605                    std::thread::sleep(delay);
1606                    on_ready(result);
1607                });
1608            }
1609            Some(MapReport::Never) => std::mem::forget(on_ready),
1610        }
1611    };
1612    if requested.swap(true, Ordering::AcqRel) {
1613        on_ready(Err(RenderError::TextureReadback(
1614            ReadbackFailure::DuplicateMapRequest,
1615        )));
1616        return;
1617    }
1618    gpu.map_async(
1619        &buffer.buffer().slice(..),
1620        wgpu::MapMode::Read,
1621        move |result| {
1622            // `BufferAsyncError` names no cause; device diagnostics may explain it.
1623            on_ready(result.map_err(|_| RenderError::TextureReadback(ReadbackFailure::MapFailed)));
1624        },
1625    );
1626}
1627
1628#[derive(Clone, Debug)]
1629struct TexelStaging {
1630    format: wgpu::TextureFormat,
1631    pitch: u32,
1632    bytes: u64,
1633    layout: PixelLayout,
1634    encoding: YuvEncoding,
1635}
1636
1637fn decode_texels(format: wgpu::TextureFormat, row: &[u8], out: &mut [[f32; 4]]) {
1638    match format {
1639        wgpu::TextureFormat::Rgba16Float => {
1640            for (texel, bytes) in out.iter_mut().zip(row.as_chunks::<8>().0) {
1641                *texel = std::array::from_fn(|channel| {
1642                    crate::cpu::half_value(u16::from_le_bytes([
1643                        bytes[channel * 2],
1644                        bytes[channel * 2 + 1],
1645                    ]))
1646                });
1647            }
1648        }
1649        _ => {
1650            for (texel, bytes) in out.iter_mut().zip(row.as_chunks::<4>().0) {
1651                *texel = std::array::from_fn(|channel| crate::cpu::unorm8_value(bytes[channel]));
1652            }
1653        }
1654    }
1655}
1656
1657/// Where one plane sits in a mapped readback staging buffer.
1658#[derive(Copy, Clone, Debug, Eq, PartialEq)]
1659pub struct StagingPlane {
1660    /// Byte offset of the plane's first row.
1661    pub offset: u64,
1662    /// Bytes from the start of one row to the start of the next.
1663    pub stride: u32,
1664}
1665
1666fn aligned_copy_row(width: u32) -> Result<u32, RenderError> {
1667    let alignment = wgpu::COPY_BYTES_PER_ROW_ALIGNMENT;
1668    width
1669        .checked_add(alignment - 1)
1670        .map(|value| value / alignment)
1671        .and_then(|rows| rows.checked_mul(alignment))
1672        .ok_or(RenderError::TextureReadback(ReadbackFailure::Layout))
1673}
1674
1675fn yuv420_tight_bytes(size: RenderSize) -> Result<u64, RenderError> {
1676    u64::from(size.width)
1677        .checked_mul(u64::from(size.height))
1678        .and_then(|luma| luma.checked_mul(YUV420_PLANE_COUNT))
1679        .map(|three_planes| three_planes / 2)
1680        .ok_or(RenderError::TextureReadback(ReadbackFailure::Layout))
1681}
1682
1683/// Pack-shader uniforms for `encoding`: luma weights, range levels and chroma co-siting axes,
1684/// from the same definitions import decodes with so a round trip is the identity.
1685pub(crate) fn yuv_uniform(encoding: YuvEncoding) -> [f32; 12] {
1686    let YuvEncoding { color, chroma } = encoding;
1687    // Components that are already RGB have no pack: they take BT.709's.
1688    let (kr, kb) = color
1689        .matrix
1690        .luma_weights()
1691        .or(ColorMatrix::Bt709.luma_weights())
1692        .expect("BT.709 is a YCbCr matrix");
1693    let levels = color.range.levels(8);
1694    let (cosited_x, cosited_y) = match chroma {
1695        ChromaLocation::Left => (true, false),
1696        ChromaLocation::TopLeft => (true, true),
1697        ChromaLocation::Unspecified | ChromaLocation::Center => (false, false),
1698    };
1699    [
1700        kr,
1701        1.0 - kr - kb,
1702        kb,
1703        0.0,
1704        levels.luma_black / levels.maximum,
1705        levels.luma_span / levels.maximum,
1706        levels.chroma_center / levels.maximum,
1707        levels.chroma_span / levels.maximum,
1708        f32::from(u8::from(cosited_x)),
1709        f32::from(u8::from(cosited_y)),
1710        0.0,
1711        0.0,
1712    ]
1713}
1714
1715fn yuv_render_pipeline(
1716    device: &wgpu::Device,
1717    layout: &wgpu::PipelineLayout,
1718    shader: &wgpu::ShaderModule,
1719    fragment_entry: &'static str,
1720    target_formats: &[wgpu::TextureFormat],
1721) -> wgpu::RenderPipeline {
1722    let targets = target_formats
1723        .iter()
1724        .copied()
1725        .map(|format| Some(format.into()))
1726        .collect::<Vec<_>>();
1727    device.create_render_pipeline(&wgpu::RenderPipelineDescriptor {
1728        label: Some("alpha2 RGB to YUV pipeline"),
1729        layout: Some(layout),
1730        vertex: wgpu::VertexState {
1731            module: shader,
1732            entry_point: Some("vs"),
1733            buffers: &[],
1734            compilation_options: wgpu::PipelineCompilationOptions::default(),
1735        },
1736        fragment: Some(wgpu::FragmentState {
1737            module: shader,
1738            entry_point: Some(fragment_entry),
1739            targets: &targets,
1740            compilation_options: wgpu::PipelineCompilationOptions::default(),
1741        }),
1742        primitive: wgpu::PrimitiveState::default(),
1743        depth_stencil: None,
1744        multisample: wgpu::MultisampleState::default(),
1745        multiview_mask: None,
1746        cache: None,
1747    })
1748}
1749
1750fn encode_yuv_render_pass(
1751    encoder: &mut wgpu::CommandEncoder,
1752    label: &'static str,
1753    pipeline: &wgpu::RenderPipeline,
1754    bind_group: &wgpu::BindGroup,
1755    target: &wgpu::TextureView,
1756) {
1757    let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
1758        label: Some(label),
1759        color_attachments: &[Some(wgpu::RenderPassColorAttachment {
1760            view: target,
1761            depth_slice: None,
1762            resolve_target: None,
1763            ops: wgpu::Operations {
1764                load: wgpu::LoadOp::Clear(wgpu::Color::BLACK),
1765                store: wgpu::StoreOp::Store,
1766            },
1767        })],
1768        depth_stencil_attachment: None,
1769        timestamp_writes: None,
1770        occlusion_query_set: None,
1771        multiview_mask: None,
1772    });
1773    pass.set_pipeline(pipeline);
1774    pass.set_bind_group(0, bind_group, &[]);
1775    pass.draw(0..3, 0..1);
1776}
1777
1778fn encode_yuv_render_pass_pair(
1779    encoder: &mut wgpu::CommandEncoder,
1780    label: &'static str,
1781    pipeline: &wgpu::RenderPipeline,
1782    bind_group: &wgpu::BindGroup,
1783    targets: [&wgpu::TextureView; 2],
1784) {
1785    let attachments = targets.map(|target| {
1786        Some(wgpu::RenderPassColorAttachment {
1787            view: target,
1788            depth_slice: None,
1789            resolve_target: None,
1790            ops: wgpu::Operations {
1791                load: wgpu::LoadOp::Clear(wgpu::Color::BLACK),
1792                store: wgpu::StoreOp::Store,
1793            },
1794        })
1795    });
1796    let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
1797        label: Some(label),
1798        color_attachments: &attachments,
1799        depth_stencil_attachment: None,
1800        timestamp_writes: None,
1801        occlusion_query_set: None,
1802        multiview_mask: None,
1803    });
1804    pass.set_pipeline(pipeline);
1805    pass.set_bind_group(0, bind_group, &[]);
1806    pass.draw(0..3, 0..1);
1807}
1808
1809fn encode_plane_copy(
1810    encoder: &mut wgpu::CommandEncoder,
1811    texture: &wgpu::Texture,
1812    buffer: &wgpu::Buffer,
1813    offset: u64,
1814    bytes_per_row: u32,
1815    size: RenderSize,
1816) {
1817    encoder.copy_texture_to_buffer(
1818        wgpu::TexelCopyTextureInfo {
1819            texture,
1820            mip_level: 0,
1821            origin: wgpu::Origin3d::ZERO,
1822            aspect: wgpu::TextureAspect::All,
1823        },
1824        wgpu::TexelCopyBufferInfo {
1825            buffer,
1826            layout: wgpu::TexelCopyBufferLayout {
1827                offset,
1828                bytes_per_row: Some(bytes_per_row),
1829                rows_per_image: Some(size.height),
1830            },
1831        },
1832        wgpu::Extent3d {
1833            width: size.width,
1834            height: size.height,
1835            depth_or_array_layers: 1,
1836        },
1837    );
1838}
1839
1840#[cfg(test)]
1841pub(crate) fn read_rgba8(renderer: &Renderer, target: &RenderedTexture) -> Vec<u8> {
1842    renderer
1843        .read_rgba8(target)
1844        .expect("test texture readback succeeds")
1845}
1846
1847#[cfg(test)]
1848#[path = "readback_tests.rs"]
1849mod layout_tests;