alpha2_render/

readback.rs

use std::sync::atomic::{AtomicBool, Ordering};
#[cfg(not(target_arch = "wasm32"))]
use std::sync::mpsc;

use crate::canary::{CANARY_BYTES, Canary};
use crate::{
    ReadbackFailure, RenderError, RenderSize, RenderedTexture, Renderer,
    pipeline::{CachedPipeline, CachedYuvReadbackPipeline},
    pool::{ReadbackBufferLease, TextureLease},
    uniform::{UniformArenaLease, f32_bytes},
};
use alpha2_gpu::GpuContext;
use alpha2_media::{ChromaLocation, ColorMatrix, PixelLayout, YuvEncoding};
use alpha2_shaders::{READBACK_PACK_WGSL, RGB_TO_YUV420_WGSL};

const RGBA8_BYTES_PER_PIXEL: u32 = 4;
const RGBA16F_BYTES_PER_PIXEL: u32 = 8;
const YUV420_PLANE_COUNT: u64 = 3;

/// D3D12 texture-to-buffer copies must start on this; wgpu's DX12 workaround for an unaligned
/// start pads rows and overwrites whatever follows.
const PLANE_PLACEMENT_ALIGNMENT: u64 = 512;

/// Where each plane of one I420 readback sits in its staging buffer.
///
/// Each plane starts on [`PLANE_PLACEMENT_ALIGNMENT`] so DX12's padded-copy workaround cannot
/// overwrite the next plane.
#[derive(Copy, Clone, Debug, Eq, PartialEq)]
pub(crate) struct Yuv420StagingLayout {
    luma_pitch: u32,
    chroma_pitch: u32,
    u_offset: u64,
    v_offset: u64,
    bytes: u64,
}

impl Yuv420StagingLayout {
    pub(crate) fn new(size: RenderSize) -> Result<Self, RenderError> {
        let luma_pitch = aligned_copy_row(size.width)?;
        let chroma_pitch = aligned_copy_row(size.width / 2)?;
        let region = |pitch: u32, rows: u32| u64::from(pitch).checked_mul(u64::from(rows));
        let place = |end: u64| end.checked_next_multiple_of(PLANE_PLACEMENT_ALIGNMENT);
        let chroma_bytes = region(chroma_pitch, size.height / 2)
            .ok_or(RenderError::TextureReadback(ReadbackFailure::Layout))?;
        let u_offset = region(luma_pitch, size.height)
            .and_then(place)
            .ok_or(RenderError::TextureReadback(ReadbackFailure::Layout))?;
        let v_offset = u_offset
            .checked_add(chroma_bytes)
            .and_then(place)
            .ok_or(RenderError::TextureReadback(ReadbackFailure::Layout))?;
        let bytes = v_offset
            .checked_add(chroma_bytes)
            .ok_or(RenderError::TextureReadback(ReadbackFailure::Layout))?;
        Ok(Self {
            luma_pitch,
            chroma_pitch,
            u_offset,
            v_offset,
            bytes,
        })
    }

    /// `(offset, pitch, width, rows)` in Y, U, V order.
    fn planes(self, size: RenderSize) -> [(u64, u32, u32, u32); 3] {
        let (chroma_width, chroma_rows) = (size.width / 2, size.height / 2);
        [
            (0, self.luma_pitch, size.width, size.height),
            (self.u_offset, self.chroma_pitch, chroma_width, chroma_rows),
            (self.v_offset, self.chroma_pitch, chroma_width, chroma_rows),
        ]
    }
}

/// Staging bytes of one planar YUV420 readback of `size`, including row padding, plane
/// alignment and the 16-byte canary. A budget must reserve this per in-flight readback.
///
/// # Errors
/// Returns [`RenderError::TextureReadback`] when the layout overflows.
pub fn yuv420p_staging_bytes(size: RenderSize) -> Result<u64, RenderError> {
    Yuv420StagingLayout::new(size).map(|layout| layout.bytes + CANARY_BYTES)
}

fn plane_rows(
    mapped: &[u8],
    (offset, pitch, width, rows): (u64, u32, u32, u32),
) -> Result<std::iter::Take<std::slice::ChunksExact<'_, u8>>, RenderError> {
    let offset = usize::try_from(offset)
        .map_err(|_| RenderError::TextureReadback(ReadbackFailure::Layout))?;
    let pitch = usize::try_from(pitch)
        .map_err(|_| RenderError::TextureReadback(ReadbackFailure::Layout))?;
    let width = usize::try_from(width)
        .map_err(|_| RenderError::TextureReadback(ReadbackFailure::Layout))?;
    let rows =
        usize::try_from(rows).map_err(|_| RenderError::TextureReadback(ReadbackFailure::Layout))?;
    let region = pitch
        .checked_mul(rows)
        .and_then(|length| length.checked_add(offset))
        .and_then(|end| mapped.get(offset..end))
        .filter(|_| width <= pitch)
        .ok_or(RenderError::TextureReadback(ReadbackFailure::Layout))?;
    Ok(region.chunks_exact(pitch).take(rows))
}

pub(crate) struct EncodedYuv420pReadback {
    buffer: ReadbackBufferLease,
    size: RenderSize,
    layout: Yuv420StagingLayout,
    staging_bytes: u64,
    canary: Canary,
    y_texture: TextureLease,
    u_texture: TextureLease,
    v_texture: TextureLease,
    uniforms: UniformArenaLease,
}

impl EncodedYuv420pReadback {
    pub(crate) fn flush_uniforms(&self, queue: &wgpu::Queue) {
        self.uniforms.flush(queue);
    }
}

impl Renderer {
    /// Uploads tightly packed premultiplied sRGB RGBA8 pixels into a pooled upload texture.
    ///
    /// # Errors
    /// [`RenderError::InvalidImageData`] when `pixels` is not four bytes per pixel.
    pub fn upload_rgba8(
        &self,
        size: RenderSize,
        pixels: &[u8],
    ) -> Result<crate::UploadTexture, RenderError> {
        self.write_rgba8(
            size,
            crate::SourcePlane {
                bytes: pixels,
                stride: size
                    .width
                    .checked_mul(RGBA8_BYTES_PER_PIXEL)
                    .ok_or(RenderError::InvalidImageData)?,
            },
        )
    }

    /// Uploads tightly packed premultiplied linear RGBA half floats (BT.709, `1.0` = SDR white) for
    /// [`Renderer::prepare_linear_presented`].
    ///
    /// # Errors
    /// [`RenderError::InvalidImageData`] when `pixels` is not four halves per pixel.
    pub fn upload_linear_rgba16f(
        &self,
        size: RenderSize,
        pixels: &[u16],
    ) -> Result<crate::UploadTexture, RenderError> {
        // Avoids a frame-sized copy per frame on little-endian targets.
        let bytes: std::borrow::Cow<'_, [u8]> = if cfg!(target_endian = "little") {
            std::borrow::Cow::Borrowed(bytemuck::cast_slice(pixels))
        } else {
            std::borrow::Cow::Owned(pixels.iter().flat_map(|half| half.to_le_bytes()).collect())
        };
        self.write_linear_rgba16f(
            size,
            crate::SourcePlane {
                bytes: &bytes,
                stride: size
                    .width
                    .checked_mul(RGBA16F_BYTES_PER_PIXEL)
                    .ok_or(RenderError::InvalidImageData)?,
            },
        )
    }

    /// Writes strided little-endian RGBA half-float rows into a pooled upload texture.
    ///
    /// # Errors
    /// [`RenderError::InvalidImageData`] for a plane shorter than `size`.
    pub fn write_linear_rgba16f(
        &self,
        size: RenderSize,
        plane: crate::SourcePlane<'_>,
    ) -> Result<crate::UploadTexture, RenderError> {
        self.validate_size(size)?;
        let texture = self.upload_textures.acquire(
            self.gpu.device(),
            size,
            wgpu::TextureFormat::Rgba16Float,
            "alpha2 linear RGBA upload texture",
        );
        self.write_plane(&texture, size, plane, RGBA16F_BYTES_PER_PIXEL)?;
        Ok(texture)
    }

    pub(crate) fn write_rgba8(
        &self,
        size: RenderSize,
        plane: crate::SourcePlane<'_>,
    ) -> Result<crate::UploadTexture, RenderError> {
        let texture = self.acquire_rgba8_upload_texture(size)?;
        self.write_plane(&texture, size, plane, RGBA8_BYTES_PER_PIXEL)?;
        Ok(texture)
    }

    fn write_plane(
        &self,
        texture: &crate::UploadTexture,
        size: RenderSize,
        plane: crate::SourcePlane<'_>,
        bytes_per_pixel: u32,
    ) -> Result<(), RenderError> {
        let row_bytes = size
            .width
            .checked_mul(bytes_per_pixel)
            .ok_or(RenderError::InvalidImageData)?;
        let required = usize::try_from(plane.stride)
            .ok()
            .zip(usize::try_from(size.height).ok())
            .and_then(|(stride, rows)| stride.checked_mul(rows));
        if plane.stride < row_bytes || required != Some(plane.bytes.len()) {
            return Err(RenderError::InvalidImageData);
        }
        self.gpu.queue().write_texture(
            wgpu::TexelCopyTextureInfo {
                texture: texture.texture(),
                mip_level: 0,
                origin: wgpu::Origin3d::ZERO,
                aspect: wgpu::TextureAspect::All,
            },
            plane.bytes,
            wgpu::TexelCopyBufferLayout {
                offset: 0,
                bytes_per_row: Some(plane.stride),
                rows_per_image: Some(size.height),
            },
            wgpu::Extent3d {
                width: size.width,
                height: size.height,
                depth_or_array_layers: 1,
            },
        );
        Ok(())
    }

    /// Copies one RGBA8 render result into tightly packed premultiplied host memory.
    ///
    /// Blocks on the GPU, so native and test only; callers that cannot block use
    /// [`Renderer::begin_rgba8_readback`]. Half-float intermediates are quantized to RGBA8 on the
    /// GPU first.
    ///
    /// # Errors
    /// [`RenderError::TextureReadback`] for another target format or a failed mapping.
    #[cfg(not(target_arch = "wasm32"))]
    pub fn read_rgba8(&self, target: &RenderedTexture) -> Result<Vec<u8>, RenderError> {
        self.begin_rgba8_readback(target)?.wait()
    }

    /// Submits a readback copy and returns an unmapped handle.
    ///
    /// The copy is queued on return, so mapping can resolve on an event loop via
    /// [`Rgba8Readback::map_async`]. Used for stills, thumbnails and goldens; exports use YUV or
    /// NV12 readbacks instead.
    ///
    /// # Errors
    /// [`RenderError::TextureReadback`] for an unsupported format or unrepresentable row layout.
    pub fn begin_rgba8_readback(
        &self,
        target: &RenderedTexture,
    ) -> Result<Rgba8Readback, RenderError> {
        self.ensure_gpu_runs_work()?;
        self.validate_size(target.size)?;
        if !matches!(
            target.format,
            wgpu::TextureFormat::Rgba8Unorm | wgpu::TextureFormat::Rgba16Float
        ) {
            return Err(RenderError::TextureReadback(
                ReadbackFailure::UnsupportedTarget,
            ));
        }

        let unpadded_bytes_per_row = target
            .size
            .width
            .checked_mul(RGBA8_BYTES_PER_PIXEL)
            .ok_or(RenderError::TextureReadback(ReadbackFailure::Layout))?;
        let alignment = wgpu::COPY_BYTES_PER_ROW_ALIGNMENT;
        let padded_bytes_per_row = unpadded_bytes_per_row
            .checked_add(alignment - 1)
            .map(|value| value / alignment)
            .and_then(|rows| rows.checked_mul(alignment))
            .ok_or(RenderError::TextureReadback(ReadbackFailure::Layout))?;
        let image_bytes = u64::from(padded_bytes_per_row)
            .checked_mul(u64::from(target.size.height))
            .ok_or(RenderError::TextureReadback(ReadbackFailure::Layout))?;
        // Rows, then the canary.
        let buffer_size = image_bytes
            .checked_add(CANARY_BYTES)
            .ok_or(RenderError::TextureReadback(ReadbackFailure::Layout))?;
        let mut encoder = self.command_encoder("alpha2 RGBA8 readback encoder");
        let buffer =
            self.readback_buffers
                .acquire(self.gpu.device(), buffer_size, "alpha2 RGBA8 readback");
        let packed_texture = self.encode_readback_pack(target, &mut encoder);
        let copy_source = packed_texture
            .as_ref()
            .map_or(&target.texture, TextureLease::texture);
        encoder.copy_texture_to_buffer(
            wgpu::TexelCopyTextureInfo {
                texture: copy_source,
                mip_level: 0,
                origin: wgpu::Origin3d::ZERO,
                aspect: wgpu::TextureAspect::All,
            },
            wgpu::TexelCopyBufferInfo {
                buffer: buffer.buffer(),
                layout: wgpu::TexelCopyBufferLayout {
                    offset: 0,
                    bytes_per_row: Some(padded_bytes_per_row),
                    rows_per_image: Some(target.size.height),
                },
            },
            wgpu::Extent3d {
                width: target.size.width,
                height: target.size.height,
                depth_or_array_layers: 1,
            },
        );
        // Same encoder, after the copy, so it lands only if the copy ran.
        let canary = self
            .canaries
            .encode(&self.gpu, &mut encoder, buffer.buffer(), image_bytes)?;
        let command_buffer = encoder.finish();
        #[cfg(not(target_arch = "wasm32"))]
        let submission = self.submit([command_buffer])?;
        #[cfg(target_arch = "wasm32")]
        self.submit([command_buffer])?;
        let render_passes = u64::from(packed_texture.is_some());
        let copy_bytes =
            u64::from(unpadded_bytes_per_row).saturating_mul(u64::from(target.size.height));
        crate::diagnostics::record_readback_work(
            &self.diagnostics,
            render_passes,
            1,
            copy_bytes,
            true,
        );
        Ok(Rgba8Readback {
            buffer,
            gpu: self.gpu.clone(),
            size: target.size,
            padded_bytes_per_row,
            unpadded_bytes_per_row,
            #[cfg(not(target_arch = "wasm32"))]
            submission,
            #[cfg(not(target_arch = "wasm32"))]
            timeout: self.gpu_wait_timeout,
            staging_bytes: buffer_size,
            image_bytes,
            canary,
            mapping_requested: AtomicBool::new(false),
        })
    }

    /// Converts a render result to 8-bit planar YUV420 on the GPU and submits an async readback.
    ///
    /// The size must be even. Chroma is filtered for the siting `encoding.chroma` names
    /// ([`alpha2_media::VideoCodec::chroma_location`]).
    ///
    /// # Errors
    /// [`RenderError::TextureReadback`] for unsupported formats, odd dimensions or unrepresentable
    /// layouts.
    pub fn begin_yuv420p_readback(
        &self,
        target: &RenderedTexture,
        encoding: YuvEncoding,
    ) -> Result<Yuv420pReadback, RenderError> {
        let mut command_encoder = self.command_encoder("alpha2 YUV420P readback encoder");
        let readback_work = self.encode_yuv420p_readback(target, encoding, &mut command_encoder)?;
        readback_work.flush_uniforms(self.gpu.queue());
        #[cfg(not(target_arch = "wasm32"))]
        let submission = self.submit([command_encoder.finish()])?;
        #[cfg(target_arch = "wasm32")]
        self.submit([command_encoder.finish()])?;
        Ok(self.complete_yuv420p_readback(
            readback_work,
            #[cfg(not(target_arch = "wasm32"))]
            submission,
            true,
        ))
    }

    /// Reads a render result back as stored and packs it into `layout` on the CPU
    /// ([`Yuv420pReadback::finish_into_planes`]), for outputs other than 8-bit 4:2:0.
    ///
    /// 10-bit output requires the `rgba16float` intermediate.
    ///
    /// # Errors
    /// [`RenderError::TextureReadback`] for an unsupported format, a 10-bit layout over an 8-bit
    /// target, or an unrepresentable row layout.
    pub fn begin_layout_readback(
        &self,
        target: &RenderedTexture,
        encoding: YuvEncoding,
        layout: PixelLayout,
    ) -> Result<Yuv420pReadback, RenderError> {
        self.ensure_gpu_runs_work()?;
        self.validate_size(target.size)?;
        let texel_bytes = match target.format {
            wgpu::TextureFormat::Rgba16Float => 8,
            wgpu::TextureFormat::Rgba8Unorm
                if layout.bit_depth == alpha2_media::BitDepth::Eight =>
            {
                4
            }
            _ => {
                return Err(RenderError::TextureReadback(
                    ReadbackFailure::UnsupportedTarget,
                ));
            }
        };
        let row_bytes = target
            .size
            .width
            .checked_mul(texel_bytes)
            .ok_or(RenderError::TextureReadback(ReadbackFailure::Layout))?;
        let pitch = aligned_copy_row(row_bytes)?;
        let image_bytes = u64::from(pitch)
            .checked_mul(u64::from(target.size.height))
            .ok_or(RenderError::TextureReadback(ReadbackFailure::Layout))?;
        let buffer_size = image_bytes
            .checked_add(CANARY_BYTES)
            .ok_or(RenderError::TextureReadback(ReadbackFailure::Layout))?;
        let mut encoder = self.command_encoder("alpha2 texel readback encoder");
        let buffer =
            self.readback_buffers
                .acquire(self.gpu.device(), buffer_size, "alpha2 texel readback");
        encoder.copy_texture_to_buffer(
            wgpu::TexelCopyTextureInfo {
                texture: &target.texture,
                mip_level: 0,
                origin: wgpu::Origin3d::ZERO,
                aspect: wgpu::TextureAspect::All,
            },
            wgpu::TexelCopyBufferInfo {
                buffer: buffer.buffer(),
                layout: wgpu::TexelCopyBufferLayout {
                    offset: 0,
                    bytes_per_row: Some(pitch),
                    rows_per_image: Some(target.size.height),
                },
            },
            wgpu::Extent3d {
                width: target.size.width,
                height: target.size.height,
                depth_or_array_layers: 1,
            },
        );
        let canary = self
            .canaries
            .encode(&self.gpu, &mut encoder, buffer.buffer(), image_bytes)?;
        #[cfg(not(target_arch = "wasm32"))]
        let submission = self.submit([encoder.finish()])?;
        #[cfg(target_arch = "wasm32")]
        self.submit([encoder.finish()])?;
        crate::diagnostics::record_readback_work(
            &self.diagnostics,
            0,
            1,
            u64::from(row_bytes).saturating_mul(u64::from(target.size.height)),
            true,
        );
        Ok(Yuv420pReadback {
            buffer,
            gpu: self.gpu.clone(),
            size: target.size,
            layout: Yuv420StagingLayout {
                luma_pitch: 0,
                chroma_pitch: 0,
                u_offset: 0,
                v_offset: 0,
                bytes: 0,
            },
            texels: Some(Box::new(TexelStaging {
                format: target.format,
                pitch,
                bytes: image_bytes,
                layout,
                encoding,
            })),
            #[cfg(not(target_arch = "wasm32"))]
            submission,
            #[cfg(not(target_arch = "wasm32"))]
            timeout: self.gpu_wait_timeout,
            staging_bytes: buffer_size,
            canary,
            mapping_requested: AtomicBool::new(false),
        })
    }

    pub(crate) fn encode_yuv420p_readback(
        &self,
        target: &RenderedTexture,
        encoding: YuvEncoding,
        encoder: &mut wgpu::CommandEncoder,
    ) -> Result<EncodedYuv420pReadback, RenderError> {
        self.ensure_gpu_runs_work()?;
        self.validate_size(target.size)?;
        if !matches!(
            target.format,
            wgpu::TextureFormat::Rgba8Unorm | wgpu::TextureFormat::Rgba16Float
        ) || !target.size.width.is_multiple_of(2)
            || !target.size.height.is_multiple_of(2)
        {
            return Err(RenderError::TextureReadback(
                ReadbackFailure::UnsupportedTarget,
            ));
        }
        let layout = Yuv420StagingLayout::new(target.size)?;
        let uv_rows = target.size.height / 2;
        let chroma_width = target.size.width / 2;
        let buffer_size = layout.bytes + CANARY_BYTES;
        let buffer = self.readback_buffers.acquire(
            self.gpu.device(),
            buffer_size,
            "alpha2 YUV420P readback",
        );
        let y_texture = self.textures.acquire(
            self.gpu.device(),
            target.size,
            wgpu::TextureFormat::R8Unorm,
            "alpha2 GPU Y readback texture",
        );
        let uv_size = RenderSize {
            width: chroma_width,
            height: uv_rows,
        };
        let u_texture = self.textures.acquire(
            self.gpu.device(),
            uv_size,
            wgpu::TextureFormat::R8Unorm,
            "alpha2 GPU U readback texture",
        );
        let v_texture = self.textures.acquire(
            self.gpu.device(),
            uv_size,
            wgpu::TextureFormat::R8Unorm,
            "alpha2 GPU V readback texture",
        );
        let uniform_bytes = f32_bytes(yuv_uniform(encoding));
        let mut uniforms = self.uniform_buffers.acquire(
            self.gpu.device(),
            u64::try_from(uniform_bytes.len()).unwrap_or(u64::MAX),
        );
        let uniform = uniforms.write(&uniform_bytes);
        self.encode_yuv420_pack(
            target,
            [y_texture.view(), u_texture.view(), v_texture.view()],
            encoder,
            &uniforms,
            uniform,
        );
        encode_plane_copy(
            encoder,
            y_texture.texture(),
            buffer.buffer(),
            0,
            layout.luma_pitch,
            target.size,
        );
        encode_plane_copy(
            encoder,
            u_texture.texture(),
            buffer.buffer(),
            layout.u_offset,
            layout.chroma_pitch,
            uv_size,
        );
        encode_plane_copy(
            encoder,
            v_texture.texture(),
            buffer.buffer(),
            layout.v_offset,
            layout.chroma_pitch,
            uv_size,
        );
        // Same encoder, after the plane copies. Not counted as a copy in diagnostics.
        let canary = self
            .canaries
            .encode(&self.gpu, encoder, buffer.buffer(), layout.bytes)?;
        Ok(EncodedYuv420pReadback {
            buffer,
            size: target.size,
            layout,
            staging_bytes: buffer_size,
            canary,
            y_texture,
            u_texture,
            v_texture,
            uniforms,
        })
    }

    pub(crate) fn complete_yuv420p_readback(
        &self,
        encoded: EncodedYuv420pReadback,
        #[cfg(not(target_arch = "wasm32"))] submission: wgpu::SubmissionIndex,
        standalone_submission: bool,
    ) -> Yuv420pReadback {
        let EncodedYuv420pReadback {
            buffer,
            size,
            layout,
            staging_bytes,
            canary,
            y_texture,
            u_texture,
            v_texture,
            uniforms,
        } = encoded;
        uniforms.recycle();
        drop((y_texture, u_texture, v_texture));
        let texture_bytes = yuv420_tight_bytes(size).expect("validated YUV420 dimensions");
        crate::diagnostics::record_readback_work(
            &self.diagnostics,
            2,
            3,
            texture_bytes,
            standalone_submission,
        );
        Yuv420pReadback {
            buffer,
            gpu: self.gpu.clone(),
            size,
            layout,
            texels: None,
            #[cfg(not(target_arch = "wasm32"))]
            submission,
            #[cfg(not(target_arch = "wasm32"))]
            timeout: self.gpu_wait_timeout,
            staging_bytes,
            canary,
            mapping_requested: AtomicBool::new(false),
        }
    }

    /// Packs one rendered frame into caller-owned NV12 plane textures.
    ///
    /// Same conversion as [`Self::begin_yuv420p_readback`] without the readback. wgpu allows only
    /// `TEXTURE_BINDING` on `TextureFormat::NV12`, so the native layer copies these `R8Unorm` and
    /// `Rg8Unorm` targets into the encoder's image afterwards.
    ///
    /// # Errors
    /// [`RenderError::TextureReadback`] for an unaccepted target format or size,
    /// [`RenderError::InvalidPlaneTarget`] for mismatched plane textures, and
    /// [`RenderError::GpuWorkDropped`] once the device was found dropping work.
    #[cfg_attr(not(feature = "wgpu-interop"), doc(hidden))]
    pub fn encode_nv12_pack(
        &self,
        target: &RenderedTexture,
        planes: Nv12PlaneTargets<'_>,
        encoding: YuvEncoding,
        encoder: &mut wgpu::CommandEncoder,
    ) -> Result<EncodedNv12Pack, RenderError> {
        // A pack has no canary of its own; an earlier failed readback canary applies.
        self.ensure_gpu_runs_work()?;
        let encode = self.open_encode();
        self.validate_size(target.size)?;
        if !matches!(
            target.format,
            wgpu::TextureFormat::Rgba8Unorm | wgpu::TextureFormat::Rgba16Float
        ) || !target.size.width.is_multiple_of(2)
            || !target.size.height.is_multiple_of(2)
        {
            return Err(RenderError::TextureReadback(
                ReadbackFailure::UnsupportedTarget,
            ));
        }
        let chroma_size = RenderSize {
            width: target.size.width / 2,
            height: target.size.height / 2,
        };
        // Planes may belong to a hardware encoder, so check them here rather than hit a validation
        // error.
        let attachment = |texture: &wgpu::Texture| {
            texture
                .usage()
                .contains(wgpu::TextureUsages::RENDER_ATTACHMENT)
                && texture.mip_level_count() == 1
                && texture.sample_count() == 1
                && texture.depth_or_array_layers() == 1
        };
        if planes.luma.format() != wgpu::TextureFormat::R8Unorm
            || planes.chroma.format() != wgpu::TextureFormat::Rg8Unorm
            || planes.luma.width() != target.size.width
            || planes.luma.height() != target.size.height
            || planes.chroma.width() != chroma_size.width
            || planes.chroma.height() != chroma_size.height
            || !attachment(planes.luma)
            || !attachment(planes.chroma)
        {
            return Err(RenderError::InvalidPlaneTarget);
        }
        let uniform_bytes = f32_bytes(yuv_uniform(encoding));
        let mut uniforms = self.uniform_buffers.acquire(
            self.gpu.device(),
            u64::try_from(uniform_bytes.len()).unwrap_or(u64::MAX),
        );
        let uniform = uniforms.write(&uniform_bytes);
        let cached = self.yuv_readback_pipeline();
        let bind_group = self.yuv_pack_bind_group(target, cached, &uniforms, uniform);
        let luma_view = crate::pool::default_view(planes.luma);
        encode_yuv_render_pass(
            encoder,
            "alpha2 GPU NV12 Y pack pass",
            &cached.y_pipeline,
            &bind_group,
            &luma_view,
        );
        let chroma_view = crate::pool::default_view(planes.chroma);
        encode_yuv_render_pass(
            encoder,
            "alpha2 GPU NV12 UV pack pass",
            &cached.uv_nv12_pipeline,
            &bind_group,
            &chroma_view,
        );
        // `write` fills only the host copy; the shader reads stale data without the flush.
        uniforms.flush(self.gpu.queue());
        // Keep the uniforms until the caller submits `encoder`, or the next call could overwrite
        // them.
        Ok(EncodedNv12Pack {
            uniforms: Some(uniforms),
            _encode: encode,
        })
    }

    fn yuv_pack_bind_group(
        &self,
        target: &RenderedTexture,
        cached: &CachedYuvReadbackPipeline,
        uniforms: &crate::uniform::UniformArenaLease,
        uniform: crate::uniform::UniformSlice,
    ) -> wgpu::BindGroup {
        self.gpu
            .device()
            .create_bind_group(&wgpu::BindGroupDescriptor {
                label: Some("alpha2 RGB to YUV bind group"),
                layout: &cached.bind_group_layout,
                entries: &[
                    wgpu::BindGroupEntry {
                        binding: 0,
                        resource: wgpu::BindingResource::TextureView(target.view()),
                    },
                    wgpu::BindGroupEntry {
                        binding: 1,
                        resource: uniforms.binding(uniform),
                    },
                ],
            })
    }

    fn encode_yuv420_pack(
        &self,
        target: &RenderedTexture,
        plane_views: [&wgpu::TextureView; 3],
        encoder: &mut wgpu::CommandEncoder,
        uniforms: &crate::uniform::UniformArenaLease,
        uniform: crate::uniform::UniformSlice,
    ) {
        let cached = self.yuv_readback_pipeline();
        let bind_group = self.yuv_pack_bind_group(target, cached, uniforms, uniform);
        let [y_view, u_view, v_view] = plane_views;
        encode_yuv_render_pass(
            encoder,
            "alpha2 GPU Y pack pass",
            &cached.y_pipeline,
            &bind_group,
            y_view,
        );
        encode_yuv_render_pass_pair(
            encoder,
            "alpha2 GPU UV pack pass",
            &cached.uv_pipeline,
            &bind_group,
            [u_view, v_view],
        );
    }

    fn yuv_readback_pipeline(&self) -> &CachedYuvReadbackPipeline {
        self.pipelines.yuv_readback.get_or_init(|| {
            let bind_group_layout =
                self.gpu
                    .device()
                    .create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
                        label: Some("alpha2 RGB to YUV layout"),
                        entries: &[
                            wgpu::BindGroupLayoutEntry {
                                binding: 0,
                                visibility: wgpu::ShaderStages::FRAGMENT,
                                ty: wgpu::BindingType::Texture {
                                    sample_type: wgpu::TextureSampleType::Float {
                                        filterable: false,
                                    },
                                    view_dimension: wgpu::TextureViewDimension::D2,
                                    multisampled: false,
                                },
                                count: None,
                            },
                            wgpu::BindGroupLayoutEntry {
                                binding: 1,
                                visibility: wgpu::ShaderStages::FRAGMENT,
                                ty: wgpu::BindingType::Buffer {
                                    ty: wgpu::BufferBindingType::Uniform,
                                    has_dynamic_offset: false,
                                    min_binding_size: None,
                                },
                                count: None,
                            },
                        ],
                    });
            let shader = self
                .gpu
                .device()
                .create_shader_module(wgpu::ShaderModuleDescriptor {
                    label: Some("alpha2 RGB to YUV shader"),
                    source: wgpu::ShaderSource::Wgsl(RGB_TO_YUV420_WGSL.into()),
                });
            let layout =
                self.gpu
                    .device()
                    .create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
                        label: Some("alpha2 RGB to YUV pipeline layout"),
                        bind_group_layouts: &[Some(&bind_group_layout)],
                        immediate_size: 0,
                    });
            CachedYuvReadbackPipeline {
                y_pipeline: yuv_render_pipeline(
                    self.gpu.device(),
                    &layout,
                    &shader,
                    "y_fs",
                    &[wgpu::TextureFormat::R8Unorm],
                ),
                uv_pipeline: yuv_render_pipeline(
                    self.gpu.device(),
                    &layout,
                    &shader,
                    "uv_fs",
                    &[wgpu::TextureFormat::R8Unorm, wgpu::TextureFormat::R8Unorm],
                ),
                uv_nv12_pipeline: yuv_render_pipeline(
                    self.gpu.device(),
                    &layout,
                    &shader,
                    "uv_nv12_fs",
                    &[wgpu::TextureFormat::Rg8Unorm],
                ),
                bind_group_layout,
            }
        })
    }

    fn encode_readback_pack(
        &self,
        target: &RenderedTexture,
        encoder: &mut wgpu::CommandEncoder,
    ) -> Option<TextureLease> {
        if target.format != wgpu::TextureFormat::Rgba16Float {
            return None;
        }
        let texture = self.textures.acquire(
            self.gpu.device(),
            target.size,
            wgpu::TextureFormat::Rgba8Unorm,
            "alpha2 GPU-packed RGBA8 readback texture",
        );
        let cached = self.readback_pack_pipeline();
        let bind_group = self
            .gpu
            .device()
            .create_bind_group(&wgpu::BindGroupDescriptor {
                label: Some("alpha2 readback pack bind group"),
                layout: &cached.bind_group_layout,
                entries: &[wgpu::BindGroupEntry {
                    binding: 0,
                    resource: wgpu::BindingResource::TextureView(target.view()),
                }],
            });
        {
            let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
                label: Some("alpha2 GPU RGBA8 readback pack pass"),
                color_attachments: &[Some(wgpu::RenderPassColorAttachment {
                    view: texture.view(),
                    depth_slice: None,
                    resolve_target: None,
                    ops: wgpu::Operations {
                        load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT),
                        store: wgpu::StoreOp::Store,
                    },
                })],
                depth_stencil_attachment: None,
                timestamp_writes: None,
                occlusion_query_set: None,
                multiview_mask: None,
            });
            pass.set_pipeline(&cached.pipeline);
            pass.set_bind_group(0, &bind_group, &[]);
            pass.draw(0..3, 0..1);
        }
        Some(texture)
    }

    fn readback_pack_pipeline(&self) -> &CachedPipeline {
        self.pipelines.readback_pack.get_or_init(|| {
            let bind_group_layout =
                self.gpu
                    .device()
                    .create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
                        label: Some("alpha2 readback pack layout"),
                        entries: &[wgpu::BindGroupLayoutEntry {
                            binding: 0,
                            visibility: wgpu::ShaderStages::FRAGMENT,
                            ty: wgpu::BindingType::Texture {
                                sample_type: wgpu::TextureSampleType::Float { filterable: false },
                                view_dimension: wgpu::TextureViewDimension::D2,
                                multisampled: false,
                            },
                            count: None,
                        }],
                    });
            let shader = self
                .gpu
                .device()
                .create_shader_module(wgpu::ShaderModuleDescriptor {
                    label: Some("alpha2 readback pack shader"),
                    source: wgpu::ShaderSource::Wgsl(READBACK_PACK_WGSL.into()),
                });
            let layout =
                self.gpu
                    .device()
                    .create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
                        label: Some("alpha2 readback pack pipeline layout"),
                        bind_group_layouts: &[Some(&bind_group_layout)],
                        immediate_size: 0,
                    });
            let pipeline =
                self.gpu
                    .device()
                    .create_render_pipeline(&wgpu::RenderPipelineDescriptor {
                        label: Some("alpha2 readback pack pipeline"),
                        layout: Some(&layout),
                        vertex: wgpu::VertexState {
                            module: &shader,
                            entry_point: Some("vs"),
                            buffers: &[],
                            compilation_options: wgpu::PipelineCompilationOptions::default(),
                        },
                        fragment: Some(wgpu::FragmentState {
                            module: &shader,
                            entry_point: Some("fs"),
                            targets: &[Some(wgpu::TextureFormat::Rgba8Unorm.into())],
                            compilation_options: wgpu::PipelineCompilationOptions::default(),
                        }),
                        primitive: wgpu::PrimitiveState::default(),
                        depth_stencil: None,
                        multisample: wgpu::MultisampleState::default(),
                        multiview_mask: None,
                        cache: None,
                    });
            CachedPipeline {
                bind_group_layout,
                pipeline,
            }
        })
    }
}

/// A submitted texture readback whose buffer has not been mapped yet.
///
/// Owns its staging buffer, so it can move into an async browser task.
#[derive(Debug)]
pub struct Rgba8Readback {
    buffer: ReadbackBufferLease,
    gpu: GpuContext,
    size: RenderSize,
    padded_bytes_per_row: u32,
    unpadded_bytes_per_row: u32,
    #[cfg(not(target_arch = "wasm32"))]
    submission: wgpu::SubmissionIndex,
    #[cfg(not(target_arch = "wasm32"))]
    timeout: std::time::Duration,
    staging_bytes: u64,
    image_bytes: u64,
    canary: Canary,
    mapping_requested: AtomicBool,
}

impl Rgba8Readback {
    /// Test-only: expect a different canary, simulating a stale staging buffer.
    #[cfg(test)]
    pub(crate) fn expecting(mut self, canary: Canary) -> Self {
        self.canary = canary;
        self
    }

    /// The pixel dimensions of the result.
    #[must_use]
    pub const fn size(&self) -> RenderSize {
        self.size
    }

    /// Bytes held by the staging buffer.
    #[must_use]
    pub const fn staging_bytes(&self) -> u64 {
        self.staging_bytes
    }

    /// Blocks a native caller until this readback is mapped, without draining the whole queue.
    ///
    /// # Errors
    /// [`RenderError::TextureReadback`] when mapping or device polling fails.
    #[cfg(not(target_arch = "wasm32"))]
    pub fn wait_ready(&self) -> Result<(), RenderError> {
        let (sender, receiver) = mpsc::channel();
        self.map_async(move |mapped| {
            let _ = sender.send(mapped);
        });
        self.gpu
            .device()
            .poll(wgpu::PollType::Wait {
                submission_index: Some(self.submission.clone()),
                timeout: Some(self.timeout),
            })
            .map_err(crate::error::poll_error)?;
        receiver
            .recv()
            .map_err(|_| RenderError::TextureReadback(ReadbackFailure::CallbackDropped))?
    }

    /// Waits for this readback and returns tightly packed RGBA8 pixels.
    ///
    /// # Errors
    /// [`RenderError::TextureReadback`] when mapping, polling or row packing fails.
    #[cfg(not(target_arch = "wasm32"))]
    pub fn wait(self) -> Result<Vec<u8>, RenderError> {
        self.wait_ready()?;
        self.finish()
    }

    /// Requests buffer mapping and reports readiness exactly once.
    ///
    /// A refused or repeated request reports a failure, which callers must treat as a failed frame.
    pub fn map_async(&self, on_ready: impl FnOnce(Result<(), RenderError>) + Send + 'static) {
        map_staging(&self.gpu, &self.buffer, &self.mapping_requested, on_ready);
    }

    /// Polls backends that need it to resolve a mapping; browser `WebGPU` ignores this.
    pub fn poll(&self) {
        let _ = self.gpu.device().poll(wgpu::PollType::Poll);
    }

    /// Packs the mapped rows into tightly packed premultiplied RGBA8.
    ///
    /// # Errors
    /// [`RenderError::TextureReadback`] when the buffer is not mapped, and
    /// [`RenderError::GpuWorkDropped`] when the GPU did not run the copy; the buffer is then not
    /// recycled and the renderer refuses further work.
    pub fn finish(self) -> Result<Vec<u8>, RenderError> {
        let Ok(view) = self.buffer.buffer().slice(..).get_mapped_range() else {
            // Unmapping an unmapped buffer is a validation error. Dropping the lease destroys the
            // buffer and ends any pending mapping.
            return Err(RenderError::TextureReadback(ReadbackFailure::NotMapped));
        };
        let image_bytes = usize::try_from(self.image_bytes)
            .map_err(|_| RenderError::TextureReadback(ReadbackFailure::Layout))?;
        let checked = view
            .get(image_bytes..)
            .ok_or(RenderError::TextureReadback(ReadbackFailure::Layout))
            .and_then(|found| self.canary.check(found));
        if let Err(error) = checked {
            drop(view);
            self.buffer.buffer().unmap();
            return Err(error);
        }
        let mapped = &view[..image_bytes];
        let padded_row = usize::try_from(self.padded_bytes_per_row)
            .map_err(|_| RenderError::TextureReadback(ReadbackFailure::Layout))?;
        let unpadded_row = usize::try_from(self.unpadded_bytes_per_row)
            .map_err(|_| RenderError::TextureReadback(ReadbackFailure::Layout))?;
        let packed = if padded_row == unpadded_row {
            mapped.to_vec()
        } else {
            let capacity = unpadded_row
                .checked_mul(
                    usize::try_from(self.size.height)
                        .map_err(|_| RenderError::TextureReadback(ReadbackFailure::Layout))?,
                )
                .ok_or(RenderError::TextureReadback(ReadbackFailure::Layout))?;
            let mut packed = Vec::with_capacity(capacity);
            for row in mapped.chunks_exact(padded_row) {
                packed.extend_from_slice(&row[..unpadded_row]);
            }
            packed
        };
        drop(view);
        self.buffer.buffer().unmap();
        self.buffer.recycle();
        Ok(packed)
    }
}

/// GPU state an NV12 pack recorded into a caller's encoder still needs.
///
/// Drop it only after that encoder is submitted; dropping earlier lets a later renderer call
/// overwrite the pack's uniforms.
#[must_use = "hold the pack until its encoder is submitted"]
#[derive(Debug)]
#[cfg_attr(not(feature = "wgpu-interop"), doc(hidden))]
pub struct EncodedNv12Pack {
    uniforms: Option<crate::uniform::UniformArenaLease>,
    /// Defers destruction of released pool resources until the pack is dropped.
    _encode: crate::retire::EncodeScope,
}

impl Drop for EncodedNv12Pack {
    fn drop(&mut self) {
        if let Some(uniforms) = self.uniforms.take() {
            uniforms.recycle();
        }
    }
}

/// Caller-owned NV12 plane textures for a readback-free pack.
///
/// `luma` is `R8Unorm` at the output size; `chroma` is `Rg8Unorm` at half size with U in red and
/// V in green.
#[derive(Clone, Copy, Debug)]
#[cfg_attr(not(feature = "wgpu-interop"), doc(hidden))]
pub struct Nv12PlaneTargets<'a> {
    /// Full-size single-channel luma target.
    pub luma: &'a wgpu::Texture,
    /// Half-size two-channel interleaved chroma target.
    pub chroma: &'a wgpu::Texture,
}

/// A caller-owned destination for one packed plane.
#[derive(Debug)]
pub struct PlaneTarget<'a> {
    pub(crate) bytes: &'a mut [u8],
    pub(crate) stride: usize,
}

impl<'a> PlaneTarget<'a> {
    /// Describes plane memory whose rows start `stride` bytes apart.
    #[must_use]
    pub fn new(bytes: &'a mut [u8], stride: usize) -> Self {
        Self { bytes, stride }
    }

    /// The plane memory.
    #[must_use]
    pub fn bytes(&self) -> &[u8] {
        self.bytes
    }

    /// Reborrows the plane memory.
    pub fn reborrow(&mut self) -> PlaneTarget<'_> {
        PlaneTarget {
            bytes: self.bytes,
            stride: self.stride,
        }
    }

    pub(crate) fn fits(&self, width: usize, height: usize) -> Result<(), RenderError> {
        if self.stride < width {
            return Err(RenderError::TextureReadback(
                ReadbackFailure::DestinationTooSmall,
            ));
        }
        // The last row needs only its visible width, as `FFmpeg` sizes a frame's final plane.
        let needed = self
            .stride
            .checked_mul(height.saturating_sub(1))
            .and_then(|rows| rows.checked_add(width))
            .ok_or(RenderError::TextureReadback(
                ReadbackFailure::DestinationTooSmall,
            ))?;
        if self.bytes.len() < needed {
            return Err(RenderError::TextureReadback(
                ReadbackFailure::DestinationTooSmall,
            ));
        }
        Ok(())
    }

    fn row(&mut self, index: usize, width: usize) -> Result<&mut [u8], RenderError> {
        let start = index
            .checked_mul(self.stride)
            .ok_or(RenderError::TextureReadback(
                ReadbackFailure::DestinationTooSmall,
            ))?;
        let end = start
            .checked_add(width)
            .ok_or(RenderError::TextureReadback(
                ReadbackFailure::DestinationTooSmall,
            ))?;
        self.bytes
            .get_mut(start..end)
            .ok_or(RenderError::TextureReadback(
                ReadbackFailure::DestinationTooSmall,
            ))
    }
}

/// Destinations for the three planes of one packed frame.
#[derive(Debug)]
pub struct Yuv420pTargets<'a> {
    /// Full-resolution luma.
    pub y: PlaneTarget<'a>,
    /// Half-resolution blue-difference chroma.
    pub u: PlaneTarget<'a>,
    /// Half-resolution red-difference chroma.
    pub v: PlaneTarget<'a>,
}

/// Tightly packed planar YUV420 data produced by a GPU readback.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Yuv420pPlanes {
    /// Full-resolution luma samples.
    pub y: Vec<u8>,
    /// Half-width, half-height blue-difference chroma samples.
    pub u: Vec<u8>,
    /// Half-width, half-height red-difference chroma samples.
    pub v: Vec<u8>,
}

/// A submitted GPU RGB-to-YUV420 conversion whose buffer is not mapped yet.
#[derive(Debug)]
pub struct Yuv420pReadback {
    buffer: ReadbackBufferLease,
    gpu: GpuContext,
    size: RenderSize,
    layout: Yuv420StagingLayout,
    /// Set for a texel readback packed on the CPU ([`Renderer::begin_layout_readback`]); `layout`
    /// is unused then.
    texels: Option<Box<TexelStaging>>,
    #[cfg(not(target_arch = "wasm32"))]
    submission: wgpu::SubmissionIndex,
    #[cfg(not(target_arch = "wasm32"))]
    timeout: std::time::Duration,
    staging_bytes: u64,
    canary: Canary,
    mapping_requested: AtomicBool,
}

impl Yuv420pReadback {
    /// The luma-plane dimensions.
    #[must_use]
    pub const fn size(&self) -> RenderSize {
        self.size
    }

    /// Bytes held by the staging buffer.
    #[must_use]
    pub const fn staging_bytes(&self) -> u64 {
        self.staging_bytes
    }

    /// Blocks a native caller until this readback is mapped.
    ///
    /// # Errors
    /// Returns [`RenderError::TextureReadback`] when mapping or polling fails.
    #[cfg(not(target_arch = "wasm32"))]
    pub fn wait_ready(&self) -> Result<(), RenderError> {
        let (sender, receiver) = mpsc::channel();
        self.map_async(move |mapped| {
            let _ = sender.send(mapped);
        });
        self.wait_for_submission()?;
        receiver
            .recv()
            .map_err(|_| RenderError::TextureReadback(ReadbackFailure::CallbackDropped))?
    }

    /// Blocks until this readback's submission completes; may follow an earlier
    /// [`Self::map_async`].
    ///
    /// # Errors
    /// [`RenderError::GpuTimeout`] past the renderer's wait timeout, or
    /// [`RenderError::TextureReadback`] when device polling fails.
    #[cfg(not(target_arch = "wasm32"))]
    pub fn wait_for_submission(&self) -> Result<(), RenderError> {
        self.gpu
            .device()
            .poll(wgpu::PollType::Wait {
                submission_index: Some(self.submission.clone()),
                timeout: Some(self.timeout),
            })
            .map_err(crate::error::poll_error)?;
        Ok(())
    }

    /// Waits for this readback and returns tightly packed planes.
    ///
    /// # Errors
    /// [`RenderError::TextureReadback`] when mapping, polling or row packing fails.
    #[cfg(not(target_arch = "wasm32"))]
    pub fn wait(self) -> Result<Yuv420pPlanes, RenderError> {
        self.wait_ready()?;
        self.finish()
    }

    /// Requests buffer mapping and reports readiness exactly once.
    pub fn map_async(&self, on_ready: impl FnOnce(Result<(), RenderError>) + Send + 'static) {
        map_staging(&self.gpu, &self.buffer, &self.mapping_requested, on_ready);
    }

    /// Polls backends that need it to resolve a mapping.
    pub fn poll(&self) {
        let _ = self.gpu.device().poll(wgpu::PollType::Poll);
    }

    /// Packs mapped planes into caller-owned, possibly strided, plane buffers, avoiding the
    /// allocation and copy of [`Yuv420pReadback::finish`].
    ///
    /// # Errors [`RenderError::TextureReadback`] when the buffer is not mapped, the layout is
    /// unrepresentable or a target is too small, and [`RenderError::GpuWorkDropped`] when the GPU
    /// did not run the copy (the buffer is not recycled and the renderer refuses further work).
    pub fn finish_into(self, targets: &mut Yuv420pTargets<'_>) -> Result<(), RenderError> {
        if self.texels.is_some() {
            return Err(RenderError::TextureReadback(ReadbackFailure::Layout));
        }
        let Ok(mapped) = self.buffer.buffer().slice(..).get_mapped_range() else {
            // Not mapped, so nothing to unmap.
            return Err(RenderError::TextureReadback(ReadbackFailure::NotMapped));
        };
        let result = (|| {
            self.check_canary(&mapped)?;
            let planes = self.layout.planes(self.size);
            let destinations = [&mut targets.y, &mut targets.u, &mut targets.v];
            for (plane, target) in planes.into_iter().zip(destinations) {
                let (_, _, width, rows) = plane;
                let width = usize::try_from(width)
                    .map_err(|_| RenderError::TextureReadback(ReadbackFailure::Layout))?;
                let rows = usize::try_from(rows)
                    .map_err(|_| RenderError::TextureReadback(ReadbackFailure::Layout))?;
                target.fits(width, rows)?;
                for (index, row) in plane_rows(&mapped, plane)?.enumerate() {
                    target.row(index, width)?.copy_from_slice(&row[..width]);
                }
            }
            Ok(())
        })();
        drop(mapped);
        self.buffer.buffer().unmap();
        if result.is_ok() {
            self.buffer.recycle();
        }
        result
    }

    /// Packs mapped planes into three tightly packed planes.
    ///
    /// # Errors
    /// As [`Self::finish_into`].
    pub fn finish(self) -> Result<Yuv420pPlanes, RenderError> {
        let planes = self.unpack(|[y, u, v]| Yuv420pPlanes { y, u, v })?;
        Ok(planes)
    }

    /// Plane offsets and strides in the bytes [`Self::finish_mapped`] hands over.
    #[must_use]
    #[cfg_attr(not(feature = "wgpu-interop"), doc(hidden))]
    pub fn plane_layout(&self) -> [StagingPlane; 3] {
        self.layout
            .planes(self.size)
            .map(|(offset, stride, _, _)| StagingPlane { offset, stride })
    }

    /// Hands the mapped staging buffer to `read`, then unmaps it.
    ///
    /// Planes are at the offsets and strides [`Self::plane_layout`] reports, rows padded to the
    /// copy alignment. A strided consumer such as a `WebCodecs` `VideoFrame` can copy straight from
    /// the mapping; in a browser that copy can stay in JavaScript (`BufferView::as_uint8array`).
    ///
    /// # Errors
    /// As [`Self::finish_into`]; `read` is not called then.
    #[cfg_attr(not(feature = "wgpu-interop"), doc(hidden))]
    pub fn finish_mapped<T>(
        self,
        read: impl FnOnce(&wgpu::BufferView) -> T,
    ) -> Result<T, RenderError> {
        let Ok(canary) = self
            .buffer
            .buffer()
            .slice(self.image_bytes()..)
            .get_mapped_range()
        else {
            // Not mapped, so nothing to unmap.
            return Err(RenderError::TextureReadback(ReadbackFailure::NotMapped));
        };
        let checked = self.canary.check(&canary);
        drop(canary);
        if let Err(error) = checked {
            self.buffer.buffer().unmap();
            return Err(error);
        }
        let Ok(mapped) = self
            .buffer
            .buffer()
            .slice(..self.image_bytes())
            .get_mapped_range()
        else {
            self.buffer.buffer().unmap();
            return Err(RenderError::TextureReadback(ReadbackFailure::NotMapped));
        };
        let result = read(&mapped);
        drop(mapped);
        self.buffer.buffer().unmap();
        self.buffer.recycle();
        Ok(result)
    }

    /// Copies the three planes out tightly packed and hands them to `pack`.
    fn unpack<T>(self, pack: impl FnOnce([Vec<u8>; 3]) -> T) -> Result<T, RenderError> {
        if self.texels.is_some() {
            return Err(RenderError::TextureReadback(ReadbackFailure::Layout));
        }
        let Ok(mapped) = self.buffer.buffer().slice(..).get_mapped_range() else {
            // Not mapped, so nothing to unmap.
            return Err(RenderError::TextureReadback(ReadbackFailure::NotMapped));
        };
        if let Err(error) = self.check_canary(&mapped) {
            drop(mapped);
            self.buffer.buffer().unmap();
            return Err(error);
        }
        let planes = self.layout.planes(self.size).map(|plane| {
            let (_, _, width, rows) = plane;
            let width = usize::try_from(width)
                .map_err(|_| RenderError::TextureReadback(ReadbackFailure::Layout))?;
            let rows = usize::try_from(rows)
                .map_err(|_| RenderError::TextureReadback(ReadbackFailure::Layout))?;
            plane_rows(&mapped, plane).map(|source| {
                let mut tight = Vec::with_capacity(width * rows);
                for row in source {
                    tight.extend_from_slice(&row[..width]);
                }
                tight
            })
        });
        drop(mapped);
        self.buffer.buffer().unmap();
        let [y, u, v] = planes;
        let planes = [y?, u?, v?];
        self.buffer.recycle();
        Ok(pack(planes))
    }

    /// Bytes before the canary.
    fn image_bytes(&self) -> u64 {
        self.texels
            .as_ref()
            .map_or(self.layout.bytes, |texels| texels.bytes)
    }

    /// The layout a texel readback packs into, `None` for a GPU-packed 8-bit 4:2:0 readback.
    #[must_use]
    pub fn pixel_layout(&self) -> Option<PixelLayout> {
        self.texels.as_ref().map(|texels| texels.layout)
    }

    /// Packs the mapped frame into one target per plane of its layout, in storage order
    /// ([`PixelLayout::planes`]).
    ///
    /// # Errors
    /// As [`Self::finish_into`], and [`RenderError::TextureReadback`] for a wrong target count.
    pub fn finish_into_planes(self, targets: &mut [PlaneTarget<'_>]) -> Result<(), RenderError> {
        let Some(texels) = self.texels.as_deref().cloned() else {
            let [y, u, v] = targets else {
                return Err(RenderError::TextureReadback(ReadbackFailure::Layout));
            };
            let mut planes = Yuv420pTargets {
                y: PlaneTarget::new(y.bytes, y.stride),
                u: PlaneTarget::new(u.bytes, u.stride),
                v: PlaneTarget::new(v.bytes, v.stride),
            };
            return self.finish_into(&mut planes);
        };
        let Ok(mapped) = self.buffer.buffer().slice(..).get_mapped_range() else {
            // Not mapped, so nothing to unmap.
            return Err(RenderError::TextureReadback(ReadbackFailure::NotMapped));
        };
        let result = (|| {
            self.check_canary(&mapped)?;
            let (width, height) = (self.size.width as usize, self.size.height as usize);
            let pitch = texels.pitch as usize;
            let rows = mapped
                .get(..pitch * height)
                .ok_or(RenderError::TextureReadback(ReadbackFailure::Layout))?;
            let row = |y: usize, out: &mut [[f32; 4]]| {
                let texels_row = &rows[y * pitch..];
                decode_texels(texels.format, texels_row, &mut out[..width]);
            };
            let picture = crate::layout_pack::Picture {
                width,
                height,
                row: &row,
            };
            crate::layout_pack::pack(&picture, texels.encoding, texels.layout, targets)
        })();
        drop(mapped);
        self.buffer.buffer().unmap();
        if result.is_ok() {
            self.buffer.recycle();
        }
        result
    }

    /// Checks that the mapped buffer ends in this readback's canary; a failing buffer is not
    /// recycled.
    fn check_canary(&self, mapped: &[u8]) -> Result<(), RenderError> {
        let start = usize::try_from(self.image_bytes())
            .map_err(|_| RenderError::TextureReadback(ReadbackFailure::Layout))?;
        let found = mapped
            .get(start..)
            .ok_or(RenderError::TextureReadback(ReadbackFailure::Layout))?;
        self.canary.check(found)
    }
}

fn map_staging(
    gpu: &GpuContext,
    buffer: &ReadbackBufferLease,
    requested: &AtomicBool,
    on_ready: impl FnOnce(Result<(), RenderError>) + Send + 'static,
) {
    if requested.swap(true, Ordering::AcqRel) {
        on_ready(Err(RenderError::TextureReadback(
            ReadbackFailure::DuplicateMapRequest,
        )));
        return;
    }
    gpu.map_async(
        &buffer.buffer().slice(..),
        wgpu::MapMode::Read,
        move |result| {
            // `BufferAsyncError` names no cause; device diagnostics may explain it.
            on_ready(result.map_err(|_| RenderError::TextureReadback(ReadbackFailure::MapFailed)));
        },
    );
}

#[derive(Clone, Debug)]
struct TexelStaging {
    format: wgpu::TextureFormat,
    pitch: u32,
    bytes: u64,
    layout: PixelLayout,
    encoding: YuvEncoding,
}

fn decode_texels(format: wgpu::TextureFormat, row: &[u8], out: &mut [[f32; 4]]) {
    match format {
        wgpu::TextureFormat::Rgba16Float => {
            for (texel, bytes) in out.iter_mut().zip(row.as_chunks::<8>().0) {
                *texel = std::array::from_fn(|channel| {
                    crate::cpu::half_value(u16::from_le_bytes([
                        bytes[channel * 2],
                        bytes[channel * 2 + 1],
                    ]))
                });
            }
        }
        _ => {
            for (texel, bytes) in out.iter_mut().zip(row.as_chunks::<4>().0) {
                *texel = std::array::from_fn(|channel| crate::cpu::unorm8_value(bytes[channel]));
            }
        }
    }
}

/// Where one plane sits in a mapped readback staging buffer.
#[derive(Copy, Clone, Debug, Eq, PartialEq)]
pub struct StagingPlane {
    /// Byte offset of the plane's first row.
    pub offset: u64,
    /// Bytes from the start of one row to the start of the next.
    pub stride: u32,
}

fn aligned_copy_row(width: u32) -> Result<u32, RenderError> {
    let alignment = wgpu::COPY_BYTES_PER_ROW_ALIGNMENT;
    width
        .checked_add(alignment - 1)
        .map(|value| value / alignment)
        .and_then(|rows| rows.checked_mul(alignment))
        .ok_or(RenderError::TextureReadback(ReadbackFailure::Layout))
}

fn yuv420_tight_bytes(size: RenderSize) -> Result<u64, RenderError> {
    u64::from(size.width)
        .checked_mul(u64::from(size.height))
        .and_then(|luma| luma.checked_mul(YUV420_PLANE_COUNT))
        .map(|three_planes| three_planes / 2)
        .ok_or(RenderError::TextureReadback(ReadbackFailure::Layout))
}

/// Pack-shader uniforms for `encoding`: luma weights, range levels and chroma co-siting axes,
/// from the same definitions import decodes with so a round trip is the identity.
pub(crate) fn yuv_uniform(encoding: YuvEncoding) -> [f32; 12] {
    let YuvEncoding { color, chroma } = encoding;
    // Components that are already RGB have no pack: they take BT.709's.
    let (kr, kb) = color
        .matrix
        .luma_weights()
        .or(ColorMatrix::Bt709.luma_weights())
        .expect("BT.709 is a YCbCr matrix");
    let levels = color.range.levels(8);
    let (cosited_x, cosited_y) = match chroma {
        ChromaLocation::Left => (true, false),
        ChromaLocation::TopLeft => (true, true),
        ChromaLocation::Unspecified | ChromaLocation::Center => (false, false),
    };
    [
        kr,
        1.0 - kr - kb,
        kb,
        0.0,
        levels.luma_black / levels.maximum,
        levels.luma_span / levels.maximum,
        levels.chroma_center / levels.maximum,
        levels.chroma_span / levels.maximum,
        f32::from(u8::from(cosited_x)),
        f32::from(u8::from(cosited_y)),
        0.0,
        0.0,
    ]
}

fn yuv_render_pipeline(
    device: &wgpu::Device,
    layout: &wgpu::PipelineLayout,
    shader: &wgpu::ShaderModule,
    fragment_entry: &'static str,
    target_formats: &[wgpu::TextureFormat],
) -> wgpu::RenderPipeline {
    let targets = target_formats
        .iter()
        .copied()
        .map(|format| Some(format.into()))
        .collect::<Vec<_>>();
    device.create_render_pipeline(&wgpu::RenderPipelineDescriptor {
        label: Some("alpha2 RGB to YUV pipeline"),
        layout: Some(layout),
        vertex: wgpu::VertexState {
            module: shader,
            entry_point: Some("vs"),
            buffers: &[],
            compilation_options: wgpu::PipelineCompilationOptions::default(),
        },
        fragment: Some(wgpu::FragmentState {
            module: shader,
            entry_point: Some(fragment_entry),
            targets: &targets,
            compilation_options: wgpu::PipelineCompilationOptions::default(),
        }),
        primitive: wgpu::PrimitiveState::default(),
        depth_stencil: None,
        multisample: wgpu::MultisampleState::default(),
        multiview_mask: None,
        cache: None,
    })
}

fn encode_yuv_render_pass(
    encoder: &mut wgpu::CommandEncoder,
    label: &'static str,
    pipeline: &wgpu::RenderPipeline,
    bind_group: &wgpu::BindGroup,
    target: &wgpu::TextureView,
) {
    let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
        label: Some(label),
        color_attachments: &[Some(wgpu::RenderPassColorAttachment {
            view: target,
            depth_slice: None,
            resolve_target: None,
            ops: wgpu::Operations {
                load: wgpu::LoadOp::Clear(wgpu::Color::BLACK),
                store: wgpu::StoreOp::Store,
            },
        })],
        depth_stencil_attachment: None,
        timestamp_writes: None,
        occlusion_query_set: None,
        multiview_mask: None,
    });
    pass.set_pipeline(pipeline);
    pass.set_bind_group(0, bind_group, &[]);
    pass.draw(0..3, 0..1);
}

fn encode_yuv_render_pass_pair(
    encoder: &mut wgpu::CommandEncoder,
    label: &'static str,
    pipeline: &wgpu::RenderPipeline,
    bind_group: &wgpu::BindGroup,
    targets: [&wgpu::TextureView; 2],
) {
    let attachments = targets.map(|target| {
        Some(wgpu::RenderPassColorAttachment {
            view: target,
            depth_slice: None,
            resolve_target: None,
            ops: wgpu::Operations {
                load: wgpu::LoadOp::Clear(wgpu::Color::BLACK),
                store: wgpu::StoreOp::Store,
            },
        })
    });
    let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
        label: Some(label),
        color_attachments: &attachments,
        depth_stencil_attachment: None,
        timestamp_writes: None,
        occlusion_query_set: None,
        multiview_mask: None,
    });
    pass.set_pipeline(pipeline);
    pass.set_bind_group(0, bind_group, &[]);
    pass.draw(0..3, 0..1);
}

fn encode_plane_copy(
    encoder: &mut wgpu::CommandEncoder,
    texture: &wgpu::Texture,
    buffer: &wgpu::Buffer,
    offset: u64,
    bytes_per_row: u32,
    size: RenderSize,
) {
    encoder.copy_texture_to_buffer(
        wgpu::TexelCopyTextureInfo {
            texture,
            mip_level: 0,
            origin: wgpu::Origin3d::ZERO,
            aspect: wgpu::TextureAspect::All,
        },
        wgpu::TexelCopyBufferInfo {
            buffer,
            layout: wgpu::TexelCopyBufferLayout {
                offset,
                bytes_per_row: Some(bytes_per_row),
                rows_per_image: Some(size.height),
            },
        },
        wgpu::Extent3d {
            width: size.width,
            height: size.height,
            depth_or_array_layers: 1,
        },
    );
}

#[cfg(test)]
pub(crate) fn read_rgba8(renderer: &Renderer, target: &RenderedTexture) -> Vec<u8> {
    renderer
        .read_rgba8(target)
        .expect("test texture readback succeeds")
}

#[cfg(test)]
#[path = "readback_tests.rs"]
mod layout_tests;