medias 0.6.1

//! Vulkan pass over decoder frames: PQ BT.2020 SAND128 P030 dmabufs to SDR
//! BT.709 in an XRGB8888 scanout buffer (shaders/video.frag). Technique from
//! pi-dv-engine (RESEARCH.md): decoder buffers as R32UI texel buffers.

use std::os::fd::{AsFd, AsRawFd, BorrowedFd, OwnedFd};

use ash::vk;
use drm::buffer::DrmFourcc;
use drm::control::{Device as ControlDevice, framebuffer};

use super::text::{barrier, module};
use super::{Gpu, Target};
use crate::gbm::{Allocator, Scanout};
use crate::kms::{Card, Output};

const VERT_SPV: &[u8] = include_bytes!("../../shaders/video.vert.spv");
const FRAG_SPV: &[u8] = include_bytes!("../../shaders/video.frag.spv");

/// Table edge; error against the exact map in `cube_error_is_bounded`.
const CUBE: usize = 65;
/// Luminance of SDR white, cd/m2 (BT.2408).
pub const SDR_WHITE: f32 = 203.0;
/// Imports unused for this many renders are released (decoder pool turnover).
const IMPORT_AGE: u64 = 64;

/// Push constants of video.frag.
#[repr(C)]
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Push {
    /// Word offset luma, chroma; column height luma, chroma.
    pub lay: [u32; 4],
    /// Rows: 10-bit YCbCr codes to table coordinates (xyz), offset (w).
    pub m: [[f32; 4]; 3],
}

/// BT.2020 to BT.709 primaries, linear light (BT.2087).
const BT2020_TO_709: [[f32; 3]; 3] = [
    [1.6605, -0.5876, -0.0728],
    [-0.1246, 1.1329, -0.0083],
    [-0.0182, -0.1006, 1.1187],
];

impl Push {
    pub fn new(lay: [u32; 4], full_range: bool) -> Push {
        // R'G'B' 0..1 to the texel centres of the table.
        let (scale, bias) = ((CUBE - 1) as f32 / CUBE as f32, 0.5 / CUBE as f32);
        let m = ycbcr_rows(0.2627, 0.0593, full_range).map(|r| [r[0] * scale, r[1] * scale, r[2] * scale, r[3] * scale + bias]);
        Push { lay, m }
    }

    /// Table indexed by the full-range 10-bit codes (Y, Cb, Cr) / 1023.
    pub fn for_codes(lay: [u32; 4]) -> Push {
        let (scale, bias) = ((CUBE - 1) as f32 / CUBE as f32 / 1023.0, 0.5 / CUBE as f32);
        Push { lay, m: [[scale, 0.0, 0.0, bias], [0.0, scale, 0.0, bias], [0.0, 0.0, scale, bias]] }
    }

    fn bytes(&self) -> &[u8] {
        unsafe { std::slice::from_raw_parts((self as *const Push).cast::<u8>(), std::mem::size_of::<Push>()) }
    }
}

/// Rows mapping 10-bit Y'CbCr codes to R'G'B' in 0..1 for luma coefficients `kr`, `kb`.
pub fn ycbcr_rows(kr: f32, kb: f32, full_range: bool) -> [[f32; 4]; 3] {
    let kg = 1.0 - kr - kb;
    let (ys, yo, cs) = if full_range { (1.0 / 1023.0, 0.0, 1.0 / 1023.0) } else { (1.0 / 876.0, 64.0, 1.0 / 896.0) };
    let rows = [
        [1.0, 0.0, 2.0 * (1.0 - kr)],
        [1.0, -2.0 * kb * (1.0 - kb) / kg, -2.0 * kr * (1.0 - kr) / kg],
        [1.0, 2.0 * (1.0 - kb), 0.0],
    ];
    rows.map(|[y, cb, cr]| [y * ys, cb * cs, cr * cs, -(y * ys * yo + (cb + cr) * cs * 512.0)])
}

/// SMPTE ST 2084 EOTF: signal 0..1 to cd/m2.
pub fn pq_eotf(e: f32) -> f32 {
    let p = e.max(0.0).powf(1.0 / 78.84375);
    10000.0 * ((p - 0.8359375).max(0.0) / (18.8515625 - 18.6875 * p)).powf(1.0 / 0.1593017578125)
}

/// Inverse of `pq_eotf`.
pub fn pq_oetf(nits: f32) -> f32 {
    let y = (nits.max(0.0) / 10000.0).powf(0.1593017578125);
    ((0.8359375 + 18.8515625 * y) / (1.0 + 18.6875 * y)).powf(78.84375)
}

/// BT.2390 EETF without black lift: PQ signal mastered to `peak` cd/m2 onto a `white` cd/m2 display.
pub fn eetf(e: f32, peak: f32, white: f32) -> f32 {
    let top = pq_oetf(peak);
    let max_lum = pq_oetf(white) / top;
    if max_lum >= 1.0 {
        return e.min(pq_oetf(white));
    }
    let e1 = (e / top).min(1.0);
    let ks = 1.5 * max_lum - 0.5;
    if e1 < ks {
        return e1 * top;
    }
    let t = (e1 - ks) / (1.0 - ks);
    let (t2, t3) = (t * t, t * t * t);
    ((2.0 * t3 - 3.0 * t2 + 1.0) * ks + (t3 - 2.0 * t2 + t) * (1.0 - ks) + (-2.0 * t3 + 3.0 * t2) * max_lum) * top
}

/// Mastering peak in cd/m2 from mpv's `max-luma`. mpv reports 10000 for a PQ
/// stream without mastering metadata (and 0 elsewhere); both give 1000.
pub fn mastering_peak(max_luma: Option<f64>) -> f32 {
    max_luma.filter(|v| *v > 0.0 && *v < 10000.0).unwrap_or(1000.0) as f32
}

/// Transfer function of the video signal.
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum Transfer {
    Pq,
    Hlg,
}

/// Range, transfer and mastering peak (cd/m2, PQ only) of a video.
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Signal {
    pub full_range: bool,
    pub transfer: Transfer,
    pub peak: f32,
}

/// Colour conversion of a frame: a video signal, or a prepared table indexed by the
/// full-range codes (Y, Cb, Cr) / 1023 (`cube_of`); `id` changes with the table's contents.
#[derive(Clone, Copy)]
pub enum Colour<'a> {
    Signal(Signal),
    Cube { id: u64, table: &'a Table },
}

/// A prepared table. `hdr`: values are PQ BT.2020 R'G'B' as RGBA16F for a 10-bit
/// target; else gamma 2.4 BT.709 as RGBA8 for an 8-bit target.
pub struct Table {
    pub hdr: bool,
    pub bytes: Vec<u8>,
}

impl Table {
    /// `map` sampled at (x, y, z) / (CUBE - 1), x fastest.
    pub fn of(hdr: bool, map: impl Fn([f32; 3]) -> [f32; 3]) -> Table {
        let bytes = if hdr {
            cube_points().flat_map(|p| map(p).into_iter().chain([1.0]).flat_map(|v| f16_bits(v.clamp(0.0, 1.0)).to_ne_bytes())).collect()
        } else {
            cube_of(map).into_flattened()
        };
        Table { hdr, bytes }
    }
}

fn cube_points() -> impl Iterator<Item = [f32; 3]> {
    let at = |i: usize| i as f32 / (CUBE - 1) as f32;
    (0..CUBE * CUBE * CUBE).map(move |i| [at(i % CUBE), at(i / CUBE % CUBE), at(i / (CUBE * CUBE))])
}

/// IEEE 754 binary16 bits of `v`, round to nearest.
pub fn f16_bits(v: f32) -> u16 {
    let bits = v.to_bits();
    let sign = ((bits >> 16) & 0x8000) as u16;
    let exp = ((bits >> 23) & 0xff) as i32 - 127 + 15;
    let frac = bits & 0x7f_ffff;
    if exp >= 31 {
        return sign | 0x7c00;
    }
    if exp <= 0 {
        if exp < -10 {
            return sign;
        }
        let m = (frac | 0x80_0000) >> (1 - exp);
        return sign | ((m + 0x1000) >> 13) as u16;
    }
    // A mantissa carry increments the exponent field.
    sign | ((((exp as u32) << 10) | (frac >> 13)) + ((frac >> 12) & 1)) as u16
}

#[derive(Clone, Copy, PartialEq)]
enum TableKey {
    Signal(Transfer, f32),
    Cube(u64),
}

/// Nominal peak of the HLG reference display, cd/m2 (BT.2100); system gamma 1.2.
const HLG_PEAK: f32 = 1000.0;

/// BT.2100 HLG R'G'B' (0..1) to display light in cd/m2: inverse OETF, then the
/// OOTF Y^0.2 for a 1000 cd/m2 display. Signal 0.75 gives 203 cd/m2.
pub fn hlg_display(e: [f32; 3]) -> [f32; 3] {
    let (a, b, c) = (0.17883277f32, 0.28466892f32, 0.55991073f32);
    let scene = e.map(|v| {
        let v = v.clamp(0.0, 1.0);
        if v <= 0.5 { v * v / 3.0 } else { (((v - c) / a).exp() + b) / 12.0 }
    });
    let y = 0.2627 * scene[0] + 0.6780 * scene[1] + 0.0593 * scene[2];
    let gain = HLG_PEAK * y.powf(0.2);
    scene.map(|v| v * gain)
}

/// PQ or HLG BT.2020 R'G'B' (0..1, PQ mastering peak `peak` cd/m2) to gamma 2.4 BT.709 R'G'B' with
/// `white` cd/m2 as 1.0: BT.2390 EETF as a gain by max(R'G'B'), BT.2087 primaries, clip.
pub fn tone_map(e: [f32; 3], transfer: Transfer, peak: f32, white: f32) -> [f32; 3] {
    match transfer {
        Transfer::Pq => tone_map_light(e.map(pq_eotf), peak, white),
        Transfer::Hlg => tone_map_light(hlg_display(e), HLG_PEAK, white),
    }
}

/// Linear BT.2020 light in cd/m2 (peak `peak`) to gamma 2.4 BT.709 R'G'B' with `white` cd/m2 as 1.0.
pub fn tone_map_light(nits: [f32; 3], peak: f32, white: f32) -> [f32; 3] {
    let top = nits[0].max(nits[1]).max(nits[2]);
    let gain = if top > 0.0 { pq_eotf(eetf(pq_oetf(top), peak, white)) / top } else { 1.0 };
    let lin = nits.map(|v| v * gain / white);
    BT2020_TO_709.map(|r| (r[0] * lin[0] + r[1] * lin[1] + r[2] * lin[2]).clamp(0.0, 1.0).powf(1.0 / 2.4))
}

/// `tone_map` sampled at (r, g, b) / (CUBE - 1), RGBA8, r fastest.
pub fn tone_cube(transfer: Transfer, peak: f32, white: f32) -> Vec<[u8; 4]> {
    cube_of(|e| tone_map(e, transfer, peak, white))
}

/// `map` sampled at (x, y, z) / (CUBE - 1), RGBA8, x fastest.
pub fn cube_of(map: impl Fn([f32; 3]) -> [f32; 3]) -> Vec<[u8; 4]> {
    cube_points()
        .map(|p| {
            let [x, y, z] = map(p).map(|v| (v.clamp(0.0, 1.0) * 255.0).round() as u8);
            [x, y, z, 255]
        })
        .collect()
}

/// Pipeline, table and command state; draws into a `Target`.
pub(crate) struct Pass {
    render_pass: vk::RenderPass,
    set_layout: vk::DescriptorSetLayout,
    layout: vk::PipelineLayout,
    /// Full size, half size (HALF of video.frag).
    pipelines: [vk::Pipeline; 2],
    vert: vk::ShaderModule,
    frag: vk::ShaderModule,
    desc_pool: vk::DescriptorPool,
    set: vk::DescriptorSet,
    sampler: vk::Sampler,
    lut: vk::Image,
    lut_mem: vk::DeviceMemory,
    lut_view: vk::ImageView,
    stage: vk::Buffer,
    stage_mem: vk::DeviceMemory,
    pool: vk::CommandPool,
    cmd: vk::CommandBuffer,
    fence: vk::Fence,
    /// Bytes per table texel: 4 (RGBA8) or 8 (RGBA16F).
    texel: usize,
}

const CUBE_EXTENT: vk::Extent3D = vk::Extent3D { width: CUBE as u32, height: CUBE as u32, depth: CUBE as u32 };
const LUT_RANGE: vk::ImageSubresourceRange = vk::ImageSubresourceRange {
    aspect_mask: vk::ImageAspectFlags::COLOR,
    base_mip_level: 0,
    level_count: 1,
    base_array_layer: 0,
    layer_count: 1,
};

impl Pass {
    /// `out`: format of the render targets; `wide`: the table is RGBA16F, else RGBA8.
    pub(crate) fn new(gpu: &Gpu, out: vk::Format, wide: bool) -> Result<Pass, String> {
        Pass::with_mode(gpu, 0, out, wide)
    }

    /// `mode`: MODE of video.frag (0 in production).
    fn with_mode(gpu: &Gpu, mode: u32, out: vk::Format, wide: bool) -> Result<Pass, String> {
        let d = &gpu.device;
        let err = |what: &'static str| move |e: vk::Result| format!("video pass {what}: {e}");
        unsafe {
            let color = [vk::AttachmentDescription::default()
                .format(out).samples(vk::SampleCountFlags::TYPE_1)
                .load_op(vk::AttachmentLoadOp::DONT_CARE).store_op(vk::AttachmentStoreOp::STORE)
                .stencil_load_op(vk::AttachmentLoadOp::DONT_CARE).stencil_store_op(vk::AttachmentStoreOp::DONT_CARE)
                .initial_layout(vk::ImageLayout::UNDEFINED).final_layout(vk::ImageLayout::GENERAL)];
            let cref = [vk::AttachmentReference::default().attachment(0).layout(vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL)];
            let sub = [vk::SubpassDescription::default().pipeline_bind_point(vk::PipelineBindPoint::GRAPHICS).color_attachments(&cref)];
            let render_pass = d
                .create_render_pass(&vk::RenderPassCreateInfo::default().attachments(&color).subpasses(&sub), None)
                .map_err(err("render pass"))?;

            let binding = |i: u32, ty| {
                vk::DescriptorSetLayoutBinding::default().binding(i).descriptor_type(ty).descriptor_count(1).stage_flags(vk::ShaderStageFlags::FRAGMENT)
            };
            let bindings = [
                binding(0, vk::DescriptorType::UNIFORM_TEXEL_BUFFER),
                binding(1, vk::DescriptorType::UNIFORM_TEXEL_BUFFER),
                binding(2, vk::DescriptorType::COMBINED_IMAGE_SAMPLER),
            ];
            let set_layout = d
                .create_descriptor_set_layout(&vk::DescriptorSetLayoutCreateInfo::default().bindings(&bindings), None)
                .map_err(err("set layout"))?;
            let sizes = [
                vk::DescriptorPoolSize::default().ty(vk::DescriptorType::UNIFORM_TEXEL_BUFFER).descriptor_count(2),
                vk::DescriptorPoolSize::default().ty(vk::DescriptorType::COMBINED_IMAGE_SAMPLER).descriptor_count(1),
            ];
            let desc_pool = d
                .create_descriptor_pool(&vk::DescriptorPoolCreateInfo::default().max_sets(1).pool_sizes(&sizes), None)
                .map_err(err("descriptor pool"))?;
            let layouts = [set_layout];
            let set = d
                .allocate_descriptor_sets(&vk::DescriptorSetAllocateInfo::default().descriptor_pool(desc_pool).set_layouts(&layouts))
                .map_err(err("descriptor set"))?[0];
            let push = [vk::PushConstantRange::default().stage_flags(vk::ShaderStageFlags::FRAGMENT).size(std::mem::size_of::<Push>() as u32)];
            let layout = d
                .create_pipeline_layout(&vk::PipelineLayoutCreateInfo::default().set_layouts(&layouts).push_constant_ranges(&push), None)
                .map_err(err("pipeline layout"))?;
            let vert = module(d, VERT_SPV)?;
            let frag = module(d, FRAG_SPV)?;
            let pipelines = [pipeline(d, render_pass, layout, vert, frag, mode, 0)?, pipeline(d, render_pass, layout, vert, frag, mode, 1)?];

            let sampler = d
                .create_sampler(
                    &vk::SamplerCreateInfo::default()
                        .mag_filter(vk::Filter::LINEAR).min_filter(vk::Filter::LINEAR)
                        .address_mode_u(vk::SamplerAddressMode::CLAMP_TO_EDGE)
                        .address_mode_v(vk::SamplerAddressMode::CLAMP_TO_EDGE)
                        .address_mode_w(vk::SamplerAddressMode::CLAMP_TO_EDGE),
                    None,
                )
                .map_err(err("sampler"))?;
            let format = if wide { vk::Format::R16G16B16A16_SFLOAT } else { vk::Format::R8G8B8A8_UNORM };
            let lut = d
                .create_image(
                    &vk::ImageCreateInfo::default()
                        .image_type(vk::ImageType::TYPE_3D).format(format)
                        .extent(CUBE_EXTENT)
                        .mip_levels(1).array_layers(1).samples(vk::SampleCountFlags::TYPE_1)
                        .tiling(vk::ImageTiling::OPTIMAL)
                        .usage(vk::ImageUsageFlags::SAMPLED | vk::ImageUsageFlags::TRANSFER_DST),
                    None,
                )
                .map_err(err("table image"))?;
            let req = d.get_image_memory_requirements(lut);
            let mt = gpu.find_memory_type_with(req.memory_type_bits, vk::MemoryPropertyFlags::DEVICE_LOCAL).ok_or("video pass: no device memory")?;
            let lut_mem = d
                .allocate_memory(&vk::MemoryAllocateInfo::default().allocation_size(req.size).memory_type_index(mt), None)
                .map_err(err("table memory"))?;
            d.bind_image_memory(lut, lut_mem, 0).map_err(err("table bind"))?;
            let lut_view = d
                .create_image_view(
                    &vk::ImageViewCreateInfo::default().image(lut).view_type(vk::ImageViewType::TYPE_3D).format(format).subresource_range(LUT_RANGE),
                    None,
                )
                .map_err(err("table view"))?;
            let (stage, stage_mem) = host_buffer(gpu, (CUBE * CUBE * CUBE * if wide { 8 } else { 4 }) as u64, vk::BufferUsageFlags::TRANSFER_SRC)?;

            let pool = d
                .create_command_pool(&vk::CommandPoolCreateInfo::default().queue_family_index(gpu.queue_family), None)
                .map_err(err("command pool"))?;
            let cmd = d
                .allocate_command_buffers(&vk::CommandBufferAllocateInfo::default().command_pool(pool).command_buffer_count(1))
                .map_err(err("command buffer"))?[0];
            let fence = d.create_fence(&vk::FenceCreateInfo::default(), None).map_err(err("fence"))?;
            Ok(Pass {
                render_pass, set_layout, layout, pipelines, vert, frag, desc_pool, set, sampler,
                lut, lut_mem, lut_view, stage, stage_mem, pool, cmd, fence, texel: if wide { 8 } else { 4 },
            })
        }
    }

    /// Uploads the table; the pass is idle (renders are synchronous).
    pub(crate) fn set_lut(&self, gpu: &Gpu, table: &[u8]) -> Result<(), String> {
        if table.len() != CUBE * CUBE * CUBE * self.texel {
            return Err(format!("table of {} bytes for {} bytes per texel", table.len(), self.texel));
        }
        let d = &gpu.device;
        unsafe {
            let p = d.map_memory(self.stage_mem, 0, table.len() as u64, vk::MemoryMapFlags::empty()).map_err(|e| format!("map: {e}"))? as *mut u8;
            p.copy_from_nonoverlapping(table.as_ptr(), table.len());
            d.unmap_memory(self.stage_mem);
            d.reset_command_pool(self.pool, vk::CommandPoolResetFlags::empty()).map_err(|e| format!("reset: {e}"))?;
            d.begin_command_buffer(self.cmd, &vk::CommandBufferBeginInfo::default().flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT)).map_err(|e| format!("begin: {e}"))?;
            barrier(d, self.cmd, self.lut, vk::ImageLayout::UNDEFINED, vk::ImageLayout::TRANSFER_DST_OPTIMAL,
                vk::AccessFlags::empty(), vk::AccessFlags::TRANSFER_WRITE, vk::PipelineStageFlags::TOP_OF_PIPE, vk::PipelineStageFlags::TRANSFER);
            let region = vk::BufferImageCopy::default()
                .image_subresource(vk::ImageSubresourceLayers::default().aspect_mask(vk::ImageAspectFlags::COLOR).layer_count(1))
                .image_extent(CUBE_EXTENT);
            d.cmd_copy_buffer_to_image(self.cmd, self.stage, self.lut, vk::ImageLayout::TRANSFER_DST_OPTIMAL, &[region]);
            barrier(d, self.cmd, self.lut, vk::ImageLayout::TRANSFER_DST_OPTIMAL, vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL,
                vk::AccessFlags::TRANSFER_WRITE, vk::AccessFlags::SHADER_READ, vk::PipelineStageFlags::TRANSFER, vk::PipelineStageFlags::FRAGMENT_SHADER);
            d.end_command_buffer(self.cmd).map_err(|e| format!("end: {e}"))?;
            let cmds = [self.cmd];
            d.queue_submit(gpu.queue, &[vk::SubmitInfo::default().command_buffers(&cmds)], self.fence).map_err(|e| format!("submit: {e}"))?;
            d.wait_for_fences(&[self.fence], true, u64::MAX).map_err(|e| format!("wait: {e}"))?;
            d.reset_fences(&[self.fence]).map_err(|e| format!("reset fence: {e}"))?;
        }
        Ok(())
    }

    pub(crate) fn framebuffer(&self, gpu: &Gpu, target: &Target) -> Result<vk::Framebuffer, String> {
        let views = [target.view];
        unsafe {
            gpu.device.create_framebuffer(
                &vk::FramebufferCreateInfo::default().render_pass(self.render_pass).attachments(&views).width(target.width).height(target.height).layers(1),
                None,
            )
        }
        .map_err(|e| format!("video framebuffer: {e}"))
    }

    /// Renders one frame into `target` and waits for the GPU; returns the wait. `half`: the target has half
    /// the frame's size and `luma` is an R32G32_UINT view; else R32_UINT. `chroma`: R32G32_UINT.
    #[allow(clippy::too_many_arguments)]
    pub(crate) fn draw(
        &self, gpu: &Gpu, target: &Target, fb: vk::Framebuffer, luma: vk::BufferView, chroma: vk::BufferView, push: &Push, half: bool,
    ) -> Result<std::time::Duration, String> {
        let d = &gpu.device;
        let (cmd, extent) = (target.cmd, target.extent());
        let (luma, chroma) = ([luma], [chroma]);
        let image = [vk::DescriptorImageInfo::default().sampler(self.sampler).image_view(self.lut_view).image_layout(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL)];
        unsafe {
            d.update_descriptor_sets(
                &[
                    vk::WriteDescriptorSet::default().dst_set(self.set).dst_binding(0).descriptor_type(vk::DescriptorType::UNIFORM_TEXEL_BUFFER).texel_buffer_view(&luma),
                    vk::WriteDescriptorSet::default().dst_set(self.set).dst_binding(1).descriptor_type(vk::DescriptorType::UNIFORM_TEXEL_BUFFER).texel_buffer_view(&chroma),
                    vk::WriteDescriptorSet::default().dst_set(self.set).dst_binding(2).descriptor_type(vk::DescriptorType::COMBINED_IMAGE_SAMPLER).image_info(&image),
                ],
                &[],
            );
            d.reset_command_pool(target.pool, vk::CommandPoolResetFlags::empty()).map_err(|e| format!("reset pool: {e}"))?;
            d.begin_command_buffer(cmd, &vk::CommandBufferBeginInfo::default().flags(vk::CommandBufferUsageFlags::ONE_TIME_SUBMIT)).map_err(|e| format!("begin: {e}"))?;
            let area = vk::Rect2D { offset: vk::Offset2D { x: 0, y: 0 }, extent };
            d.cmd_begin_render_pass(
                cmd,
                &vk::RenderPassBeginInfo::default().render_pass(self.render_pass).framebuffer(fb).render_area(area),
                vk::SubpassContents::INLINE,
            );
            d.cmd_set_viewport(cmd, 0, &[vk::Viewport { x: 0.0, y: 0.0, width: extent.width as f32, height: extent.height as f32, min_depth: 0.0, max_depth: 1.0 }]);
            d.cmd_set_scissor(cmd, 0, &[area]);
            d.cmd_bind_pipeline(cmd, vk::PipelineBindPoint::GRAPHICS, self.pipelines[usize::from(half)]);
            d.cmd_bind_descriptor_sets(cmd, vk::PipelineBindPoint::GRAPHICS, self.layout, 0, &[self.set], &[]);
            d.cmd_push_constants(cmd, self.layout, vk::ShaderStageFlags::FRAGMENT, 0, push.bytes());
            d.cmd_draw(cmd, 3, 1, 0, 0);
            d.cmd_end_render_pass(cmd);
            d.end_command_buffer(cmd).map_err(|e| format!("end: {e}"))?;
            let cmds = [cmd];
            d.queue_submit(gpu.queue, &[vk::SubmitInfo::default().command_buffers(&cmds)], target.fence).map_err(|e| format!("submit: {e}"))?;
            let t0 = std::time::Instant::now();
            d.wait_for_fences(&[target.fence], true, u64::MAX).map_err(|e| format!("wait: {e}"))?;
            let wait = t0.elapsed();
            d.reset_fences(&[target.fence]).map_err(|e| format!("reset fence: {e}"))?;
            Ok(wait)
        }
    }

    pub(crate) fn destroy(self, gpu: &Gpu) {
        let d = &gpu.device;
        unsafe {
            d.destroy_fence(self.fence, None);
            d.destroy_command_pool(self.pool, None);
            d.destroy_buffer(self.stage, None);
            d.free_memory(self.stage_mem, None);
            d.destroy_image_view(self.lut_view, None);
            d.destroy_image(self.lut, None);
            d.free_memory(self.lut_mem, None);
            d.destroy_sampler(self.sampler, None);
            for pipeline in self.pipelines {
                d.destroy_pipeline(pipeline, None);
            }
            d.destroy_pipeline_layout(self.layout, None);
            d.destroy_shader_module(self.vert, None);
            d.destroy_shader_module(self.frag, None);
            d.destroy_descriptor_pool(self.desc_pool, None);
            d.destroy_descriptor_set_layout(self.set_layout, None);
            d.destroy_render_pass(self.render_pass, None);
        }
    }
}

fn host_buffer(gpu: &Gpu, size: u64, usage: vk::BufferUsageFlags) -> Result<(vk::Buffer, vk::DeviceMemory), String> {
    let d = &gpu.device;
    unsafe {
        let buf = d.create_buffer(&vk::BufferCreateInfo::default().size(size).usage(usage), None).map_err(|e| format!("buffer: {e}"))?;
        let req = d.get_buffer_memory_requirements(buf);
        let mt = gpu
            .find_memory_type_with(req.memory_type_bits, vk::MemoryPropertyFlags::HOST_VISIBLE | vk::MemoryPropertyFlags::HOST_COHERENT)
            .ok_or("no host memory")?;
        let mem = d.allocate_memory(&vk::MemoryAllocateInfo::default().allocation_size(req.size).memory_type_index(mt), None).map_err(|e| format!("mem: {e}"))?;
        d.bind_buffer_memory(buf, mem, 0).map_err(|e| format!("bind: {e}"))?;
        Ok((buf, mem))
    }
}

fn pipeline(d: &ash::Device, rp: vk::RenderPass, layout: vk::PipelineLayout, vert: vk::ShaderModule, frag: vk::ShaderModule, mode: u32, half: u32) -> Result<vk::Pipeline, String> {
    let entries = [
        vk::SpecializationMapEntry::default().constant_id(0).offset(0).size(4),
        vk::SpecializationMapEntry::default().constant_id(1).offset(4).size(4),
    ];
    let data: [u8; 8] = std::array::from_fn(|i| if i < 4 { mode.to_ne_bytes()[i] } else { half.to_ne_bytes()[i - 4] });
    let spec = vk::SpecializationInfo::default().map_entries(&entries).data(&data);
    let stages = [
        vk::PipelineShaderStageCreateInfo::default().stage(vk::ShaderStageFlags::VERTEX).module(vert).name(c"main"),
        vk::PipelineShaderStageCreateInfo::default().stage(vk::ShaderStageFlags::FRAGMENT).module(frag).name(c"main").specialization_info(&spec),
    ];
    let vertex_input = vk::PipelineVertexInputStateCreateInfo::default();
    let input_assembly = vk::PipelineInputAssemblyStateCreateInfo::default().topology(vk::PrimitiveTopology::TRIANGLE_LIST);
    let viewport = vk::PipelineViewportStateCreateInfo::default().viewport_count(1).scissor_count(1);
    let raster = vk::PipelineRasterizationStateCreateInfo::default().polygon_mode(vk::PolygonMode::FILL).cull_mode(vk::CullModeFlags::NONE).line_width(1.0);
    let multisample = vk::PipelineMultisampleStateCreateInfo::default().rasterization_samples(vk::SampleCountFlags::TYPE_1);
    let attach = [vk::PipelineColorBlendAttachmentState::default().color_write_mask(vk::ColorComponentFlags::RGBA)];
    let blend = vk::PipelineColorBlendStateCreateInfo::default().attachments(&attach);
    let states = [vk::DynamicState::VIEWPORT, vk::DynamicState::SCISSOR];
    let dynamic = vk::PipelineDynamicStateCreateInfo::default().dynamic_states(&states);
    let info = [vk::GraphicsPipelineCreateInfo::default()
        .stages(&stages).vertex_input_state(&vertex_input).input_assembly_state(&input_assembly)
        .viewport_state(&viewport).rasterization_state(&raster).multisample_state(&multisample)
        .color_blend_state(&blend).dynamic_state(&dynamic).layout(layout).render_pass(rp).subpass(0)];
    unsafe { d.create_graphics_pipelines(vk::PipelineCache::null(), &info, None) }.map_err(|(_, e)| format!("video pipeline: {e}")).map(|v| v[0])
}

/// A decoder dmabuf as an R32UI texel buffer; key: (st_dev, st_ino).
struct Import {
    key: (u64, u64),
    buffer: vk::Buffer,
    memory: vk::DeviceMemory,
    /// R32_UINT and R32G32_UINT views of the whole buffer.
    views: [vk::BufferView; 2],
    used: u64,
}

/// One plane of a decoder frame.
pub struct Plane {
    pub fd: OwnedFd,
    pub offset: u32,
    /// Column height in rows.
    pub column: u32,
}

struct Out {
    scanout: Scanout,
    target: Target,
    fb: vk::Framebuffer,
}

/// Tone-mapping renderer with its own device, decoder imports and scanout buffers.
pub struct VideoRenderer {
    gpu: Gpu,
    /// The pass and the fourcc of its targets; rebuilt when the output kind changes.
    pass: Option<(Pass, DrmFourcc)>,
    imports: Vec<Import>,
    serial: u64,
    outs: Vec<Out>,
    front: usize,
    /// Transfer and peak of the uploaded table.
    table: Option<TableKey>,
    /// Summed time of `Pass::draw` since the last `take_draw_time`.
    draw_time: std::time::Duration,
    /// Running totals of `Pass::draw`: command recording and submit, wait for the GPU.
    pub submit_total: std::time::Duration,
    pub wait_total: std::time::Duration,
}

impl VideoRenderer {
    pub fn new() -> Result<VideoRenderer, String> {
        let gpu = Gpu::new()?;
        Ok(VideoRenderer { gpu, pass: None, imports: Vec::new(), serial: 0, outs: Vec::new(), front: 0, table: None, draw_time: std::time::Duration::ZERO, submit_total: std::time::Duration::ZERO, wait_total: std::time::Duration::ZERO })
    }

    /// Renders the frame in `planes` (luma, chroma; `width`x`height`) and returns the framebuffer holding the result;
    /// `half`: the result has half the frame's size.
    #[allow(clippy::too_many_arguments)]
    pub fn render(
        &mut self, alloc: &Allocator, card: &Card, output: &Output,
        width: u32, height: u32, planes: &[Plane; 2], colour: Colour<'_>, half: bool,
    ) -> Result<framebuffer::Handle, String> {
        let (width, height) = if half { (width / 2, height / 2) } else { (width, height) };
        let hdr = matches!(colour, Colour::Cube { table, .. } if table.hdr);
        self.prepare(alloc, card, output, width, height, hdr)?;
        let pass = &self.pass.as_ref().ok_or("video pass missing")?.0;
        let key = match colour {
            Colour::Signal(s) => TableKey::Signal(s.transfer, s.peak.clamp(SDR_WHITE, 10000.0)),
            Colour::Cube { id, .. } => TableKey::Cube(id),
        };
        if self.table != Some(key) {
            match (colour, key) {
                (Colour::Cube { table, .. }, _) => pass.set_lut(&self.gpu, &table.bytes)?,
                (_, TableKey::Signal(transfer, peak)) => pass.set_lut(&self.gpu, tone_cube(transfer, peak, SDR_WHITE).as_flattened())?,
                _ => unreachable!(),
            }
            self.table = Some(key);
        }
        self.serial += 1;
        if planes[0].offset % if half { 8 } else { 4 } != 0 || planes[1].offset % 8 != 0 {
            return Err(format!("plane offsets {} and {} are not word and word-pair aligned", planes[0].offset, planes[1].offset));
        }
        let luma = self.import(&planes[0].fd)?[usize::from(half)];
        let chroma = self.import(&planes[1].fd)?[1];
        let lay = [planes[0].offset / 4, planes[1].offset / 4, planes[0].column, planes[1].column];
        let push = match colour {
            Colour::Signal(s) => Push::new(lay, s.full_range),
            Colour::Cube { .. } => Push::for_codes(lay),
        };
        let back = self.front ^ 1;
        let out = &self.outs[back];
        let t0 = std::time::Instant::now();
        let pass = &self.pass.as_ref().ok_or("video pass missing")?.0;
        let wait = pass.draw(&self.gpu, &out.target, out.fb, luma, chroma, &push, half)?;
        let spent = t0.elapsed();
        self.draw_time += spent;
        self.submit_total += spent.saturating_sub(wait);
        self.wait_total += wait;
        let fb = out.scanout.fb;
        self.front = back;
        self.trim();
        Ok(fb)
    }

    /// Creates the pass and two targets of `width`x`height`: 10-bit (XRGB2101010, else
    /// XBGR2101010) for `hdr`, else XRGB8888. No change when they exist already.
    pub fn prepare(&mut self, alloc: &Allocator, card: &Card, output: &Output, width: u32, height: u32, hdr: bool) -> Result<(), String> {
        let is_hdr = |fourcc: DrmFourcc| fourcc != DrmFourcc::Xrgb8888;
        if self.pass.as_ref().is_some_and(|(_, fourcc)| is_hdr(*fourcc) != hdr) {
            self.release_outputs(card);
            if let Some((pass, _)) = self.pass.take() {
                pass.destroy(&self.gpu);
            }
            self.table = None;
        }
        if self.outs.first().is_some_and(|o| (o.target.width, o.target.height) != (width, height)) {
            self.release_outputs(card);
        }
        let candidates: &[DrmFourcc] = match (&self.pass, hdr) {
            (Some((_, fourcc)), _) => &[*fourcc],
            (None, true) => &[DrmFourcc::Xrgb2101010, DrmFourcc::Xbgr2101010],
            (None, false) => &[DrmFourcc::Xrgb8888],
        };
        let candidates = candidates.to_vec();
        let mut last = String::new();
        for fourcc in candidates {
            match self.prepare_format(alloc, card, output, width, height, fourcc) {
                Ok(()) => return Ok(()),
                Err(error) => last = format!("{fourcc:?}: {error}"),
            }
        }
        Err(last)
    }

    fn prepare_format(&mut self, alloc: &Allocator, card: &Card, output: &Output, width: u32, height: u32, fourcc: DrmFourcc) -> Result<(), String> {
        while self.outs.len() < 2 {
            let (scanout, target) = self.gpu.create_scanout_target(alloc, card, output, width, height, fourcc)?;
            let format = super::target::vk_format(fourcc as u32).ok_or("no Vulkan format")?;
            let made = match &self.pass {
                Some(_) => Ok(()),
                None => Pass::new(&self.gpu, format, fourcc != DrmFourcc::Xrgb8888).map(|pass| self.pass = Some((pass, fourcc))),
            };
            let fb = made.and_then(|()| self.pass.as_ref().unwrap().0.framebuffer(&self.gpu, &target));
            match fb {
                Ok(fb) => self.outs.push(Out { scanout, target, fb }),
                Err(error) => {
                    self.gpu.destroy_target(target);
                    let _ = card.destroy_framebuffer(scanout.fb);
                    self.release_outputs(card);
                    if let Some((pass, _)) = self.pass.take() {
                        pass.destroy(&self.gpu);
                    }
                    return Err(error);
                }
            }
        }
        Ok(())
    }

    pub fn take_draw_time(&mut self) -> std::time::Duration {
        std::mem::take(&mut self.draw_time)
    }

    fn import(&mut self, fd: &OwnedFd) -> Result<[vk::BufferView; 2], String> {
        let mut st: libc::stat = unsafe { std::mem::zeroed() };
        if unsafe { libc::fstat(fd.as_raw_fd(), &mut st) } != 0 {
            return Err(format!("dmabuf stat: {}", std::io::Error::last_os_error()));
        }
        let key = (st.st_dev as u64, st.st_ino as u64);
        if let Some(i) = self.imports.iter_mut().find(|i| i.key == key) {
            i.used = self.serial;
            return Ok(i.views);
        }
        let size = unsafe { libc::lseek(fd.as_raw_fd(), 0, libc::SEEK_END) };
        if size <= 0 {
            return Err(format!("dmabuf size: {}", std::io::Error::last_os_error()));
        }
        let limit = unsafe { self.gpu.instance.get_physical_device_properties(self.gpu.physical) }.limits.max_texel_buffer_elements;
        if size as u64 / 4 > u64::from(limit) {
            return Err(format!("decoder buffer of {size} bytes exceeds maxTexelBufferElements {limit}"));
        }
        let (buffer, memory, views) = import_texel_buffer(&self.gpu, fd.as_fd(), size as u64)?;
        self.imports.push(Import { key, buffer, memory, views, used: self.serial });
        Ok(views)
    }

    fn trim(&mut self) {
        let serial = self.serial;
        let d = &self.gpu.device;
        self.imports.retain(|i| {
            let keep = serial - i.used < IMPORT_AGE;
            if !keep {
                destroy_import(d, i);
            }
            keep
        });
    }

    /// Releases the decoder buffers (end of a file).
    pub fn release_inputs(&mut self) {
        for i in self.imports.drain(..) {
            destroy_import(&self.gpu.device, &i);
        }
    }

    fn release_outputs(&mut self, card: &Card) {
        for o in self.outs.drain(..) {
            self.gpu.destroy_framebuffer(o.fb);
            self.gpu.destroy_target(o.target);
            let _ = card.destroy_framebuffer(o.scanout.fb);
        }
        self.front = 0;
    }

    pub fn destroy(mut self, card: &Card) {
        self.release_inputs();
        self.release_outputs(card);
        if let Some((pass, _)) = self.pass.take() {
            pass.destroy(&self.gpu);
        }
    }
}

fn destroy_import(d: &ash::Device, i: &Import) {
    unsafe {
        for view in i.views {
            d.destroy_buffer_view(view, None);
        }
        d.destroy_buffer(i.buffer, None);
        d.free_memory(i.memory, None);
    }
}

/// Imports `size` bytes of a dmabuf as a buffer with R32_UINT and R32G32_UINT views. Texel-buffer
/// usage only: v3dv then requires no more memory than the dmabuf holds.
fn import_texel_buffer(gpu: &Gpu, fd: BorrowedFd<'_>, size: u64) -> Result<(vk::Buffer, vk::DeviceMemory, [vk::BufferView; 2]), String> {
    let d = &gpu.device;
    let dmabuf = vk::ExternalMemoryHandleTypeFlags::DMA_BUF_EXT;
    unsafe {
        let mut external = vk::ExternalMemoryBufferCreateInfo::default().handle_types(dmabuf);
        let buffer = d
            .create_buffer(&vk::BufferCreateInfo::default().size(size).usage(vk::BufferUsageFlags::UNIFORM_TEXEL_BUFFER).push_next(&mut external), None)
            .map_err(|e| format!("decoder buffer: {e}"))?;
        let fail = |d: &ash::Device, e: String| {
            d.destroy_buffer(buffer, None);
            e
        };
        let req = d.get_buffer_memory_requirements(buffer);
        if req.size > size {
            return Err(fail(d, format!("decoder buffer needs {} bytes, the dmabuf has {size}", req.size)));
        }
        let ext = ash::khr::external_memory_fd::Device::new(&gpu.instance, d);
        let mut props = vk::MemoryFdPropertiesKHR::default();
        if let Err(e) = (ext.fp().get_memory_fd_properties_khr)(d.handle(), dmabuf, fd.as_raw_fd(), &mut props).result() {
            return Err(fail(d, format!("decoder dmabuf properties: {e}")));
        }
        let bits = req.memory_type_bits & props.memory_type_bits;
        let Some(memory_type) = (0..32).find(|i| bits & (1 << i) != 0) else {
            return Err(fail(d, "no memory type for the decoder dmabuf".into()));
        };
        let dup = fd.try_clone_to_owned().map_err(|e| fail(d, format!("dup: {e}")))?;
        let mut import = vk::ImportMemoryFdInfoKHR::default().handle_type(dmabuf).fd(dup.as_raw_fd());
        let memory = match d.allocate_memory(&vk::MemoryAllocateInfo::default().allocation_size(req.size).memory_type_index(memory_type).push_next(&mut import), None) {
            Ok(m) => m,
            Err(e) => return Err(fail(d, format!("decoder dmabuf import: {e}"))),
        };
        // Vulkan owns the fd after a successful import.
        std::mem::forget(dup);
        let view = |format| d.create_buffer_view(&vk::BufferViewCreateInfo::default().buffer(buffer).format(format).range(size & !7), None);
        let views = d.bind_buffer_memory(buffer, memory, 0).and_then(|()| {
            let words = view(vk::Format::R32_UINT)?;
            view(vk::Format::R32G32_UINT).map(|pairs| [words, pairs]).inspect_err(|_| d.destroy_buffer_view(words, None))
        });
        match views {
            Ok(views) => Ok((buffer, memory, views)),
            Err(e) => {
                d.destroy_buffer(buffer, None);
                d.free_memory(memory, None);
                Err(format!("decoder buffer view: {e}"))
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn f16_matches_reference_values() {
        for (v, bits) in [(0.0, 0x0000), (1.0, 0x3c00), (0.5, 0x3800), (-2.0, 0xc000), (65504.0, 0x7bff), (1.0e9, 0x7c00), (6.1035156e-5, 0x0400), (5.9604645e-8, 0x0001)] {
            assert_eq!(f16_bits(v), bits, "{v}");
        }
    }

    #[test]
    fn tables_have_the_texel_size_of_their_kind() {
        let n = CUBE * CUBE * CUBE;
        assert_eq!(Table::of(false, |e| e).bytes.len(), n * 4);
        let wide = Table::of(true, |e| e);
        assert_eq!(wide.bytes.len(), n * 8);
        // Last texel: (1, 1, 1, 1).
        assert_eq!(wide.bytes[(n - 1) * 8..], [0x3c00u16; 4].map(u16::to_ne_bytes).concat()[..]);
    }

    #[test]
    fn pq_round_trips_and_matches_known_points() {
        assert!((pq_eotf(pq_oetf(100.0)) - 100.0).abs() < 0.01);
        assert!((pq_oetf(100.0) - 0.5081).abs() < 1e-3);
        assert!((pq_oetf(1000.0) - 0.7518).abs() < 1e-3);
        assert!((pq_eotf(1.0) - 10000.0).abs() < 1.0);
    }

    #[test]
    fn eetf_is_identity_below_the_knee_and_ends_at_white() {
        let (peak, white) = (1000.0, SDR_WHITE);
        let low = pq_oetf(50.0);
        assert!((eetf(low, peak, white) - low).abs() < 1e-6);
        assert!((pq_eotf(eetf(pq_oetf(peak), peak, white)) - white).abs() < 0.5);
        assert!((pq_eotf(eetf(1.0, peak, white)) - white).abs() < 0.5);
        let mut last = 0.0;
        for i in 0..=1023 {
            let v = eetf(i as f32 / 1023.0, peak, white);
            assert!(v >= last);
            last = v;
        }
        // A source within the display range passes unchanged.
        assert!((eetf(low, 150.0, white) - low).abs() < 1e-6);
    }

    #[test]
    fn tone_map_reference_points() {
        let (peak, white) = (1000.0, SDR_WHITE);
        assert_eq!(tone_map([0.0; 3], Transfer::Pq, peak, white), [0.0; 3]);
        // HLG reference white (signal 0.75, 203 cd/m2) is below the knee: maps to 1.0.
        let display = hlg_display([0.75; 3]);
        assert!((display[0] - 203.0).abs() < 1.0, "{display:?}");
        for v in tone_map([0.75; 3], Transfer::Hlg, peak, white) {
            assert!((v - 1.0).abs() < 0.1, "{v}");
        }
        assert_eq!(tone_map([0.0; 3], Transfer::Hlg, peak, white), [0.0; 3]);
        assert!((hlg_display([1.0; 3])[0] - 1000.0).abs() < 1.0);
        // Below the knee: 50 cd/m2 grey maps to (50 / 203)^(1 / 2.4).
        let grey = tone_map([pq_oetf(50.0); 3], Transfer::Pq, peak, white);
        for v in grey {
            assert!((v - (50.0f32 / 203.0).powf(1.0 / 2.4)).abs() < 2e-3, "{grey:?}");
        }
        // Mastering peak maps to SDR white.
        for v in tone_map([pq_oetf(peak); 3], Transfer::Pq, peak, white) {
            assert!((v - 1.0).abs() < 2e-3);
        }
    }

    /// Trilinear interpolation of `cube` at R'G'B' `e`, 0..255.
    fn trilinear(cube: &[[u8; 4]], e: [f32; 3]) -> [f32; 3] {
        let pos = e.map(|v| v.clamp(0.0, 1.0) * (CUBE - 1) as f32);
        let lo = pos.map(|v| (v.floor() as usize).min(CUBE - 2));
        let f = [pos[0] - lo[0] as f32, pos[1] - lo[1] as f32, pos[2] - lo[2] as f32];
        let mut out = [0.0; 3];
        for corner in 0..8 {
            let d = [corner & 1, (corner >> 1) & 1, corner >> 2];
            let weight: f32 = (0..3).map(|k| if d[k] == 1 { f[k] } else { 1.0 - f[k] }).product();
            let texel = cube[lo[0] + d[0] + CUBE * (lo[1] + d[1] + CUBE * (lo[2] + d[2]))];
            for k in 0..3 {
                out[k] += weight * f32::from(texel[k]);
            }
        }
        out
    }

    /// Linear interpolation of the table against `tone_map`.
    #[test]
    fn cube_error_is_bounded() {
        let (peak, white) = (1000.0, SDR_WHITE);
        let cube = tone_cube(Transfer::Pq, peak, white);
        let (mut big, mut sum, mut count) = (0u32, 0.0f64, 0u32);
        let mut state = 1u32;
        let mut next = || {
            state = state.wrapping_mul(1664525).wrapping_add(1013904223);
            (state >> 8) as f32 / 16777216.0
        };
        for _ in 0..200_000 {
            let e = [next(), next(), next()];
            let (got, want) = (trilinear(&cube, e), tone_map(e, Transfer::Pq, peak, white));
            for k in 0..3 {
                let error = (got[k] - want[k] * 255.0).abs();
                big += u32::from(error > 4.0);
                sum += f64::from(error);
                count += 1;
            }
        }
        let mean = sum / f64::from(count);
        // Measured, CUBE 65, uniform R'G'B': mean 0.23 of 255, 1.2 % of samples above 4,
        // largest 39 where a BT.709 channel clips at 0 (out-of-gamut green and cyan).
        assert!(mean < 0.5 && big * 50 < count, "mean {mean}, {big} of {count} above 4");
    }

    #[test]
    fn ycbcr_rows_map_reference_codes() {
        let rgb = |m: [[f32; 4]; 3], c: [f32; 3]| m.map(|r| r[0] * c[0] + r[1] * c[1] + r[2] * c[2] + r[3]);
        let limited = ycbcr_rows(0.2627, 0.0593, false);
        for v in rgb(limited, [940.0, 512.0, 512.0]) {
            assert!((v - 1.0).abs() < 1e-4);
        }
        for v in rgb(limited, [64.0, 512.0, 512.0]) {
            assert!(v.abs() < 1e-4);
        }
        // BT.2020 red: Y' = Kr, Cr' = 0.5, Cb' = -Kr / (2 (1 - Kb)).
        let red = rgb(limited, [64.0 + 876.0 * 0.2627, 512.0 - 896.0 * 0.2627 / (2.0 * (1.0 - 0.0593)), 512.0 + 448.0]);
        assert!((red[0] - 1.0).abs() < 1e-3 && red[1].abs() < 1e-3 && red[2].abs() < 1e-3, "{red:?}");
        let full = ycbcr_rows(0.2627, 0.0593, true);
        for v in rgb(full, [1023.0, 512.0, 512.0]) {
            assert!((v - 1.0).abs() < 1e-4);
        }
    }

    /// CPU model of video.frag for one pixel, 8-bit BGRA.
    fn reference(ycc: [f32; 3], push: &Push, cube: &[[u8; 4]]) -> [u8; 3] {
        // Table coordinate to R'G'B': inverse of the texel-centre mapping in `Push::new`.
        let e = push.m.map(|r| ((r[0] * ycc[0] + r[1] * ycc[1] + r[2] * ycc[2] + r[3]) * CUBE as f32 - 0.5) / (CUBE - 1) as f32);
        let rgb = trilinear(cube, e).map(|v| v.round() as u8);
        [rgb[2], rgb[1], rgb[0]]
    }

    /// GPU: a synthetic SAND128 P030 frame through the pass, against the CPU model.
    #[test]
    #[ignore]
    fn pass_matches_cpu_model_on_synthetic_sand_frame() {
        let (w, h) = (200u32, 16u32); // three columns of 96 samples
        let (colh_y, colh_c) = (h + 3, h / 2 + 1);
        let columns = w.div_ceil(96);
        let luma_words = (columns * 32 * colh_y) as usize;
        let chroma_base = luma_words as u32 + 8; // chroma plane at a word offset in the same buffer
        let mut words = vec![0u32; chroma_base as usize + (columns * 32 * colh_c) as usize];
        let put = |words: &mut [u32], base: u32, colh: u32, s: u32, row: u32, v: u32| {
            let word = s / 3;
            let i = (base + word % 32 + row * 32 + (word / 32) * 32 * colh) as usize;
            words[i] |= v << (10 * (s % 3));
        };
        let y_code = |x: u32, y: u32| 64 + (x * 37 + y * 101) % 877;
        let c_code = |cx: u32, cy: u32, k: u32| 64 + (cx * 53 + cy * 29 + k * 311) % 897;
        for y in 0..h {
            for x in 0..w {
                put(&mut words, 0, colh_y, x, y, y_code(x, y));
            }
        }
        for cy in 0..h / 2 {
            for cx in 0..w / 2 {
                put(&mut words, chroma_base, colh_c, 2 * cx, cy, c_code(cx, cy, 0));
                put(&mut words, chroma_base, colh_c, 2 * cx + 1, cy, c_code(cx, cy, 1));
            }
        }

        let gpu = Gpu::new().unwrap();
        let pass = Pass::new(&gpu, vk::Format::B8G8R8A8_UNORM, false).unwrap();
        let lut = tone_cube(Transfer::Pq, 1000.0, SDR_WHITE);
        pass.set_lut(&gpu, lut.as_flattened()).unwrap();
        let size = (words.len() * 4) as u64;
        let (buffer, memory) = host_buffer(&gpu, size, vk::BufferUsageFlags::UNIFORM_TEXEL_BUFFER).unwrap();
        let d = &gpu.device;
        let (view, pairs) = unsafe {
            let p = d.map_memory(memory, 0, size, vk::MemoryMapFlags::empty()).unwrap() as *mut u32;
            p.copy_from_nonoverlapping(words.as_ptr(), words.len());
            d.unmap_memory(memory);
            let view = |format| d.create_buffer_view(&vk::BufferViewCreateInfo::default().buffer(buffer).format(format).range(vk::WHOLE_SIZE), None).unwrap();
            (view(vk::Format::R32_UINT), view(vk::Format::R32G32_UINT))
        };
        let target = gpu.offscreen_target(w, h).unwrap();
        let fb = pass.framebuffer(&gpu, &target).unwrap();
        let push = Push::new([0, chroma_base, colh_y, colh_c], false);
        pass.draw(&gpu, &target, fb, view, pairs, &push, false).unwrap();
        let pixels = gpu.read_target(&target).unwrap();
        let full_pixels = pixels.clone();

        let mut worst = 0;
        for y in 0..h {
            for x in 0..w {
                let want = reference([y_code(x, y) as f32, c_code(x / 2, y / 2, 0) as f32, c_code(x / 2, y / 2, 1) as f32], &push, &lut);
                let got = &pixels[((y * w + x) * 4) as usize..][..3];
                for k in 0..3 {
                    worst = worst.max((i32::from(got[k]) - i32::from(want[k])).abs());
                }
            }
        }
        // Sampler filter weights have 8 fractional bits; the model interpolates exactly.
        assert!(worst <= 4, "largest 8-bit difference {worst}");

        let small = gpu.offscreen_target(w / 2, h / 2).unwrap();
        let fb_small = pass.framebuffer(&gpu, &small).unwrap();
        pass.draw(&gpu, &small, fb_small, pairs, pairs, &push, true).unwrap();
        let pixels = gpu.read_target(&small).unwrap();
        let mut worst = 0;
        for y in 0..h / 2 {
            for x in 0..w / 2 {
                let luma = (y_code(2 * x, 2 * y) + y_code(2 * x + 1, 2 * y) + y_code(2 * x, 2 * y + 1) + y_code(2 * x + 1, 2 * y + 1)) as f32 * 0.25;
                let want = reference([luma, c_code(x, y, 0) as f32, c_code(x, y, 1) as f32], &push, &lut);
                let got = &pixels[((y * (w / 2) + x) * 4) as usize..][..3];
                for k in 0..3 {
                    worst = worst.max((i32::from(got[k]) - i32::from(want[k])).abs());
                }
            }
        }
        assert!(worst <= 4, "half size: largest 8-bit difference {worst}");

        // The RGBA16F table with the same map gives the same picture.
        let wide = Pass::new(&gpu, vk::Format::B8G8R8A8_UNORM, true).unwrap();
        wide.set_lut(&gpu, &Table::of(true, |e| tone_map(e, Transfer::Pq, 1000.0, SDR_WHITE)).bytes).unwrap();
        let fb_wide = wide.framebuffer(&gpu, &target).unwrap();
        wide.draw(&gpu, &target, fb_wide, view, pairs, &push, false).unwrap();
        let wide_pixels = gpu.read_target(&target).unwrap();
        let worst = wide_pixels.iter().zip(&full_pixels).map(|(a, b)| a.abs_diff(*b)).max().unwrap();
        assert!(worst <= 2, "RGBA16F table: largest 8-bit difference {worst}");
        gpu.destroy_framebuffer(fb_wide);
        wide.destroy(&gpu);
        gpu.destroy_framebuffer(fb_small);
        gpu.destroy_target(small);

        unsafe {
            d.destroy_buffer_view(view, None);
            d.destroy_buffer_view(pairs, None);
            d.destroy_buffer(buffer, None);
            d.free_memory(memory, None);
        }
        gpu.destroy_framebuffer(fb);
        gpu.destroy_target(target);
        pass.destroy(&gpu);
    }

    /// GPU timing of the MODE variants at 3840x2160 on an offscreen (tiled) target,
    /// planes in host memory: `cargo test --release -- --ignored --nocapture bench_pass_variants`.
    #[test]
    #[ignore]
    fn bench_pass_variants() {
        let (w, h) = (3840u32, 2160u32);
        let columns = w.div_ceil(96);
        let luma_words = columns * 32 * h;
        // Smooth gradients: neighbouring pixels read neighbouring table entries, as in video.
        let mut words = vec![0u32; (luma_words + columns * 32 * (h / 2)) as usize];
        let put = |words: &mut [u32], base: u32, colh: u32, s: u32, row: u32, v: u32| {
            let word = s / 3;
            words[(base + word % 32 + row * 32 + (word / 32) * 32 * colh) as usize] |= v << (10 * (s % 3));
        };
        for y in 0..h {
            for x in 0..w {
                put(&mut words, 0, h, x, y, 64 + (x + y) * 876 / (w + h));
            }
        }
        for cy in 0..h / 2 {
            for cx in 0..w / 2 {
                put(&mut words, luma_words, h / 2, 2 * cx, cy, 384 + cx * 256 / (w / 2));
                put(&mut words, luma_words, h / 2, 2 * cx + 1, cy, 384 + cy * 256 / (h / 2));
            }
        }
        let gpu = Gpu::new().unwrap();
        let d = &gpu.device;
        let size = (words.len() * 4) as u64;
        let (buffer, memory) = host_buffer(&gpu, size, vk::BufferUsageFlags::UNIFORM_TEXEL_BUFFER).unwrap();
        let (view, pairs) = unsafe {
            let p = d.map_memory(memory, 0, size, vk::MemoryMapFlags::empty()).unwrap() as *mut u32;
            p.copy_from_nonoverlapping(words.as_ptr(), words.len());
            d.unmap_memory(memory);
            let view = |format| d.create_buffer_view(&vk::BufferViewCreateInfo::default().buffer(buffer).format(format).range(vk::WHOLE_SIZE), None).unwrap();
            (view(vk::Format::R32_UINT), view(vk::Format::R32G32_UINT))
        };
        let target = gpu.offscreen_target(w, h).unwrap();
        let small = gpu.offscreen_target(w / 2, h / 2).unwrap();
        let push = Push::new([0, luma_words, h, h / 2], false);
        let cube = tone_cube(Transfer::Pq, 1000.0, SDR_WHITE);
        for (mode, name) in [(0, "full"), (1, "constant"), (2, "luma"), (3, "luma+chroma"), (4, "table")] {
            let pass = Pass::with_mode(&gpu, mode, vk::Format::B8G8R8A8_UNORM, false).unwrap();
            pass.set_lut(&gpu, cube.as_flattened()).unwrap();
            let fb = pass.framebuffer(&gpu, &target).unwrap();
            let time = |name: &str, target: &Target, fb, half: bool| {
                let luma = if half { pairs } else { view };
                pass.draw(&gpu, target, fb, luma, pairs, &push, half).unwrap();
                let t0 = std::time::Instant::now();
                for _ in 0..20 {
                    pass.draw(&gpu, target, fb, luma, pairs, &push, half).unwrap();
                }
                println!("bench {w}x{h} {name}: {:.1} ms", t0.elapsed().as_secs_f64() * 1e3 / 20.0);
            };
            time(name, &target, fb, false);
            if mode == 0 {
                let fb_half = pass.framebuffer(&gpu, &small).unwrap();
                time("full at half size", &small, fb_half, true);
                gpu.destroy_framebuffer(fb_half);
            }
            gpu.destroy_framebuffer(fb);
            pass.destroy(&gpu);
        }
        gpu.destroy_target(small);
        unsafe {
            d.destroy_buffer_view(view, None);
            d.destroy_buffer_view(pairs, None);
            d.destroy_buffer(buffer, None);
            d.free_memory(memory, None);
        }
        gpu.destroy_target(target);
    }
}