iridia_exec/

pixel.rs

//! A [`Program`] lifted from D3D bytecode behind [`PixelShader`]: inputs and outputs by signature register, constant
//! buffers and samplers by bind slot, as a caller that evaluates a shader over many pixels drives it. A vertex program has the same register view
//! (its inputs the vertex attributes by register, its outputs the varyings by register; the position and clip distances,
//! which have no output register, are not in it), so a caller can run one per vertex through [`PixelShader::execute`].

use std::collections::BTreeMap;

use super::{
    Backend, Element, Executable, ImageAtomic, ImageLoad, ImageQuery, ImageSample, ImageStore, Runner,
    Scratch, error,
};
use crate::{AtomicAdd, Error, Execution, PixelShader, Sampler, Store};
use iridia::{
    BufferKind, D3dClass, Interface, Program, ResourceId, ResourceKind, Stage, StorageTarget, SystemValue,
    TextureDim,
};
use std::sync::Mutex;

/// One entry argument and where its lanes come from in the register file.
#[derive(Debug)]
struct Argument {
    register: usize,
    /// The register lanes it takes, in order.
    lanes: Vec<u8>,
    /// A front-facing flag: the register word is non-zero for a front face.
    boolean: bool,
    used: bool,
}

/// One member of the result and the register lanes it is written to.
#[derive(Debug)]
struct Member {
    register: usize,
    lanes: Vec<u8>,
    /// The lanes the program writes, as its signature states.
    written: u8,
}

/// A pixel program with the register view the D3D readers' callers use.
#[derive(Debug)]
pub struct ProgramShader {
    program: Program,
    executable: Executable,
    arguments: Vec<Argument>,
    members: Vec<Member>,
    input_count: usize,
    output_count: usize,
    /// A vertex program's result member that is its clip-space position (`SV_Position`), which has no place in the
    /// register view: [`ProgramShader::execute_vertex`] returns it apart.
    position: Option<usize>,
    /// Each constant buffer: bind slot, resource, rows.
    constants: Vec<(u32, ResourceId, u32)>,
    constant_buffers: Vec<(u32, u32)>,
    textures: Vec<u32>,
    samplers: Vec<u32>,
    /// The bind slot of every resource that has one.
    slots: BTreeMap<ResourceId, u32>,
    buffers: BTreeMap<ResourceId, BufferKind>,
    /// Registers kept between runs, one set per concurrent run.
    scalar_scratch: Mutex<Vec<Scratch<1>>>,
    quad_scratch: Mutex<Vec<Scratch<4>>>,
    lane_scratch: Mutex<Vec<Scratch<LANES>>>,
}

/// The invocations [`PixelShader::execute_lanes`] runs in lockstep at a time.
const LANES: usize = 16;

/// The buffers lockstep runs reuse: every invocation's entry arguments, and the execution given to the caller.
#[derive(Default)]
struct Room {
    arguments: Vec<[u32; 4]>,
    execution: Execution,
}

fn lanes_of(mask: u8) -> Vec<u8> {
    (0..4).filter(|l| mask >> l & 1 != 0).collect()
}

/// The lanes the module carries for an input: the pipeline fixes some, the interface states the rest.
fn width_of(v: &iridia::Varying) -> usize {
    match v.system {
        Some(SystemValue::FragCoord) => 4,
        Some(
            SystemValue::FrontFacing
            | SystemValue::VertexIndex
            | SystemValue::InstanceIndex
            | SystemValue::PrimitiveIndex
            | SystemValue::SampleIndex
            | SystemValue::Coverage,
        ) => 1,
        _ => v.value.width as usize,
    }
}

impl ProgramShader {
    /// Compile `program`. Its inputs and outputs need their D3D registers (the lifter keeps them in provenance),
    /// and every resource a D3D bind slot.
    pub fn new(program: Program) -> Result<Self, Error> {
        if !matches!(program.stage, Stage::Pixel | Stage::Vertex) {
            return Err(error("not a pixel or vertex program"));
        }
        let executable = Executable::new(&program)?;
        let interface = &program.interface;
        let mut arguments = Vec::new();
        let mut input_count = 0;
        // the module's arguments: the interface's inputs the entry point carries, in order
        let io = iridia::entry_io(interface).map_err(error)?;
        for item in &io.arguments {
            let v = item
                .varyings
                .first()
                .and_then(|&i| interface.inputs.get(i))
                .ok_or_else(|| error("an entry argument names no input varying"))?;
            let name = Program::varying_name(&v.label);
            // the primitive and coverage inputs have no register in the signature: the register view cannot carry
            // them, they read as 0
            if v.provenance.d3d.is_none()
                && matches!(
                    v.system,
                    Some(SystemValue::PrimitiveIndex | SystemValue::Coverage | SystemValue::SampleIndex)
                )
            {
                arguments.push(Argument {
                    register: usize::MAX,
                    lanes: Vec::new(),
                    boolean: false,
                    used: false,
                });
                continue;
            }
            let sig = v
                .provenance
                .d3d
                .ok_or_else(|| error(format!("input {name} has no D3D register")))?;
            let lanes = lanes_of(sig.mask);
            let width = width_of(v);
            if lanes.len() < width {
                return Err(error(format!(
                    "input {name} has {width} lanes but its register mask {:#x} marks {}",
                    sig.mask,
                    lanes.len()
                )));
            }
            if v.used {
                input_count = input_count.max(sig.register as usize + 1);
            }
            arguments.push(Argument {
                register: sig.register as usize,
                lanes: lanes.iter().take(width).copied().collect(),
                boolean: v.system == Some(SystemValue::FrontFacing),
                used: v.used,
            });
        }
        let mut members = Vec::new();
        let mut output_count = 0;
        let position = interface
            .outputs
            .iter()
            .position(|v| program.stage == Stage::Vertex && v.system == Some(SystemValue::ClipPosition));
        for v in &interface.outputs {
            let name = Program::varying_name(&v.label);
            // depth, coverage and stencil have no output register
            if !matches!(v.system, None | Some(SystemValue::Target(_))) {
                members.push(Member {
                    register: usize::MAX,
                    lanes: Vec::new(),
                    written: 0,
                });
                continue;
            }
            let sig = v
                .provenance
                .d3d
                .ok_or_else(|| error(format!("output {name} has no D3D register")))?;
            output_count = output_count.max(sig.register as usize + 1);
            members.push(Member {
                register: sig.register as usize,
                lanes: lanes_of(sig.mask),
                // an element the program does not write (the signature lists it, `used` is false) has no lane written
                written: if v.used {
                    sig.mask & !sig.read_write_mask
                } else {
                    0
                },
            });
        }
        let mut constants = Vec::new();
        let (mut textures, mut samplers) = (Vec::new(), Vec::new());
        // the textures and samplers the compiled code reaches, not every one the program declares
        let (sampled_textures, sampled_samplers) = executable.sampled();
        let mut slots = BTreeMap::new();
        let mut buffers = BTreeMap::new();
        for r in &interface.resources {
            let binding = r
                .provenance
                .d3d
                .ok_or_else(|| error(format!("resource {} has no D3D bind slot", r.provenance.name)))?;
            slots.insert(r.id, binding.register);
            match &r.kind {
                ResourceKind::ConstantBuffer { .. } => {
                    let rows = rows_of(&program, r.id).ok_or_else(|| {
                        error(format!(
                            "constant buffer {} has no module array",
                            r.provenance.name
                        ))
                    })?;
                    constants.push((binding.register, r.id, rows));
                }
                // the 2D textures (and arrays of them) the code reaches: what `Sampler::sample` answers, and what the
                // evaluators' `textures()` always listed; a volume or a cube is reached through `sample_image`
                ResourceKind::Texture {
                    dim: TextureDim::D2, ..
                } if binding.class == D3dClass::ShaderResource && sampled_textures.contains(&r.id) => {
                    textures.push(binding.register);
                }
                ResourceKind::Sampler { .. } if sampled_samplers.contains(&r.id) => {
                    samplers.push(binding.register);
                }
                ResourceKind::Buffer(kind)
                | ResourceKind::Storage {
                    target: StorageTarget::Buffer(kind),
                    ..
                } => {
                    buffers.insert(r.id, *kind);
                }
                _ => {}
            }
        }
        textures.sort_unstable();
        textures.dedup();
        samplers.sort_unstable();
        samplers.dedup();
        let constant_buffers = constants.iter().map(|&(slot, _, rows)| (slot, rows)).collect();
        Ok(Self {
            program,
            executable,
            arguments,
            members,
            input_count,
            output_count,
            position,
            constants,
            constant_buffers,
            textures,
            samplers,
            slots,
            buffers,
            scalar_scratch: Mutex::new(Vec::new()),
            quad_scratch: Mutex::new(Vec::new()),
            lane_scratch: Mutex::new(Vec::new()),
        })
    }

    /// The vertex or pixel program the shader runs.
    pub fn program(&self) -> &Program {
        &self.program
    }

    /// Whether a caller that wants this program's exact results must run it as a quad ([`Executable::needs_quad`]).
    pub fn needs_quad(&self) -> bool {
        self.executable.needs_quad()
    }

    /// The entry arguments for one invocation's input registers.
    fn arguments_of(&self, inputs: &[[u32; 4]]) -> Result<Vec<[u32; 4]>, Error> {
        let mut out = Vec::with_capacity(self.arguments.len());
        self.arguments_into(inputs, &mut out)?;
        Ok(out)
    }

    /// [`ProgramShader::arguments_of`] appended to `out`.
    fn arguments_into(&self, inputs: &[[u32; 4]], out: &mut Vec<[u32; 4]>) -> Result<(), Error> {
        if inputs.len() < self.input_count {
            return Err(error(format!(
                "{} input registers given, {} needed",
                inputs.len(),
                self.input_count
            )));
        }
        for a in &self.arguments {
            let mut value = [0u32; 4];
            if let Some(register) = inputs.get(a.register) {
                for (slot, lane) in value.iter_mut().zip(&a.lanes) {
                    *slot = register.get(*lane as usize).copied().unwrap_or(0);
                }
            } else if a.used {
                return Err(error(format!("input register {} not given", a.register)));
            }
            if a.boolean {
                value[0] = u32::from(value[0] != 0);
            }
            out.push(value);
        }
        Ok(())
    }

    fn derivatives_of(&self, derivatives: &[[[u32; 4]; 2]]) -> Vec<[[u32; 4]; 2]> {
        self.arguments
            .iter()
            .map(|a| {
                let mut value = [[0u32; 4]; 2];
                if let Some(d) = derivatives.get(a.register) {
                    for (out, from) in value.iter_mut().zip(d) {
                        for (slot, lane) in out.iter_mut().zip(&a.lanes) {
                            *slot = from.get(*lane as usize).copied().unwrap_or(0);
                        }
                    }
                }
                value
            })
            .collect()
    }

    fn execution(&self, o: super::Output) -> Execution {
        let mut e = Execution::default();
        self.execution_into(&o.values, o.discarded, o.steps, o.samples, &mut e);
        e
    }

    /// What a run's result members `values` are in the register view, written over `e`.
    fn execution_into(
        &self,
        values: &[[u32; 4]],
        discarded: bool,
        steps: u64,
        samples: u64,
        e: &mut Execution,
    ) {
        e.outputs.clear();
        e.outputs.resize(self.output_count, [0; 4]);
        e.output_masks.clear();
        e.output_masks.resize(self.output_count, 0);
        for (m, value) in self.members.iter().zip(values) {
            if m.register == usize::MAX {
                continue;
            }
            if let Some(register) = e.outputs.get_mut(m.register) {
                for (lane, word) in m.lanes.iter().zip(value) {
                    if let Some(slot) = register.get_mut(*lane as usize) {
                        *slot = *word;
                    }
                }
            }
            if let Some(mask) = e.output_masks.get_mut(m.register) {
                *mask |= m.written;
            }
        }
        e.instructions = usize::try_from(steps).unwrap_or(usize::MAX);
        e.samples = usize::try_from(samples).unwrap_or(usize::MAX);
        e.discarded = discarded;
    }

    fn bound<'a>(&self, buffers: &[(u32, &'a [[u32; 4]])]) -> Result<super::Bound<'a>, Error> {
        for &(slot, _, rows) in &self.constants {
            let given = buffers
                .iter()
                .find(|(s, _)| *s == slot)
                .map(|(_, v)| *v)
                .ok_or_else(|| error(format!("no constant buffer cb{slot}")))?;
            if given.len() < rows as usize {
                return Err(error(format!(
                    "cb{slot}: {} rows given, {rows} needed",
                    given.len()
                )));
            }
        }
        // the buffers were all found above, by slot
        self.executable
            .bind(
                |id| {
                    let (slot, _, _) = self.constants.iter().find(|(_, c, _)| *c == id)?;
                    buffers.iter().find(|(s, _)| s == slot).map(|(_, v)| *v)
                },
                |_| None,
            )
            .map_err(error)
    }

    fn backend<'a>(&'a self, sampler: &'a mut dyn Sampler) -> SlotBackend<'a> {
        SlotBackend {
            sampler,
            slots: &self.slots,
            buffers: &self.buffers,
        }
    }
}

/// A set of registers from the pool, if one is free.
fn take<const N: usize>(pool: &Mutex<Vec<Scratch<N>>>) -> Option<Scratch<N>> {
    pool.lock().ok()?.pop()
}

fn give_back<const N: usize>(pool: &Mutex<Vec<Scratch<N>>>, scratch: Scratch<N>) {
    if let Ok(mut pool) = pool.lock() {
        pool.push(scratch);
    }
}

/// The rows of a constant buffer's array in the module.
fn rows_of(program: &Program, id: ResourceId) -> Option<u32> {
    let global = program
        .module
        .global_variables
        .try_get(program.global(id)?)
        .ok()?;
    match &program.module.types[global.ty].inner {
        naga::TypeInner::Array {
            size: naga::ArraySize::Constant(n),
            ..
        } => Some(n.get()),
        _ => None,
    }
}

/// The program's requests in the sampler's vocabulary: bind slots instead of resource ids, and a buffer's words as the
/// element and byte offset its view addresses.
pub(super) struct SlotBackend<'a> {
    pub(super) sampler: &'a mut dyn Sampler,
    pub(super) slots: &'a BTreeMap<ResourceId, u32>,
    pub(super) buffers: &'a BTreeMap<ResourceId, BufferKind>,
}
impl SlotBackend<'_> {
    fn slot(&self, id: ResourceId) -> Result<u32, String> {
        self.slots
            .get(&id)
            .copied()
            .ok_or_else(|| format!("resource {} has no bind slot", id.0))
    }
    /// The bind slot of an element: a slot answers one binding, not an array's.
    fn slot_of(&self, e: Element) -> Result<u32, String> {
        if e.index != 0 {
            return Err(format!(
                "resource {}: element {} of an array, which a bind-slot sampler does not answer",
                e.id.0, e.index
            ));
        }
        self.slot(e.id)
    }
    /// A buffer view's address of word `word`: (element, byte offset in the element).
    fn address(&self, id: ResourceId, word: u32) -> [u32; 3] {
        match self.buffers.get(&id) {
            Some(BufferKind::Structured { stride }) if *stride >= 4 => {
                let per = stride / 4;
                [
                    word.checked_div(per).unwrap_or(0),
                    word.checked_rem(per).unwrap_or(0).saturating_mul(4),
                    0,
                ]
            }
            // raw and typed views are addressed by the word
            _ => [word, 0, 0],
        }
    }
}
impl Backend for SlotBackend<'_> {
    fn sample(&mut self, r: ImageSample<Element>) -> Result<[f32; 4], String> {
        let (texture, sampler) = (self.slot_of(r.texture)?, self.slot_of(r.sampler)?);
        self.sampler.sample_image(ImageSample {
            texture,
            sampler,
            dim: r.dim,
            coord: r.coord,
            layer: r.layer,
            lod: r.lod,
            offset: r.offset,
            gather: r.gather,
            depth_ref: r.depth_ref,
            depth: r.depth,
        })
    }
    fn load(&mut self, r: ImageLoad<Element>) -> Result<[u32; 4], String> {
        let texture = self.slot_of(r.texture)?;
        self.sampler.load_image(ImageLoad {
            texture,
            dim: r.dim,
            storage: r.storage,
            texel: r.texel,
            layer: r.layer,
            sample: r.sample,
            mip: r.mip,
        })
    }
    fn query(&mut self, r: ImageQuery<Element>) -> Result<[u32; 4], String> {
        let texture = self.slot_of(r.texture)?;
        self.sampler.query_image(ImageQuery {
            texture,
            dim: r.dim,
            kind: r.kind,
            level: r.level,
        })
    }
    fn store_image(&mut self, r: ImageStore<ResourceId>) -> Result<(), String> {
        let view = self.slot_of(Element {
            id: r.view,
            index: r.index,
        })?;
        self.sampler.store_image(ImageStore {
            view,
            index: 0,
            dim: r.dim,
            texel: r.texel,
            layer: r.layer,
            value: r.value,
            lanes: r.lanes,
        })
    }
    fn atomic_image(&mut self, r: ImageAtomic<ResourceId>) -> Result<u32, String> {
        let view = self.slot_of(Element {
            id: r.view,
            index: r.index,
        })?;
        self.sampler.atomic_image(ImageAtomic {
            view,
            index: 0,
            dim: r.dim,
            texel: r.texel,
            layer: r.layer,
            op: r.op,
            signed: r.signed,
            value: r.value,
        })
    }
    fn store_buffer(&mut self, view: ResourceId, word: u32, value: [u32; 4], mask: u8) -> Result<(), String> {
        let address = self.address(view, word);
        let view = self.slot(view)?;
        self.sampler.store(Store {
            view,
            address,
            value,
            mask,
        })
    }
    fn atomic_add(&mut self, view: ResourceId, word: u32, value: u32) -> Result<u32, String> {
        let address = self.address(view, word);
        let view = self.slot(view)?;
        self.sampler.atomic_add(AtomicAdd { view, address, value })
    }
    fn read_buffer(&mut self, view: ResourceId, word: u32, count: u8) -> Result<[u32; 4], String> {
        let slot = self.slot(view)?;
        let mut out = [0u32; 4];
        match self.buffers.get(&view) {
            // a typed view's element is four lanes, one load answers all of them
            Some(BufferKind::Typed(_)) => {
                let (mut at, end) = (word, word.saturating_add(u32::from(count)));
                while at < end {
                    let element = at / 4;
                    let lanes = self.sampler.load(crate::Load {
                        texture: slot,
                        texel: [i32::try_from(element).unwrap_or(i32::MAX), 0, 0],
                        mip: 0,
                    })?;
                    while at < end && at / 4 == element {
                        let from = lanes.get((at % 4) as usize).copied().unwrap_or(0);
                        if let Some(slot) = out.get_mut(at.saturating_sub(word) as usize) {
                            *slot = from;
                        }
                        at = at.saturating_add(1);
                    }
                }
            }
            _ => {
                for (i, w) in out.iter_mut().take(usize::from(count)).enumerate() {
                    let [element, offset, _] =
                        self.address(view, word.saturating_add(u32::try_from(i).unwrap_or(0)));
                    *w = self.sampler.structured(slot, element, offset)?;
                }
            }
        }
        Ok(out)
    }
    fn buffer_words(&mut self, view: ResourceId) -> Result<u32, String> {
        let slot = self.slot(view)?;
        // without the sampler saying how long the buffer is, no read is out of range: its own error stops it
        let Some(elements) = self.sampler.buffer_elements(slot) else {
            return Ok(u32::MAX);
        };
        let per = match self.buffers.get(&view) {
            Some(BufferKind::Typed(_)) => 4,
            Some(BufferKind::Structured { stride }) => (stride / 4).max(1),
            _ => 1,
        };
        Ok(elements.saturating_mul(per))
    }
}

impl PixelShader for ProgramShader {
    fn interface(&self) -> &Interface {
        &self.program.interface
    }
    fn input_count(&self) -> usize {
        self.input_count
    }
    fn constant_buffers(&self) -> &[(u32, u32)] {
        &self.constant_buffers
    }
    fn textures(&self) -> &[u32] {
        &self.textures
    }
    fn samplers(&self) -> &[u32] {
        &self.samplers
    }

    fn execute(
        &self,
        inputs: &[[u32; 4]],
        buffers: &[(u32, &[[u32; 4]])],
        sampler: &mut dyn Sampler,
    ) -> Result<Execution, Error> {
        self.run_scalar(inputs, None, buffers, sampler)
    }

    fn execute_with_derivatives(
        &self,
        inputs: &[[u32; 4]],
        derivatives: &[[[u32; 4]; 2]],
        buffers: &[(u32, &[[u32; 4]])],
        sampler: &mut dyn Sampler,
    ) -> Result<Execution, Error> {
        self.run_scalar(inputs, Some(derivatives), buffers, sampler)
    }

    fn execute_lanes(
        &self,
        inputs: &[&[[u32; 4]]],
        derivatives: &[[[u32; 4]; 2]],
        buffers: &[(u32, &[[u32; 4]])],
        sampler: &mut dyn Sampler,
        each: &mut dyn FnMut(usize, &Execution),
    ) -> Result<(), (usize, Error)> {
        let lane_derivatives = self.derivatives_of(derivatives);
        let mut room = Room::default();
        for (k, chunk) in inputs.chunks(LANES).enumerate() {
            let together = self.run_lanes(
                chunk,
                &lane_derivatives,
                buffers,
                sampler,
                &mut room,
                &mut |i, e| each(k.saturating_mul(LANES).saturating_add(i), e),
            );
            // a run that stops is run again one invocation after another: the error is then the first failing
            // one's, as alone, and the invocations before it are given
            if together.is_err() {
                for (i, one) in chunk.iter().enumerate() {
                    let done = self
                        .run_scalar(one, Some(derivatives), buffers, sampler)
                        .map_err(|e| (k.saturating_mul(LANES).saturating_add(i), e))?;
                    each(k.saturating_mul(LANES).saturating_add(i), &done);
                }
            }
        }
        Ok(())
    }

    fn execute_quad(
        &self,
        inputs: [&[[u32; 4]]; 4],
        buffers: &[(u32, &[[u32; 4]])],
        sampler: &mut dyn Sampler,
    ) -> Result<[Execution; 4], Error> {
        let bound = self.bound(buffers)?;
        let arguments = [
            self.arguments_of(inputs[0])?,
            self.arguments_of(inputs[1])?,
            self.arguments_of(inputs[2])?,
            self.arguments_of(inputs[3])?,
        ];
        let mut backend = self.backend(sampler);
        let scratch = take(&self.quad_scratch).unwrap_or_else(|| Scratch::new(&self.executable));
        let mut runner = Runner::<4>::with_scratch(&self.executable, &bound, &mut backend, None, scratch);
        let outputs = runner.run([&arguments[0], &arguments[1], &arguments[2], &arguments[3]]);
        give_back(&self.quad_scratch, runner.into_scratch());
        Ok(outputs.map_err(error)?.map(|o| self.execution(o)))
    }
}

impl ProgramShader {
    /// Run a vertex program once on its input registers (the vertex's attributes): its varyings in the register view,
    /// as [`PixelShader::execute`] gives them, and its clip-space position (`SV_Position`) as f32 bits. A program that
    /// is not a vertex program, or writes no position, refuses.
    pub fn execute_vertex(
        &self,
        inputs: &[[u32; 4]],
        buffers: &[(u32, &[[u32; 4]])],
        sampler: &mut dyn Sampler,
    ) -> Result<(Execution, [u32; 4]), Error> {
        let at = self
            .position
            .ok_or_else(|| error("not a vertex program that writes a position"))?;
        let bound = self.bound(buffers)?;
        let arguments = self.arguments_of(inputs)?;
        let mut backend = self.backend(sampler);
        let scratch = take(&self.scalar_scratch).unwrap_or_else(|| Scratch::new(&self.executable));
        let mut runner = Runner::<1>::with_scratch(&self.executable, &bound, &mut backend, None, scratch);
        let output = runner.run([&arguments]);
        give_back(&self.scalar_scratch, runner.into_scratch());
        let [output] = output.map_err(error)?;
        let position = output.values.get(at).copied().unwrap_or([0; 4]);
        Ok((self.execution(output), position))
    }

    /// [`PixelShader::execute_lanes`] without its second run: the invocations in lockstep, 16 at a time, and on an
    /// error the one the lockstep run met (of whichever invocation met one first in lockstep, which need not be the
    /// first to fail alone).
    pub fn execute_lockstep(
        &self,
        inputs: &[&[[u32; 4]]],
        derivatives: &[[[u32; 4]; 2]],
        buffers: &[(u32, &[[u32; 4]])],
        sampler: &mut dyn Sampler,
    ) -> Result<Vec<Execution>, Error> {
        let derivatives = self.derivatives_of(derivatives);
        let mut room = Room::default();
        let mut out = Vec::with_capacity(inputs.len());
        for chunk in inputs.chunks(LANES) {
            self.run_lanes(chunk, &derivatives, buffers, sampler, &mut room, &mut |_, e| {
                out.push(e.clone())
            })?;
        }
        Ok(out)
    }

    /// Up to [`LANES`] invocations in lockstep, sharing `derivatives` (the entry arguments'), each one's execution
    /// given to `each` (with its index in `inputs`) once all have run, in `room`'s buffers.
    fn run_lanes(
        &self,
        inputs: &[&[[u32; 4]]],
        derivatives: &[[[u32; 4]; 2]],
        buffers: &[(u32, &[[u32; 4]])],
        sampler: &mut dyn Sampler,
        room: &mut Room,
        each: &mut dyn FnMut(usize, &Execution),
    ) -> Result<(), Error> {
        let bound = self.bound(buffers)?;
        let width = self.arguments.len();
        room.arguments.clear();
        for one in inputs {
            self.arguments_into(one, &mut room.arguments)?;
        }
        let count = inputs.len();
        if count == 0 {
            return Ok(());
        }
        // the lanes past the invocations are given the last one's arguments and not run
        let arguments = &room.arguments;
        let args: [&[[u32; 4]]; LANES] = std::array::from_fn(|l| {
            let i = l.min(count.saturating_sub(1));
            let from = i.saturating_mul(width);
            arguments
                .get(from..from.saturating_add(width))
                .unwrap_or_default()
        });
        let mut backend = self.backend(sampler);
        let scratch = take(&self.lane_scratch).unwrap_or_else(|| Scratch::new(&self.executable));
        let mut runner = Runner::<LANES, u16>::with_scratch(
            &self.executable,
            &bound,
            &mut backend,
            Some(derivatives),
            scratch,
        );
        let run = runner.run_in_place(args, count);
        if run.is_ok() {
            for l in 0..count {
                let (values, discarded, steps, samples) = runner.lane(l);
                self.execution_into(values, discarded, steps, samples, &mut room.execution);
                each(l, &room.execution);
            }
        }
        give_back(&self.lane_scratch, runner.into_scratch());
        run.map_err(error)
    }

    fn run_scalar(
        &self,
        inputs: &[[u32; 4]],
        derivatives: Option<&[[[u32; 4]; 2]]>,
        buffers: &[(u32, &[[u32; 4]])],
        sampler: &mut dyn Sampler,
    ) -> Result<Execution, Error> {
        let bound = self.bound(buffers)?;
        let arguments = self.arguments_of(inputs)?;
        let derivatives = derivatives.map(|d| self.derivatives_of(d));
        let mut backend = self.backend(sampler);
        let scratch = take(&self.scalar_scratch).unwrap_or_else(|| Scratch::new(&self.executable));
        let mut runner = Runner::<1>::with_scratch(
            &self.executable,
            &bound,
            &mut backend,
            derivatives.as_deref(),
            scratch,
        );
        let output = runner.run([&arguments]);
        give_back(&self.scalar_scratch, runner.into_scratch());
        let [output] = output.map_err(error)?;
        Ok(self.execution(output))
    }
}