wai-quantum 0.5.0

//! The fabric's OpenCL member: the gate kernel of
//! [`quantum_fabric::opencl`](crate::quantum_fabric::opencl) on any OpenCL
//! device, GPU or CPU, built at run time by the device's own compiler. See that
//! module for the kernel and the argument that it is bit-exact.
//!
//! - **Discovery** ([`discover`]): the ICD loader is opened when the process
//!   first asks for it, so a build with this feature runs on a host with no
//!   OpenCL at all and finds no member there ([`Discovery::unavailable`] says
//!   why). Every device of every platform is opened, the kernel is built for
//!   it, and its arithmetic is probed against [`lanes`] in this process. A
//!   device that cannot host the kernel, whose arithmetic disagrees, or which
//!   no fabric class names (an OpenCL accelerator), is listed with the reason
//!   ([`Refusal`]); one that passes is an [`OpenClMember`], present as
//!   [`Presence::Detected`].
//! - **Running** ([`OpenClMember::run_program`]): the state stays on the
//!   device; one launch per gate, in op-log order, on one in-order queue; then
//!   the report word and the state come back. The kernel reports exactly when
//!   the reference's result leaves 32 bits; the member then returns
//!   [`OpenClError::OutOfRange`], never a state.
//! - **Certification, placement, receipts and metering** are as for the CUDA
//!   member ([`crate::quantum_fabric_cuda`]).
//!
//! Every call is synchronous; the device is waited on before a call returns.
//! No type of the OpenCL bindings appears in this module's public API.
//!
//! # Honest boundaries
//!
//! - **A member is what this process measured.** The probe and the
//!   certificates are earned per process; the member id binds the platform,
//!   the device, its OpenCL and driver versions, its UUID where the platform
//!   reports one, its compute units and its memory.
//! - **A CPU device is a CPU member.** Its class is [`FabricClass::Cpu`], and
//!   its id sorts after the CPU reference's, so with no energy figures
//!   placement keeps the reference.
//! - **Plans are per gate** ([`Config::PER_GATE`]), as for the CUDA member.
//! - **Unmetered by default**, as for every member.

use crate::quantum::{Amp, Circuit, Gateset, MAX_QUBITS, ONE, QuantumError, StateVector};
use crate::quantum_energy::EnergyClass;
use crate::quantum_fabric::lanes::{self, GateWords, Lane};
use crate::quantum_fabric::opencl::{self, DeviceIdentity, DeviceKind};
use crate::quantum_fabric::plan::{self, Config, LaneRun, Program, Step};
use crate::quantum_fabric::{
    Certificate, Certification, Corpus, EnergyCalibration, FabricClass, KernelEntry, Member, MemberError, Presence, RunFuture, RunMeter, Runner, Unmetered, Window,
    apply_kernel, pair_updates,
};
use opencl3::command_queue::CommandQueue;
use opencl3::context::Context;
use opencl3::device::{CL_DEVICE_TYPE_ALL, CL_DEVICE_TYPE_CPU, CL_DEVICE_TYPE_GPU, Device};
use opencl3::error_codes::ClError;
use opencl3::kernel::{ExecuteKernel, Kernel as ClKernel};
use opencl3::memory::{Buffer, CL_MEM_READ_WRITE};
use opencl3::platform::{Platform, get_platforms};
use opencl3::program::Program as ClProgram;
use opencl3::types::{CL_BLOCKING, cl_device_id, cl_int, cl_uint, cl_ulong};
use std::sync::{Arc, Mutex, MutexGuard};

/// One probe case's device results: [`lanes::mul_wide`] and
/// [`lanes::fxmul_wide`], each as `(high, low)` words.
pub type ProbeOutput = ((u32, u32), (u32, u32));

/// A run's lanes and report, with the meter's accepted reading (zero and
/// `None` when unmetered).
type MeteredRun = (LaneRun, (u64, Option<EnergyClass>));

/// The OpenCL status codes that mean memory or resources ran out:
/// `CL_MEM_OBJECT_ALLOCATION_FAILURE`, `CL_OUT_OF_RESOURCES` and
/// `CL_OUT_OF_HOST_MEMORY`.
const OUT_OF_MEMORY: [cl_int; 3] = [-4, -5, -6];

/// The status the bindings return when the ICD loader cannot be opened.
const LOADER_MISSING: cl_int = -2000;

/// Why a device is not a member, or a run has no state.
#[derive(Debug, PartialEq, Eq)]
#[non_exhaustive]
pub enum OpenClError {
    /// The ICD loader could not be used, or it lists no platform, and why.
    Unavailable(String),
    /// The device's compiler refused the kernel; its build log.
    Build(String),
    /// The runtime reported an error, or the device was lost.
    Device(String),
    /// Out of memory or resources on the device or the host.
    Memory(String),
    /// Exactly what the CPU reference refuses.
    Circuit(QuantumError),
    /// More qubits than this member's memory holds.
    TooWide { n_qubits: u8, max: u8 },
    /// The reference's result leaves 32 bits at this op; the member returns
    /// no state.
    OutOfRange { gate: u32 },
    /// A program the host cannot lay out (malformed words, a start outside the
    /// state, a step this member does not run): a defect of the host, never a
    /// result.
    Internal(&'static str),
    /// The device's arithmetic disagreed with the CPU's on this pair.
    Probe { a: i32, b: i32 },
    /// A device the fabric cannot use as a member, and why.
    Unsupported(String),
}

impl std::fmt::Display for OpenClError {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            OpenClError::Unavailable(why) => write!(f, "OpenCL is unavailable: {why}"),
            OpenClError::Build(log) => write!(f, "the kernel did not build: {log}"),
            OpenClError::Device(e) => write!(f, "device: {e}"),
            OpenClError::Memory(e) => write!(f, "out of memory: {e}"),
            OpenClError::Circuit(e) => write!(f, "{e}"),
            OpenClError::TooWide { n_qubits, max } => write!(f, "{n_qubits} qubits; this member holds at most {max}"),
            OpenClError::OutOfRange { gate } => write!(f, "the result of op {gate} leaves 32 bits"),
            OpenClError::Internal(why) => write!(f, "host defect: {why}"),
            OpenClError::Probe { a, b } => write!(f, "the device's arithmetic differs from the CPU's at ({a}, {b})"),
            OpenClError::Unsupported(why) => write!(f, "not a member: {why}"),
        }
    }
}

impl std::error::Error for OpenClError {}

impl From<OpenClError> for MemberError {
    fn from(e: OpenClError) -> MemberError {
        match e {
            OpenClError::TooWide { n_qubits, max } => MemberError::TooWide { n_qubits, max },
            OpenClError::OutOfRange { gate } => MemberError::OutOfRange { gate },
            OpenClError::Memory(why) => MemberError::Memory(why),
            OpenClError::Device(why) => MemberError::Device(why),
            OpenClError::Circuit(e) => MemberError::Circuit(e),
            other => MemberError::Other(other.to_string()),
        }
    }
}

/// A runtime error as this module reports it: out of memory or resources as
/// [`OpenClError::Memory`], anything else as [`OpenClError::Device`].
fn cl(e: ClError) -> OpenClError {
    if OUT_OF_MEMORY.contains(&e.0) { OpenClError::Memory(e.to_string()) } else { OpenClError::Device(e.to_string()) }
}

/// A device that is not a member, and why.
#[derive(Debug)]
pub struct Refusal {
    /// The device's name, or its platform and position when it gave none.
    pub device: String,
    /// Its CI key ([`DeviceIdentity::key`]), when the device could be read.
    pub key: String,
    pub error: OpenClError,
}

/// What [`discover`] found.
#[derive(Debug)]
pub struct Discovery {
    /// Devices that built the kernel and passed the probe, in platform order.
    pub found: Vec<OpenClMember>,
    /// Devices that did not, with why.
    pub refused: Vec<Refusal>,
    /// Why no device was even listed: no ICD loader, or no platform. `None`
    /// when a platform was listed.
    pub unavailable: Option<String>,
}

/// The platforms the ICD loader lists, or why there are none.
fn platforms() -> Result<Vec<Platform>, String> {
    match get_platforms() {
        Err(e) if e.0 == LOADER_MISSING => Err("no OpenCL ICD loader on this host".into()),
        Err(e) => Err(format!("the ICD loader lists no platform: {e}")),
        Ok(p) if p.is_empty() => Err("the ICD loader lists no platform".into()),
        Ok(p) => Ok(p),
    }
}

/// Whether the ICD loader opens and lists a platform, or why not.
pub fn available() -> Result<(), String> {
    platforms().map(|_| ())
}

/// Every device of every platform the ICD loader lists, each opened, built for
/// and probed in this process before it is listed. On a host without OpenCL,
/// nothing is found and [`Discovery::unavailable`] says why.
pub fn discover() -> Discovery {
    let mut d = Discovery { found: Vec::new(), refused: Vec::new(), unavailable: None };
    let platforms = match platforms() {
        Ok(p) => p,
        Err(why) => {
            d.unavailable = Some(why);
            return d;
        }
    };
    for platform in platforms {
        let info = PlatformInfo {
            name: platform.name().unwrap_or_default(),
            version: platform.version().unwrap_or_default(),
            vendor: platform.vendor().unwrap_or_default(),
        };
        let ids = match platform.get_devices(CL_DEVICE_TYPE_ALL) {
            Ok(ids) => ids,
            Err(e) => {
                d.refused.push(Refusal { device: format!("platform {:?}", info.name), key: String::new(), error: cl(e) });
                continue;
            }
        };
        for (k, id) in ids.into_iter().enumerate() {
            match open(&info, k, id) {
                Ok(m) => d.found.push(m),
                Err(r) => d.refused.push(r),
            }
        }
    }
    d
}

struct PlatformInfo {
    name: String,
    version: String,
    vendor: String,
}

/// The widest state of `2^n` interleaved 8-byte lanes that `bytes` holds, at
/// most [`MAX_QUBITS`].
fn qubits_for(bytes: u64) -> u8 {
    let lanes = bytes / 8;
    if lanes < 2 { 0 } else { (lanes.ilog2() as u8).min(MAX_QUBITS) }
}

fn hex(bytes: &[u8]) -> String {
    bytes.iter().map(|b| format!("{b:02x}")).collect()
}

/// Reads the device, builds the kernel for it and probes it.
fn open(platform: &PlatformInfo, position: usize, id: cl_device_id) -> Result<OpenClMember, Refusal> {
    let unread = |error: OpenClError| Refusal { device: format!("{:?} device {position}", platform.name), key: String::new(), error };
    let device = Device::new(id);
    let name = device.name().map_err(|e| unread(cl(e)))?;
    let kind = match device.dev_type().map_err(|e| unread(cl(e)))? {
        t if t & CL_DEVICE_TYPE_GPU != 0 => DeviceKind::Gpu,
        t if t & CL_DEVICE_TYPE_CPU != 0 => DeviceKind::Cpu,
        _ => DeviceKind::Accelerator,
    };
    let extensions = device.extensions().unwrap_or_default();
    let uuid = if extensions.split_whitespace().any(|x| x == "cl_khr_device_uuid") { device.uuid_khr().map(|u| hex(&u)).unwrap_or_default() } else { String::new() };
    let memory = device.global_mem_size().map_err(|e| unread(cl(e)))?;
    let identity = DeviceIdentity {
        platform: platform.name.clone(),
        platform_version: platform.version.clone(),
        vendor: platform.vendor.clone(),
        name: name.clone(),
        version: device.version().unwrap_or_default(),
        driver: device.driver_version().unwrap_or_default(),
        kind,
        uuid,
        compute_units: device.max_compute_units().unwrap_or(0),
        memory,
    };
    let key = identity.key();
    let refuse = |error: OpenClError| Refusal { device: name.clone(), key: key.clone(), error };
    let Some(class) = identity.class() else {
        return Err(refuse(OpenClError::Unsupported("an OpenCL accelerator, which no fabric class names".into())));
    };
    if !device.available().unwrap_or(false) {
        return Err(refuse(OpenClError::Unsupported("the device reports itself unavailable".into())));
    }
    let context = Context::from_device(&device).map_err(|e| refuse(cl(e)))?;
    #[allow(deprecated)] // clCreateCommandQueue: OpenCL 1.2 platforms have no other.
    let queue = CommandQueue::create_default(&context, 0).map_err(|e| refuse(cl(e)))?;
    let kernel = Kernel::build(&context, opencl::SOURCE).map_err(refuse)?;
    // The state is one buffer: no larger than one allocation, or half the
    // device's memory.
    let max_alloc = device.max_mem_alloc_size().unwrap_or(0);
    let max_qubits = qubits_for(max_alloc.min(memory / 2));
    let detail = format!(
        "opencl; platform {:?} ({}); device {name:?}, {}; {} compute units; {}; driver {}; uuid {}; memory {memory} bytes, {max_alloc} per allocation; up to {max_qubits} qubits; a metered run's window runs from the first upload to the last readback",
        identity.platform,
        identity.platform_version,
        kind_label(kind),
        identity.compute_units,
        identity.version,
        identity.driver,
        if identity.uuid.is_empty() { "not reported" } else { &identity.uuid },
    );
    let member = OpenClMember {
        context: Arc::new(context),
        queue: Arc::new(queue),
        kernel: Arc::new(Mutex::new(kernel)),
        id: identity.member_id(),
        key: key.clone(),
        class,
        implementation: opencl::implementation_id(),
        detail,
        max_qubits,
        binary64: extensions.split_whitespace().any(|x| x == "cl_khr_fp64"),
        identity,
    };
    let pairs = lanes::probe_pairs(4096, 0x5eed);
    match member.probe(&pairs) {
        Ok(out) => match lanes::first_mismatch(&pairs, &out) {
            None => Ok(member),
            Some((a, b)) => Err(refuse(OpenClError::Probe { a, b })),
        },
        Err(e) => Err(refuse(e)),
    }
}

fn kind_label(kind: DeviceKind) -> &'static str {
    match kind {
        DeviceKind::Gpu => "a GPU",
        DeviceKind::Cpu => "a CPU",
        DeviceKind::Accelerator => "an accelerator",
    }
}

/// The kernel built for one device: its two entry points.
struct Kernel {
    apply: ClKernel,
    probe: ClKernel,
}

impl Kernel {
    fn build(context: &Context, source: &str) -> Result<Kernel, OpenClError> {
        let program = ClProgram::create_and_build_from_source(context, source, opencl::BUILD_OPTIONS).map_err(OpenClError::Build)?;
        Ok(Kernel { apply: ClKernel::create(&program, "apply_gate").map_err(cl)?, probe: ClKernel::create(&program, "probe").map_err(cl)? })
    }
}

/// The global work size for `work` items: at most [`opencl::GLOBAL_MAX`],
/// the kernel striding over the rest; the work-group size is the runtime's.
fn global(work: u64) -> usize {
    work.clamp(1, opencl::GLOBAL_MAX) as usize
}

/// One OpenCL device as a fabric member: a context and an in-order queue on
/// it, the kernel built for it, and the arithmetic probe passed in this
/// process.
#[derive(Clone)]
pub struct OpenClMember {
    context: Arc<Context>,
    queue: Arc<CommandQueue>,
    /// Behind a lock: setting a kernel's arguments is the one OpenCL call that
    /// is not thread-safe, so each launch holds it from the first argument to
    /// the enqueue.
    kernel: Arc<Mutex<Kernel>>,
    identity: DeviceIdentity,
    id: String,
    key: String,
    class: FabricClass,
    implementation: String,
    detail: String,
    max_qubits: u8,
    /// Whether the device has binary64 (`cl_khr_fp64`).
    binary64: bool,
}

// SAFETY: every OpenCL API call is thread-safe except those that change a
// kernel object's state (OpenCL 3.0 API, "Multiple Host Threads"):
// `clSetKernelArg` and the enqueue that reads the arguments run only while
// `kernel`'s lock is held. The context and the queue are only passed to calls
// that are thread-safe.
unsafe impl Send for OpenClMember {}
// SAFETY: as above.
unsafe impl Sync for OpenClMember {}

impl std::fmt::Debug for OpenClMember {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("OpenClMember").field("id", &self.id).field("key", &self.key).field("implementation", &self.implementation).field("max_qubits", &self.max_qubits).finish()
    }
}

impl OpenClMember {
    /// The member id ([`DeviceIdentity::member_id`]).
    pub fn id(&self) -> &str {
        &self.id
    }

    /// The key a runner declares for it ([`DeviceIdentity::key`]).
    pub fn key(&self) -> &str {
        &self.key
    }

    /// [`FabricClass::Gpu`] or [`FabricClass::Cpu`].
    pub fn class(&self) -> FabricClass {
        self.class
    }

    /// What discovery found: platform, device, versions, memory, and what a
    /// metered run's window covers. Not signed.
    pub fn detail(&self) -> &str {
        &self.detail
    }

    /// The most qubits a run holds on this member.
    pub fn max_qubits(&self) -> u8 {
        self.max_qubits
    }

    /// The implementation id ([`opencl::implementation_id`], or a mutant's).
    pub fn implementation(&self) -> &str {
        &self.implementation
    }

    /// What the member id hashes.
    pub fn identity(&self) -> &DeviceIdentity {
        &self.identity
    }

    /// Whether the device has binary64 arithmetic (`cl_khr_fp64`), which the
    /// state vector needs.
    pub fn has_binary64(&self) -> bool {
        self.binary64
    }

    fn kernel(&self) -> MutexGuard<'_, Kernel> {
        self.kernel.lock().unwrap_or_else(|p| p.into_inner())
    }

    /// A device buffer of `data`.
    fn upload<T: Copy>(&self, data: &[T]) -> Result<Buffer<T>, OpenClError> {
        // SAFETY: no host pointer is given; the buffer is `data.len()` items.
        let mut b = unsafe { Buffer::<T>::create(&self.context, CL_MEM_READ_WRITE, data.len(), std::ptr::null_mut()) }.map_err(cl)?;
        // SAFETY: a blocking write of exactly the buffer's size.
        unsafe { self.queue.enqueue_write_buffer(&mut b, CL_BLOCKING, 0, data, &[]) }.map_err(cl)?;
        Ok(b)
    }

    /// The contents of `b`, `len` items.
    fn download<T: Copy + Default>(&self, b: &Buffer<T>, len: usize) -> Result<Vec<T>, OpenClError> {
        let mut out: Vec<T> = Vec::new();
        out.try_reserve_exact(len).map_err(|e| OpenClError::Memory(e.to_string()))?;
        out.resize(len, T::default());
        // SAFETY: a blocking read of exactly the buffer's size into `out`.
        unsafe { self.queue.enqueue_read_buffer(b, CL_BLOCKING, 0, &mut out, &[]) }.map_err(cl)?;
        Ok(out)
    }

    /// The device's [`lanes::mul_wide`] and [`lanes::fxmul_wide`] of each
    /// pair, in order.
    pub fn probe(&self, pairs: &[(i32, i32)]) -> Result<Vec<ProbeOutput>, OpenClError> {
        if pairs.is_empty() {
            return Ok(Vec::new());
        }
        let flat: Vec<cl_int> = pairs.iter().flat_map(|&(a, b)| [a, b]).collect();
        let cases = self.upload(&flat)?;
        let out = self.upload(&vec![0 as cl_uint; pairs.len() * 4])?;
        let count = pairs.len() as cl_ulong;
        {
            let k = self.kernel();
            // SAFETY: `cases` holds `count` int2 cases and `out` `count` uint4
            // outputs, the kernel's arguments in order.
            unsafe { ExecuteKernel::new(&k.probe).set_arg(&cases).set_arg(&out).set_arg(&count).set_global_work_size(global(count)).enqueue_nd_range(&self.queue) }
                .map_err(cl)?;
        }
        let words = self.download(&out, pairs.len() * 4)?;
        Ok(words.as_chunks::<4>().0.iter().map(|w| ((w[0], w[1]), (w[2], w[3]))).collect())
    }

    /// The planning this member runs: [`Config::PER_GATE`].
    pub fn config(&self) -> Config {
        Config::PER_GATE
    }

    /// Runs `p` from its basis state: the lanes, and the first op whose exact
    /// result left 32 bits. A program another config planned (one with tile
    /// steps) is [`OpenClError::Internal`]: this member runs gate steps only.
    pub fn run_program(&self, p: &Program) -> Result<LaneRun, OpenClError> {
        Ok(self.run_program_metered(p, &mut Unmetered)?.0)
    }

    fn run_program_metered<M: RunMeter + ?Sized>(&self, p: &Program, meter: &mut M) -> Result<MeteredRun, OpenClError> {
        let n = p.n_qubits;
        if n == 0 {
            return Ok((LaneRun { amps: vec![[ONE as i32, 0]], out_of_range: None }, (0, None)));
        }
        if n > self.max_qubits {
            return Err(OpenClError::TooWide { n_qubits: n, max: self.max_qubits });
        }
        if n > MAX_QUBITS || p.start >= 1usize << n {
            return Err(OpenClError::Internal("the start is outside the state"));
        }
        let len = 1usize << n;
        let mut gates = Vec::with_capacity(p.steps.len());
        for step in &p.steps {
            let Step::Gate { gate } = *step else { return Err(OpenClError::Internal("a tile step; this member runs gate steps only")) };
            let g = p.gates.get(gate as usize).ok_or(OpenClError::Internal("a step names a gate the program does not hold"))?;
            if !g.is_well_formed() {
                return Err(OpenClError::Internal("malformed gate words for this state"));
            }
            gates.push((&g.words, g.index));
        }
        let mut init: Vec<i32> = Vec::new();
        init.try_reserve_exact(2 * len).map_err(|e| OpenClError::Memory(e.to_string()))?;
        init.resize(2 * len, 0);
        init[2 * p.start] = ONE as i32;
        self.execute(init, &gates, meter)
    }

    /// Gates in order from `state`, as [`lanes::run_words`] runs them: the
    /// lanes, and the lowest position in `gates` of a gate whose exact result
    /// left 32 bits. A state that is not `2^n` lanes (`n ≥ 1`) or words
    /// [`lanes::apply_words`] refuses for it are [`OpenClError::Internal`].
    pub fn run_lanes(&self, state: &[Lane], gates: &[GateWords]) -> Result<LaneRun, OpenClError> {
        let len = state.len();
        if len < 2 || !len.is_power_of_two() {
            return Err(OpenClError::Internal("a state that is not 2^n lanes, n at least 1"));
        }
        let n = len.trailing_zeros() as u8;
        if n > self.max_qubits {
            return Err(OpenClError::TooWide { n_qubits: n, max: self.max_qubits });
        }
        let planned: Vec<(&GateWords, u32)> = gates.iter().zip(0u32..).collect();
        let mut init: Vec<i32> = Vec::new();
        init.try_reserve_exact(2 * len).map_err(|e| OpenClError::Memory(e.to_string()))?;
        init.extend(state.iter().flat_map(|l| [l[0], l[1]]));
        Ok(self.execute(init, &planned, &mut Unmetered)?.0)
    }

    /// Uploads `init` (interleaved lanes, `2·2^n` words), runs `gates` in order
    /// inside a window of `meter`, and reads the report and the state back.
    /// Every gate's words are checked against the state first: the words
    /// [`lanes::apply_words`] refuses, the host refuses.
    fn execute<M: RunMeter + ?Sized>(&self, init: Vec<i32>, gates: &[(&GateWords, u32)], meter: &mut M) -> Result<MeteredRun, OpenClError> {
        let len = init.len() / 2;
        for (w, _) in gates {
            let (tbit, ctrl) = (w.tbit as usize, w.ctrl as usize);
            if tbit == 0 || !tbit.is_power_of_two() || tbit > len / 2 || ctrl & tbit != 0 || ctrl & !(len - 1) != 0 {
                return Err(OpenClError::Internal("malformed gate words for this state"));
            }
        }
        let pairs = (len / 2) as cl_ulong;
        let mut window = Window::new(meter);
        window.open();
        let amps = self.upload(&init)?;
        drop(init);
        let report = self.upload(&[u32::MAX as cl_uint])?;
        {
            let k = self.kernel();
            for &(w, index) in gates {
                let m: [cl_int; 8] = w.m.map(|x| x as i32);
                let mut launch = ExecuteKernel::new(&k.apply);
                // SAFETY: `amps` holds `2·pairs` int2 lanes and `report` one word,
                // the words were checked against the state above, and the
                // arguments are the kernel's, in order.
                unsafe {
                    launch.set_arg(&amps).set_arg(&pairs).set_arg(&w.tbit).set_arg(&w.ctrl);
                    for v in &m {
                        launch.set_arg(v);
                    }
                    launch.set_arg(&index).set_arg(&report).set_global_work_size(global(pairs)).enqueue_nd_range(&self.queue)
                }
                .map_err(cl)?;
            }
        }
        let bad = self.download(&report, 1)?;
        let words = self.download(&amps, 2 * len)?;
        let reading = window.close();
        let out_of_range = bad.first().copied().filter(|&k| k != u32::MAX);
        let mut lanes_out: Vec<Lane> = Vec::new();
        lanes_out.try_reserve_exact(len).map_err(|e| OpenClError::Memory(e.to_string()))?;
        lanes_out.extend(words.as_chunks::<2>().0.iter().copied());
        Ok((LaneRun { amps: lanes_out, out_of_range }, reading))
    }

    /// Equal to `c.simulate_from(start)`, or a typed refusal: what the CPU
    /// reference refuses ([`OpenClError::Circuit`]), more qubits than the
    /// member holds, or a result that leaves 32 bits.
    pub fn simulate_from(&self, c: &Circuit, start: usize) -> Result<StateVector, OpenClError> {
        Ok(self.simulate_with(c, start, &mut Unmetered)?.0)
    }

    /// [`Self::simulate_from`] inside a window of `meter`: the state, and the
    /// microjoules and class the meter read, which are zero and `None` unless
    /// the crate's label check accepts them. The window opens just before the
    /// first upload and closes after the last readback.
    pub fn simulate_metered(&self, c: &Circuit, start: usize, meter: &mut dyn RunMeter) -> Result<(StateVector, u64, Option<EnergyClass>), OpenClError> {
        self.simulate_with(c, start, meter)
    }

    fn simulate_with<M: RunMeter + ?Sized>(&self, c: &Circuit, start: usize, meter: &mut M) -> Result<(StateVector, u64, Option<EnergyClass>), OpenClError> {
        let p = plan::compile_with(c, start, &self.config()).map_err(OpenClError::Circuit)?;
        if p.n_qubits > self.max_qubits {
            return Err(OpenClError::TooWide { n_qubits: p.n_qubits, max: self.max_qubits });
        }
        let (run, (joules_micro, class)) = self.run_program_metered(&p, meter)?;
        if let Some(gate) = run.out_of_range {
            return Err(OpenClError::OutOfRange { gate });
        }
        let mut amps = Vec::new();
        amps.try_reserve_exact(run.amps.len()).map_err(|e| OpenClError::Memory(e.to_string()))?;
        amps.extend(run.amps.iter().map(|l| Amp { re: i64::from(l[0]), im: i64::from(l[1]) }));
        Ok((StateVector { n_qubits: p.n_qubits, amps }, joules_micro, class))
    }

    /// [`Self::simulate_from`] the all-zeros state.
    pub fn simulate(&self, c: &Circuit) -> Result<StateVector, OpenClError> {
        self.simulate_from(c, 0)
    }

    /// The certificate of [`apply_kernel`]`(gs)` as [`Self::implementation`]
    /// on this member, over `corpus`: each circuit run from `|0…0⟩`, against
    /// the native simulator. A range report inside a corpus is an error: its
    /// circuits are valid and within the horizon.
    pub fn certify(&self, gs: Gateset, corpus: Corpus) -> Result<Certificate, OpenClError> {
        Ok(self.certify_metered(gs, corpus, &mut Unmetered)?.0)
    }

    fn certify_metered<M: RunMeter + ?Sized>(&self, gs: Gateset, corpus: Corpus, meter: &mut M) -> Result<(Certificate, Option<(u64, EnergyClass)>), OpenClError> {
        let mut cert = Certification::new();
        let mut calibration = EnergyCalibration::new();
        for c in corpus.circuits(gs) {
            let reference = lanes::reference_output(&c).map_err(OpenClError::Circuit)?;
            let (run, (joules_micro, class)) = self.run_program_metered(&plan::compile_with(&c, 0, &self.config()).map_err(OpenClError::Circuit)?, &mut *meter)?;
            if let Some(gate) = run.out_of_range {
                return Err(OpenClError::OutOfRange { gate });
            }
            cert.add(&lanes::circuit_bytes(&c), &reference, &lanes::output(&run.amps));
            calibration.add(pair_updates(&c), joules_micro, class.as_ref());
        }
        Ok((cert.finish(apply_kernel(gs), &self.implementation, &self.id, &corpus.generator(gs)), calibration.finish()))
    }

    /// A fabric [`Member`]: present as [`Presence::Detected`], with one entry
    /// per gateset, each certified in this call on `corpus`, and no energy
    /// figure.
    pub fn member(&self, corpus: Corpus) -> Result<Member, OpenClError> {
        self.member_with(corpus, &mut Unmetered)
    }

    /// [`Self::member`], each corpus run inside its own window of `meter`:
    /// every entry also carries the energy figure its runs earn
    /// ([`EnergyCalibration`]), or none if any run went unmetered.
    pub fn member_metered(&self, corpus: Corpus, meter: &mut dyn RunMeter) -> Result<Member, OpenClError> {
        self.member_with(corpus, meter)
    }

    fn member_with<M: RunMeter + ?Sized>(&self, corpus: Corpus, meter: &mut M) -> Result<Member, OpenClError> {
        let mut kernels = Vec::new();
        for gs in Gateset::ALL {
            let (certificate, energy) = self.certify_metered(gs, corpus, &mut *meter)?;
            kernels.push(KernelEntry { kernel: apply_kernel(gs).into(), implementation: self.implementation.clone(), certificate, energy });
        }
        Ok(Member { id: self.id.clone(), class: self.class, presence: Presence::Detected, kernels, detail: self.detail.clone() })
    }

    /// The same member holding at most `max_qubits` qubits (no more than it
    /// holds), so that tests can make it refuse a circuit as too wide.
    #[doc(hidden)]
    pub fn with_max_qubits(&self, max_qubits: u8) -> OpenClMember {
        OpenClMember { max_qubits: max_qubits.min(self.max_qubits), ..self.clone() }
    }

    /// The same device running `source` in place of the kernel, under the
    /// implementation id that text earns: for the mutation tests, which
    /// require every mutant to be caught.
    #[doc(hidden)]
    pub fn with_source(&self, source: &str) -> Result<OpenClMember, OpenClError> {
        let kernel = Kernel::build(&self.context, source)?;
        Ok(OpenClMember { kernel: Arc::new(Mutex::new(kernel)), implementation: opencl::implementation_id_of(source), ..self.clone() })
    }
}

/// The dense f64 state vector on this device
/// ([`quantum_fabric::sv64`](crate::quantum_fabric::sv64)), as for the CUDA
/// member: the CPU's launches, each formula one IEEE binary64 operation at a
/// time, contraction off. A device without binary64 refuses it.
#[cfg(feature = "quantum_sv")]
#[derive(Clone)]
pub struct OpenClSv {
    member: OpenClMember,
    kernels: Arc<Mutex<SvKernels>>,
    implementation: String,
}

#[cfg(feature = "quantum_sv")]
struct SvKernels {
    diag: ClKernel,
    one: ClKernel,
    many: ClKernel,
}

// SAFETY: as for [`OpenClMember`]: the kernels' arguments are set and enqueued
// only under their lock.
#[cfg(feature = "quantum_sv")]
unsafe impl Send for OpenClSv {}
// SAFETY: as above.
#[cfg(feature = "quantum_sv")]
unsafe impl Sync for OpenClSv {}

#[cfg(feature = "quantum_sv")]
impl std::fmt::Debug for OpenClSv {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("OpenClSv").field("member", &self.member.id).field("implementation", &self.implementation).finish()
    }
}

#[cfg(feature = "quantum_sv")]
impl OpenClMember {
    /// The state-vector kernels built for this device, or
    /// [`OpenClError::Unsupported`] on a device without binary64.
    pub fn sv(&self) -> Result<OpenClSv, OpenClError> {
        self.sv_with_source(crate::quantum_fabric::sv64::CL_SOURCE)
    }

    /// The same, from another kernel text: for the mutation tests.
    #[doc(hidden)]
    pub fn sv_with_source(&self, source: &str) -> Result<OpenClSv, OpenClError> {
        use crate::quantum_fabric::sv64;
        if !self.binary64 {
            return Err(OpenClError::Unsupported("the device has no binary64 (cl_khr_fp64)".into()));
        }
        let program = ClProgram::create_and_build_from_source(&self.context, source, sv64::CL_BUILD_OPTIONS).map_err(OpenClError::Build)?;
        let kernels = SvKernels {
            diag: ClKernel::create(&program, "sv_diag").map_err(cl)?,
            one: ClKernel::create(&program, "sv_one").map_err(cl)?,
            many: ClKernel::create(&program, "sv_many").map_err(cl)?,
        };
        Ok(OpenClSv { member: self.clone(), kernels: Arc::new(Mutex::new(kernels)), implementation: sv64::cl_implementation_id_of(source) })
    }
}

#[cfg(feature = "quantum_sv")]
impl OpenClSv {
    pub fn implementation(&self) -> &str {
        &self.implementation
    }

    /// `gates` on `n` qubits from `|0…0⟩`, fused to `width` as the CPU fuses
    /// them: the real and imaginary parts.
    pub fn run(&self, n: u32, gates: &[crate::gates::Gate], width: usize) -> Result<(Vec<f64>, Vec<f64>), OpenClError> {
        use crate::quantum_fabric::sv64::{Launch, plan};
        let launches = plan(n, gates, width).map_err(|_| OpenClError::Internal("a gate the state vector refuses"))?;
        let max = self.member.max_qubits.saturating_sub(1);
        if n == 0 || n > u32::from(max) {
            return Err(OpenClError::TooWide { n_qubits: n.min(255) as u8, max });
        }
        let len = 1usize << n;
        let mut init = vec![0.0f64; len];
        init[0] = 1.0;
        let re = self.member.upload(&init)?;
        init[0] = 0.0;
        let im = self.member.upload(&init)?;
        drop(init);
        let flat = |m: &[crate::linalg::C]| -> Vec<f64> { m.iter().flat_map(|c| [c.re, c.im]).collect() };
        {
            let k = self.kernels.lock().unwrap_or_else(|p| p.into_inner());
            for l in &launches {
                match l {
                    Launch::Diag { bitpos, diag } => {
                        let bp = self.member.upload(bitpos)?;
                        let dg = self.member.upload(&flat(diag))?;
                        let (count, nk): (cl_ulong, cl_uint) = (len as u64, bitpos.len() as u32);
                        // SAFETY: `re`, `im` hold `len` doubles, `bp` k bit
                        // positions below n, `dg` 2·2^k doubles: the kernel's
                        // arguments, in order.
                        unsafe { ExecuteKernel::new(&k.diag).set_arg(&re).set_arg(&im).set_arg(&count).set_arg(&bp).set_arg(&nk).set_arg(&dg).set_global_work_size(global(count)).enqueue_nd_range(&self.member.queue) }.map_err(cl)?;
                    }
                    Launch::One { s, m } => {
                        let pairs = (len / 2) as cl_ulong;
                        let v = flat(m);
                        let mut launch = ExecuteKernel::new(&k.one);
                        // SAFETY: as above; `s` is a power of two below 2^n.
                        unsafe {
                            launch.set_arg(&re).set_arg(&im).set_arg(&pairs).set_arg(s);
                            for x in &v {
                                launch.set_arg(x);
                            }
                            launch.set_global_work_size(global(pairs)).enqueue_nd_range(&self.member.queue)
                        }
                        .map_err(cl)?;
                    }
                    Launch::Many { sorted, offsets, m } => {
                        let so = self.member.upload(sorted)?;
                        let of = self.member.upload(offsets)?;
                        let mm = self.member.upload(&flat(m))?;
                        let (groups, nk): (cl_ulong, cl_uint) = ((len >> sorted.len()) as u64, sorted.len() as u32);
                        // SAFETY: as above; 2^k is at most the kernel's SV_MAX_D,
                        // which `plan` holds.
                        unsafe { ExecuteKernel::new(&k.many).set_arg(&re).set_arg(&im).set_arg(&groups).set_arg(&so).set_arg(&nk).set_arg(&of).set_arg(&mm).set_global_work_size(global(groups)).enqueue_nd_range(&self.member.queue) }.map_err(cl)?;
                    }
                }
            }
        }
        Ok((self.member.download(&re, len)?, self.member.download(&im, len)?))
    }

    /// The certificate of [`KERNEL_SV_RUN`](crate::quantum_fabric::sv64::KERNEL_SV_RUN)
    /// on this device over `corpus`, against the CPU's state vector.
    pub fn certify(&self, corpus: crate::quantum_fabric::sv64::SvCorpus) -> Result<Certificate, OpenClError> {
        use crate::quantum_fabric::sv64::{KERNEL_SV_RUN, state_output};
        let mut cert = Certification::new();
        for case in corpus.cases() {
            let reference = case.reference().map_err(|_| OpenClError::Internal("the corpus holds a gate the state vector refuses"))?;
            let (re, im) = self.run(case.n, &case.gates, case.width)?;
            cert.add(&case.bytes(), &reference, &state_output(&re, &im));
        }
        Ok(cert.finish(KERNEL_SV_RUN, &self.implementation, &self.member.id, &corpus.generator()))
    }

    /// The member's entry for the state vector, certified in this call.
    pub fn entry(&self, corpus: crate::quantum_fabric::sv64::SvCorpus) -> Result<KernelEntry, OpenClError> {
        Ok(KernelEntry { kernel: crate::quantum_fabric::sv64::KERNEL_SV_RUN.into(), implementation: self.implementation.clone(), certificate: self.certify(corpus)?, energy: None })
    }
}

/// The member on the fabric ([`Fabric`](crate::quantum_fabric::Fabric)): its
/// own implementation, through [`OpenClMember::simulate_metered`]. Pair it
/// with a [`Member`] whose entries name that implementation, such as
/// [`OpenClMember::member`]'s; another implementation is
/// [`MemberError::Other`].
impl Runner for OpenClMember {
    fn member_id(&self) -> String {
        self.id.clone()
    }

    fn run<'r>(&'r self, implementation: &'r str, c: &'r Circuit, start: usize, meter: &'r mut dyn RunMeter) -> RunFuture<'r> {
        Box::pin(async move {
            if implementation != self.implementation {
                return Err(MemberError::Other(format!("member {} runs implementation {}, not {implementation:?}", self.id, self.implementation)));
            }
            Ok(self.simulate_with(c, start, meter)?)
        })
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn running_out_of_memory_or_resources_is_a_memory_error_and_every_other_a_device_error() {
        for code in OUT_OF_MEMORY {
            assert!(matches!(cl(ClError(code)), OpenClError::Memory(_)), "{code}");
        }
        assert!(matches!(cl(ClError(-36)), OpenClError::Device(_)));
        assert!(MemberError::from(OpenClError::Memory("x".into())).falls_back());
        assert!(MemberError::from(OpenClError::OutOfRange { gate: 3 }).falls_back());
        assert!(!MemberError::from(OpenClError::Probe { a: 1, b: 2 }).falls_back());
        assert!(!MemberError::from(OpenClError::Internal("x")).falls_back());
    }

    #[test]
    fn the_state_a_member_holds_is_what_one_allocation_holds() {
        assert_eq!(qubits_for(0), 0);
        assert_eq!(qubits_for(15), 0);
        assert_eq!(qubits_for(16), 1);
        assert_eq!(qubits_for(1536 << 20), MAX_QUBITS.min(27));
        assert_eq!(qubits_for(u64::MAX), MAX_QUBITS);
    }

    #[test]
    fn the_global_size_covers_the_work_and_is_capped() {
        assert_eq!(global(0), 1);
        assert_eq!(global(7), 7);
        assert_eq!(global(1 << 40), opencl::GLOBAL_MAX as usize);
    }
}