wai-quantum 0.4.1

//! The fabric's WebGPU member: the gate kernel of
//! [`quantum_fabric::wgsl`](crate::quantum_fabric::wgsl) on a GPU, through the
//! platform's GPU interface natively (Metal, Vulkan, DX12) and through a
//! browser's WebGPU on `wasm32-unknown-unknown`. See that module for the
//! kernel and the argument that it is bit-exact.
//!
//! - **Discovery** ([`discover`]): every adapter the platform offers is opened
//!   at the WebGPU downlevel limits, with only the two buffer limits raised to
//!   the adapter's, and its arithmetic is probed against [`lanes`] in this
//!   process. An adapter that cannot host the kernel, or whose arithmetic
//!   disagrees, is listed with the reason ([`Refusal`]); one that passes is a
//!   [`WebGpuMember`], present as [`Presence::Detected`].
//! - **Running** ([`WebGpuMember::run_program`]): one dispatch per gate, in
//!   place, in op-log order, with the gate records in a uniform ring selected by
//!   dynamic offset; submissions are bounded by [`SUBMIT_BUDGET`] pair-updates
//!   and natively each is waited on, within [`POLL_TIMEOUT`], before the next;
//!   the state comes back in [`READBACK_CHUNK`] pieces. The kernel reports
//!   exactly when the reference's result leaves 32 bits; the member then returns
//!   [`WebGpuError::OutOfRange`], never a state.
//! - **Certification** ([`WebGpuMember::certify`], [`WebGpuMember::member`]):
//!   the member's own run of a corpus, folded case by case into a
//!   [`Certification`]. The corpus digest covers inputs and outputs only, so a
//!   bit-exact member reproduces the CPU's pinned digests for random-v1 and
//!   fabric-v1; a device that runs nothing, skips a case or gets one bit wrong
//!   cannot.
//! - **Placement and receipts** go through
//!   [`place`](crate::quantum_fabric::place) and
//!   [`FabricReceipt`](crate::quantum_fabric::FabricReceipt) unchanged. A member
//!   carries no energy figure, so beside the CPU reference, whose id sorts
//!   first, the reference is placed; a caller who wants the GPU places over the
//!   GPU member alone.
//!
//! Every runtime entry point is an `async fn`, and the crate brings no
//! executor. Natively, every future the GPU interface returns is already
//! resolved and a buffer map completes inside the device poll that waits for
//! its submission, so [`blocking`] resolves each call with one poll; a future
//! still pending after it is [`WebGpuError::Pending`], never a spin. A native
//! run blocks the polling thread until the device finishes, so under an async
//! runtime it belongs in a blocking context. In a browser the same futures are
//! awaited.
//!
//! The GPU interface reports a failed native wait, such as the loss of the
//! device during a run, by panicking inside its device poll. The member catches
//! that panic, returns [`WebGpuError::Device`] and treats the device as lost
//! from then on; the panic message is still printed by the panic hook. A build
//! with `panic = "abort"` cannot catch it and aborts there.
//!
//! No type of the GPU interface appears in this module's public API.
//!
//! # Honest boundaries
//!
//! - **A member is what this process measured.** The probe and the
//!   certificates are earned per process; the member id binds the adapter, its
//!   driver as far as the platform reports one, the translator and the granted
//!   buffer limits, so a certificate does not carry over to another adapter.
//! - **A software adapter is class `Cpu`.** Its id begins `local/swgpu-`, which
//!   sorts after every `local/cpu-<arch>`, so it never displaces the CPU
//!   reference by its name.
//! - **Unmetered.** Receipts over a member's runs carry no joules and no
//!   acquisition class.

use crate::quantum::{Amp, Circuit, Gateset, MAX_QUBITS, ONE, QuantumError, StateVector};
use crate::quantum_fabric::lanes::{self, GateWords, Lane};
use crate::quantum_fabric::plan::{self, LaneRun, Program, Step};
use crate::quantum_fabric::wgsl::{self, AdapterIdentity};
use crate::quantum_fabric::{Certificate, Certification, Corpus, FabricClass, KernelEntry, Member, Presence, apply_kernel};
use core::future::Future;
use core::pin::Pin;
use core::task::{Context, Poll, Waker};
use std::borrow::Cow;
use std::num::NonZeroU64;
use std::sync::{Arc, Mutex};

/// Pair-updates per submission, at most: each command buffer stays well
/// inside a GPU watchdog's window.
pub const SUBMIT_BUDGET: u64 = 1 << 28;

/// Gate records per uniform ring, at most: gates per submission.
pub const RING_RECORDS: u32 = 4096;

/// Bytes of state read back per copy.
pub const READBACK_CHUNK: u64 = 64 << 20;

/// How long a native wait on one submission may take. A run waits on each of
/// its submissions before it makes the next: each chunk of gates (at most
/// [`SUBMIT_BUDGET`] pair-updates and [`RING_RECORDS`] gates) and each
/// readback copy. So the timeout bounds the GPU time of one chunk, not of a
/// run. A browser schedules submissions itself, and the member does not wait
/// there between them.
pub const POLL_TIMEOUT: core::time::Duration = core::time::Duration::from_secs(300);

/// Why the member did not return a result.
#[derive(Debug, PartialEq, Eq)]
#[non_exhaustive]
pub enum WebGpuError {
    /// The instance offered no adapter at all.
    NoAdapter(String),
    /// An adapter that cannot host the kernel, and why (limits, no compute).
    Unsuitable { adapter: String, why: String },
    /// Device creation, validation, an internal error or device loss,
    /// including a failed native wait (see the module documentation).
    Device(String),
    /// Out of memory on the device or the host.
    Memory(String),
    /// A native wait passed [`POLL_TIMEOUT`].
    Timeout,
    /// A native future was still pending after one poll.
    Pending,
    /// Exactly what the CPU reference refuses.
    Circuit(QuantumError),
    /// More qubits than this member's buffers hold.
    TooWide { n_qubits: u8, max: u8 },
    /// The reference's result leaves 32 bits at this op; the member returns
    /// no state.
    OutOfRange { gate: u32 },
    /// The kernel's report bits 2 (an index outside the state) or 4 (a
    /// malformed record): a defect of the host, never a result. A program or
    /// state the host cannot lay out is refused with the same bits.
    Internal(u32),
    /// The device's arithmetic disagreed with the CPU's on this pair.
    Probe { a: i32, b: i32 },
}

impl std::fmt::Display for WebGpuError {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            WebGpuError::NoAdapter(why) => write!(f, "no adapter: {why}"),
            WebGpuError::Unsuitable { adapter, why } => write!(f, "adapter {adapter:?} cannot host the kernel: {why}"),
            WebGpuError::Device(e) => write!(f, "device: {e}"),
            WebGpuError::Memory(e) => write!(f, "out of memory: {e}"),
            WebGpuError::Timeout => write!(f, "a wait on the device passed {} s", POLL_TIMEOUT.as_secs()),
            WebGpuError::Pending => write!(f, "a native future was pending after one poll"),
            WebGpuError::Circuit(e) => write!(f, "{e}"),
            WebGpuError::TooWide { n_qubits, max } => write!(f, "{n_qubits} qubits; this member holds at most {max}"),
            WebGpuError::OutOfRange { gate } => write!(f, "the result of op {gate} leaves 32 bits"),
            WebGpuError::Internal(bits) => write!(f, "host defect, report bits {bits:#x}"),
            WebGpuError::Probe { a, b } => write!(f, "the device's arithmetic differs from the CPU's on {a} × {b}"),
        }
    }
}

impl std::error::Error for WebGpuError {}

/// An adapter that is not a member, and why.
#[derive(Debug)]
pub struct Refusal {
    /// The adapter's name, as the platform reports it.
    pub adapter: String,
    /// Its CI key ([`AdapterIdentity::key`]).
    pub key: String,
    pub error: WebGpuError,
}

/// What [`discover`] found.
#[derive(Debug)]
pub struct Discovery {
    /// Adapters that created a device and passed the probe, in enumeration
    /// order.
    pub found: Vec<WebGpuMember>,
    /// Adapters that did not, with why.
    pub refused: Vec<Refusal>,
}

/// Every adapter of the platform's primary GPU interfaces natively (Metal,
/// Vulkan, DX12), or the browser's adapter on the web, each opened and probed
/// in this process before it is listed. On a platform none of them is built
/// for, nothing is found.
pub async fn discover() -> Discovery {
    let mut d = Discovery { found: Vec::new(), refused: Vec::new() };
    if wgpu::Instance::enabled_backend_features().is_empty() {
        return d;
    }
    for adapter in instance().enumerate_adapters(wgpu::Backends::PRIMARY).await {
        match open(adapter).await {
            Ok(m) => d.found.push(m),
            Err(r) => d.refused.push(r),
        }
    }
    d
}

fn instance() -> wgpu::Instance {
    let mut d = wgpu::InstanceDescriptor::new_without_display_handle();
    d.backends = wgpu::Backends::PRIMARY;
    // No backend validation layers, and nothing read from the environment.
    d.flags = wgpu::InstanceFlags::empty();
    // The default compiler choice depends on what the host has installed; the
    // member pins the one that ships with the platform, and names it in its id.
    d.backend_options.dx12.shader_compiler = wgpu::Dx12Compiler::Fxc;
    wgpu::Instance::new(d)
}

/// The backend's name in a member id, or `None` for a backend the member does
/// not run on.
fn backend_label(b: wgpu::Backend) -> Option<&'static str> {
    match b {
        wgpu::Backend::Metal => Some("metal"),
        wgpu::Backend::Vulkan => Some("vulkan"),
        wgpu::Backend::Dx12 => Some("dx12"),
        wgpu::Backend::BrowserWebGpu => Some("webgpu"),
        _ => None,
    }
}

fn device_type_label(t: wgpu::DeviceType) -> &'static str {
    match t {
        wgpu::DeviceType::DiscreteGpu => "discrete",
        wgpu::DeviceType::IntegratedGpu => "integrated",
        wgpu::DeviceType::VirtualGpu => "virtual",
        wgpu::DeviceType::Cpu => "cpu",
        wgpu::DeviceType::Other => "other",
    }
}

/// What turns WGSL into the platform's shader language: the built-in
/// translator, followed on DX12 by the bytecode format of the pinned compiler,
/// or the browser's own compiler.
fn translator(b: wgpu::Backend) -> &'static str {
    match b {
        wgpu::Backend::Dx12 => "naga+dxbc",
        wgpu::Backend::BrowserWebGpu => "user-agent",
        _ => "naga",
    }
}

fn identity(info: &wgpu::AdapterInfo, max_storage_binding: u64, max_buffer: u64) -> AdapterIdentity {
    AdapterIdentity {
        backend: backend_label(info.backend).map_or_else(|| format!("{:?}", info.backend).to_lowercase(), str::to_string),
        name: info.name.clone(),
        vendor: info.vendor,
        device: info.device,
        device_type: device_type_label(info.device_type).into(),
        pci_bus: info.device_pci_bus_id.clone(),
        driver: info.driver.clone(),
        driver_info: info.driver_info.clone(),
        translator: translator(info.backend).into(),
        max_storage_binding,
        max_buffer,
    }
}

/// The widest state two planes of `4·2^n` bytes can hold under these limits.
fn qubits_for(max_storage_binding: u64, max_buffer: u64) -> u8 {
    let words = max_storage_binding.min(max_buffer) / 4;
    if words < 2 { 0 } else { (words.ilog2() as u8).min(MAX_QUBITS) }
}

/// What the device reported outside an error scope: uncaptured errors, and
/// the loss of the device, which every later run reports.
#[derive(Default)]
struct Sink {
    messages: Vec<String>,
    lost: Option<String>,
}

/// The kernel compiled on a device: both entry points, with their layouts.
#[derive(Clone)]
struct Pipelines {
    apply_layout: wgpu::BindGroupLayout,
    probe_layout: wgpu::BindGroupLayout,
    apply: wgpu::ComputePipeline,
    probe: wgpu::ComputePipeline,
}

/// One WebGPU adapter as a fabric member: a device opened on it, the kernel
/// compiled for it, and the arithmetic probe passed in this process.
#[derive(Clone)]
pub struct WebGpuMember {
    device: wgpu::Device,
    queue: wgpu::Queue,
    pipelines: Pipelines,
    identity: AdapterIdentity,
    id: String,
    key: String,
    class: FabricClass,
    backend: &'static str,
    detail: String,
    max_qubits: u8,
    max_x: u32,
    /// Gate records in the uniform ring: gates per submission, at most.
    ring_records: u32,
    /// Bytes of state read back per copy.
    readback_chunk: u64,
    /// Bytes between gate records in the uniform ring.
    stride: u64,
    implementation: String,
    sink: Arc<Mutex<Sink>>,
}

impl std::fmt::Debug for WebGpuMember {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("WebGpuMember").field("id", &self.id).field("key", &self.key).field("implementation", &self.implementation).field("max_x", &self.max_x).field("ring_records", &self.ring_records).field("readback_chunk", &self.readback_chunk).field("detail", &self.detail).finish()
    }
}

/// Opens `adapter` as a member, or says why it is not one.
async fn open(adapter: wgpu::Adapter) -> Result<WebGpuMember, Refusal> {
    let info = adapter.get_info();
    let offered = adapter.limits();
    let mut identity = identity(&info, offered.max_storage_buffer_binding_size, offered.max_buffer_size);
    let refuse = |identity: &AdapterIdentity, error: WebGpuError| Refusal { adapter: identity.name.clone(), key: identity.key(), error };
    let unsuitable = |identity: &AdapterIdentity, why: String| refuse(identity, WebGpuError::Unsuitable { adapter: identity.name.clone(), why });

    // 1. A backend the member runs on.
    let Some(backend) = backend_label(info.backend) else {
        return Err(unsuitable(&identity, format!("backend {:?} is not supported", info.backend)));
    };
    // 2. Compute shaders, and every WebGPU downlevel limit.
    if !adapter.get_downlevel_capabilities().flags.contains(wgpu::DownlevelFlags::COMPUTE_SHADERS) {
        return Err(unsuitable(&identity, "no compute shaders".into()));
    }
    let mut short: Option<String> = None;
    wgpu::Limits::downlevel_defaults().check_limits_with_fail_fn(&offered, true, |name, needed, has| {
        short.get_or_insert_with(|| format!("limit {name}: the kernel needs {needed}, the adapter offers {has}"));
    });
    if let Some(why) = short {
        return Err(unsuitable(&identity, why));
    }
    // 3. The downlevel limits, with the two buffer limits raised to the adapter's.
    let required_limits = wgpu::Limits { max_storage_buffer_binding_size: offered.max_storage_buffer_binding_size, max_buffer_size: offered.max_buffer_size, ..wgpu::Limits::downlevel_defaults() };
    let descriptor = wgpu::DeviceDescriptor { label: Some("wai-quantum fabric"), required_features: wgpu::Features::empty(), required_limits, ..Default::default() };
    let (device, queue) = match adapter.request_device(&descriptor).await {
        Ok(dq) => dq,
        Err(e) => return Err(refuse(&identity, WebGpuError::Device(e.to_string()))),
    };
    // 4. Errors outside a scope, and the loss of the device, are recorded,
    //    never left to the default handler.
    let sink = Arc::new(Mutex::new(Sink::default()));
    let errors = Arc::clone(&sink);
    device.on_uncaptured_error(Arc::new(move |e: wgpu::Error| lock(&errors).messages.push(e.to_string())));
    let lost = Arc::clone(&sink);
    device.set_device_lost_callback(move |reason, message| {
        let mut s = lock(&lost);
        if s.lost.is_none() {
            s.lost = Some(format!("device lost ({reason:?}): {message}"));
        }
    });
    // 5. The kernel.
    let pipelines = match kernel_pipelines(&device, Cow::Borrowed(wgsl::SHADER)).await {
        Ok(p) => p,
        Err(e) => return Err(refuse(&identity, e)),
    };
    // 6. Identity, from what was granted.
    let granted = device.limits();
    identity.max_storage_binding = granted.max_storage_buffer_binding_size;
    identity.max_buffer = granted.max_buffer_size;
    let max_qubits = qubits_for(identity.max_storage_binding, identity.max_buffer);
    let detail = format!(
        "{backend}; adapter {:?}; {}; driver {:?} {:?}; vendor {:#x}, device {:#x}; translator {}; storage binding {} bytes, buffer {} bytes; up to {max_qubits} qubits; probe passed in this process",
        identity.name, identity.device_type, identity.driver, identity.driver_info, identity.vendor, identity.device, identity.translator, identity.max_storage_binding, identity.max_buffer,
    );
    let member = WebGpuMember {
        stride: u64::from(granted.min_uniform_buffer_offset_alignment).max(wgsl::GATE_RECORD_BYTES as u64),
        device,
        queue,
        pipelines,
        id: identity.member_id(),
        key: identity.key(),
        class: identity.class(),
        backend,
        detail,
        max_qubits,
        max_x: wgsl::X_GROUPS_MAX,
        ring_records: RING_RECORDS,
        readback_chunk: READBACK_CHUNK,
        implementation: wgsl::implementation_id(),
        identity,
        sink,
    };
    // 7. The probe: the edges pairwise, and splitmix pairs across the range.
    let pairs = probe_pairs(4096, 0x5eed);
    match member.probe(&pairs).await {
        Ok(out) => match first_mismatch(&pairs, &out) {
            None => Ok(member),
            Some((a, b)) => Err(refuse(&member.identity, WebGpuError::Probe { a, b })),
        },
        Err(e) => Err(refuse(&member.identity, e)),
    }
}

fn lock(sink: &Mutex<Sink>) -> std::sync::MutexGuard<'_, Sink> {
    sink.lock().unwrap_or_else(|p| p.into_inner())
}

/// The probe's operands: every pair of [`lanes::PROBE_EDGES`], then `random`
/// pairs of splitmix64 draws from `seed`.
#[doc(hidden)]
pub fn probe_pairs(random: usize, seed: u64) -> Vec<(i32, i32)> {
    let edges = lanes::PROBE_EDGES;
    let mut pairs: Vec<(i32, i32)> = edges.iter().flat_map(|&a| edges.iter().map(move |&b| (a, b))).collect();
    let mut s = seed;
    pairs.extend((0..random).map(|_| (splitmix(&mut s) as i32, splitmix(&mut s) as i32)));
    pairs
}

fn splitmix(s: &mut u64) -> u64 {
    *s = s.wrapping_add(0x9E37_79B9_7F4A_7C15);
    let mut z = *s;
    z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
    z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
    z ^ (z >> 31)
}

/// The first pair whose device result differs from the CPU's
/// ([`lanes::mul_wide`], [`lanes::fxmul_wide`]).
#[doc(hidden)]
pub fn first_mismatch(pairs: &[(i32, i32)], out: &[((u32, u32), (u32, u32))]) -> Option<(i32, i32)> {
    if pairs.len() != out.len() {
        return pairs.first().copied().or(Some((0, 0)));
    }
    pairs.iter().zip(out).find(|((a, b), o)| **o != (lanes::mul_wide(*a, *b), lanes::fxmul_wide(*a, *b))).map(|(p, _)| *p)
}

/// The three error scopes a step of work runs inside. The fields drop in
/// declaration order, the reverse of the order they were pushed in, as the
/// scope stack requires.
struct Scopes {
    internal: wgpu::ErrorScopeGuard,
    validation: wgpu::ErrorScopeGuard,
    memory: wgpu::ErrorScopeGuard,
}

impl Scopes {
    fn push(device: &wgpu::Device) -> Scopes {
        let memory = device.push_error_scope(wgpu::ErrorFilter::OutOfMemory);
        let validation = device.push_error_scope(wgpu::ErrorFilter::Validation);
        let internal = device.push_error_scope(wgpu::ErrorFilter::Internal);
        Scopes { internal, validation, memory }
    }

    /// Pops the three (each pop takes effect at once, innermost first), then
    /// reads what they caught: memory first, then validation, then internal.
    async fn close(self) -> Result<(), WebGpuError> {
        let Scopes { internal, validation, memory } = self;
        let internal = internal.pop();
        let validation = validation.pop();
        let memory = memory.pop();
        let (internal, validation, memory) = (internal.await, validation.await, memory.await);
        if let Some(e) = memory {
            return Err(WebGpuError::Memory(e.to_string()));
        }
        match validation.or(internal) {
            Some(e) => Err(WebGpuError::Device(e.to_string())),
            None => Ok(()),
        }
    }
}

/// The kernel's two entry points with explicit layouts: `apply` over
/// `{re, im, report: storage; gate record: uniform at a dynamic offset}`, and
/// `probe` over `{cases: storage; probe grid: uniform}`.
async fn kernel_pipelines(device: &wgpu::Device, source: Cow<'_, str>) -> Result<Pipelines, WebGpuError> {
    let storage = |binding: u32| wgpu::BindGroupLayoutEntry {
        binding,
        visibility: wgpu::ShaderStages::COMPUTE,
        ty: wgpu::BindingType::Buffer { ty: wgpu::BufferBindingType::Storage { read_only: false }, has_dynamic_offset: false, min_binding_size: None },
        count: None,
    };
    let uniform = |binding: u32, dynamic: bool, size: u64| wgpu::BindGroupLayoutEntry {
        binding,
        visibility: wgpu::ShaderStages::COMPUTE,
        ty: wgpu::BindingType::Buffer { ty: wgpu::BufferBindingType::Uniform, has_dynamic_offset: dynamic, min_binding_size: NonZeroU64::new(size) },
        count: None,
    };
    let scopes = Scopes::push(device);
    let module = device.create_shader_module(wgpu::ShaderModuleDescriptor { label: Some("wai-quantum lanes32"), source: wgpu::ShaderSource::Wgsl(source) });
    let apply_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
        label: Some("wai-quantum apply"),
        entries: &[storage(0), storage(1), storage(2), uniform(3, true, wgsl::GATE_RECORD_BYTES as u64)],
    });
    let probe_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { label: Some("wai-quantum probe"), entries: &[storage(4), uniform(5, false, 16)] });
    let pipeline = |layout: &wgpu::BindGroupLayout, entry: &str| {
        let pl = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { label: Some("wai-quantum"), bind_group_layouts: &[Some(layout)], immediate_size: 0 });
        device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
            label: Some(entry),
            layout: Some(&pl),
            module: &module,
            entry_point: Some(entry),
            compilation_options: Default::default(),
            cache: None,
        })
    };
    let apply = pipeline(&apply_layout, "apply");
    let probe = pipeline(&probe_layout, "probe");
    scopes.close().await?;
    Ok(Pipelines { apply_layout, probe_layout, apply, probe })
}

/// Resolves when `map_async`'s callback has run. Natively it waits on its own
/// submission inside the device poll, so it is ready on the first poll; in a
/// browser the event loop runs the callback, which wakes it.
struct Mapped {
    slot: Arc<Mutex<MapSlot>>,
    #[cfg(not(target_arch = "wasm32"))]
    device: wgpu::Device,
    #[cfg(not(target_arch = "wasm32"))]
    sink: Arc<Mutex<Sink>>,
    #[cfg(not(target_arch = "wasm32"))]
    index: wgpu::SubmissionIndex,
}

#[derive(Default)]
struct MapSlot {
    result: Option<Result<(), wgpu::BufferAsyncError>>,
    waker: Option<Waker>,
}

impl Future for Mapped {
    type Output = Result<(), WebGpuError>;

    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
        #[cfg(not(target_arch = "wasm32"))]
        if let Err(e) = wait(&self.device, &self.sink, self.index.clone()) {
            return Poll::Ready(Err(e));
        }
        let mut s = self.slot.lock().unwrap_or_else(|p| p.into_inner());
        match s.result.take() {
            Some(r) => Poll::Ready(r.map_err(|e| WebGpuError::Device(e.to_string()))),
            None => {
                s.waker = Some(cx.waker().clone());
                Poll::Pending
            }
        }
    }
}

/// Waits, within [`POLL_TIMEOUT`], until the submission `index` has run and
/// its buffer maps have completed.
#[cfg(not(target_arch = "wasm32"))]
fn wait(device: &wgpu::Device, sink: &Mutex<Sink>, index: wgpu::SubmissionIndex) -> Result<(), WebGpuError> {
    guarded(sink, || device.poll(wgpu::PollType::Wait { submission_index: Some(index), timeout: Some(POLL_TIMEOUT) }))
}

/// A native device poll as a result. The GPU interface returns a timeout or a
/// wrong submission index as an error, and reports any other failed wait,
/// such as a lost device, by panicking: that panic is caught, recorded as the
/// loss of the device, which every later call reports, and returned as
/// [`WebGpuError::Device`].
#[cfg(not(target_arch = "wasm32"))]
fn guarded(sink: &Mutex<Sink>, poll: impl FnOnce() -> Result<wgpu::PollStatus, wgpu::PollError>) -> Result<(), WebGpuError> {
    match std::panic::catch_unwind(std::panic::AssertUnwindSafe(poll)) {
        Ok(Ok(_)) => Ok(()),
        Ok(Err(wgpu::PollError::Timeout)) => Err(WebGpuError::Timeout),
        Ok(Err(e)) => Err(WebGpuError::Device(e.to_string())),
        Err(payload) => {
            let message = payload.downcast_ref::<String>().cloned().or_else(|| payload.downcast_ref::<&str>().map(|m| m.to_string())).unwrap_or_else(|| "no message".into());
            let why = format!("the device poll failed: {message}");
            let mut s = lock(sink);
            if s.lost.is_none() {
                s.lost = Some(why.clone());
            }
            Err(WebGpuError::Device(why))
        }
    }
}

/// How the planes start: one basis state, or given lanes.
enum Init<'a> {
    Basis(usize),
    Lanes(&'a [Lane]),
}

/// The record, sentinel and readback layout of one probe case: `a, b, hi, lo,
/// fx_hi, fx_lo` and two pad words.
const PROBE_RECORD: u64 = 32;

fn bytes_of(words: impl IntoIterator<Item = u32>, len: usize) -> Result<Vec<u8>, WebGpuError> {
    let mut v = Vec::new();
    v.try_reserve_exact(len.saturating_mul(4)).map_err(|e| WebGpuError::Memory(e.to_string()))?;
    words.into_iter().for_each(|w| v.extend_from_slice(&w.to_le_bytes()));
    Ok(v)
}

impl WebGpuMember {
    /// The member id ([`AdapterIdentity::member_id`]).
    pub fn id(&self) -> &str {
        &self.id
    }

    /// The key a runner declares for it ([`AdapterIdentity::key`]).
    pub fn key(&self) -> &str {
        &self.key
    }

    /// [`FabricClass::Gpu`], or [`FabricClass::Cpu`] for a software adapter.
    pub fn class(&self) -> FabricClass {
        self.class
    }

    /// `"metal"`, `"vulkan"`, `"dx12"` or `"webgpu"`.
    pub fn backend(&self) -> &'static str {
        self.backend
    }

    /// What discovery found: adapter, driver, translator, granted limits. Not
    /// signed.
    pub fn detail(&self) -> &str {
        &self.detail
    }

    /// The widest circuit this member's buffers hold.
    pub fn max_qubits(&self) -> u8 {
        self.max_qubits
    }

    /// [`wgsl::implementation_id`], or `"mutant"` for a member built by
    /// [`Self::with_shader`].
    pub fn implementation(&self) -> &str {
        &self.implementation
    }

    /// What the member id hashes.
    pub fn identity(&self) -> &AdapterIdentity {
        &self.identity
    }

    /// `mul_wide` and the exact `fx` of each pair on the device:
    /// `((hi, lo), (fx_hi, fx_lo))`, which a conforming device makes equal to
    /// [`lanes::mul_wide`] and [`lanes::fxmul_wide`].
    pub async fn probe(&self, pairs: &[(i32, i32)]) -> Result<Vec<((u32, u32), (u32, u32))>, WebGpuError> {
        self.probe_with_sentinel(pairs, 0xA5A5_A5A5).await
    }

    /// [`Self::probe`] with the result words filled with `sentinel` before
    /// the dispatch, so that a slot the device never wrote shows.
    #[doc(hidden)]
    pub async fn probe_with_sentinel(&self, pairs: &[(i32, i32)], sentinel: u32) -> Result<Vec<((u32, u32), (u32, u32))>, WebGpuError> {
        self.alive()?;
        let mut out = Vec::new();
        out.try_reserve_exact(pairs.len()).map_err(|e| WebGpuError::Memory(e.to_string()))?;
        // As many cases per dispatch as one storage binding holds.
        let per_batch = usize::try_from(self.identity.max_storage_binding / PROBE_RECORD).unwrap_or(usize::MAX).clamp(1, 1 << 24);
        for batch in pairs.chunks(per_batch) {
            self.probe_batch(batch, sentinel, &mut out).await?;
        }
        Ok(out)
    }

    async fn probe_batch(&self, pairs: &[(i32, i32)], sentinel: u32, out: &mut Vec<((u32, u32), (u32, u32))>) -> Result<(), WebGpuError> {
        let count = u32::try_from(pairs.len()).map_err(|_| WebGpuError::Internal(2))?;
        let bytes = u64::from(count) * PROBE_RECORD;
        let records = bytes_of(pairs.iter().flat_map(|&(a, b)| [a as u32, b as u32, sentinel, sentinel, sentinel, sentinel, sentinel, sentinel]), pairs.len() * 8)?;
        let grid = wgsl::grid(count, wgsl::X_GROUPS_MAX);
        let grid_bytes = bytes_of([count, grid.row_stride, 0, 0], 4)?;
        let scopes = Scopes::push(&self.device);
        let cases = self.buffer("wai-quantum probe cases", bytes, wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC | wgpu::BufferUsages::COPY_DST);
        let probe_grid = self.buffer("wai-quantum probe grid", 16, wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST);
        let staging = self.buffer("wai-quantum probe staging", bytes.min(self.readback_chunk), wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST);
        self.queue.write_buffer(&cases, 0, &records);
        self.queue.write_buffer(&probe_grid, 0, &grid_bytes);
        let bind = self.device.create_bind_group(&wgpu::BindGroupDescriptor {
            label: Some("wai-quantum probe"),
            layout: &self.pipelines.probe_layout,
            entries: &[wgpu::BindGroupEntry { binding: 4, resource: cases.as_entire_binding() }, wgpu::BindGroupEntry { binding: 5, resource: probe_grid.as_entire_binding() }],
        });
        let mut encoder = self.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("wai-quantum probe") });
        {
            let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor { label: Some("wai-quantum probe"), timestamp_writes: None });
            pass.set_pipeline(&self.pipelines.probe);
            pass.set_bind_group(0, &bind, &[]);
            pass.dispatch_workgroups(grid.x, grid.y, 1);
        }
        let _index = self.queue.submit([encoder.finish()]);
        drop(records);
        scopes.close().await?;
        #[cfg(not(target_arch = "wasm32"))]
        wait(&self.device, &self.sink, _index)?;
        let start = out.len();
        let mut words: Vec<u32> = Vec::new();
        words.try_reserve_exact(pairs.len() * 8).map_err(|e| WebGpuError::Memory(e.to_string()))?;
        self.read(&cases, bytes, &staging, |_, chunk| words.extend(chunk.chunks_exact(4).map(|w| u32::from_le_bytes([w[0], w[1], w[2], w[3]])))).await?;
        out.extend(words.chunks_exact(8).map(|r| ((r[2], r[3]), (r[4], r[5]))));
        if out.len() - start != pairs.len() {
            return Err(WebGpuError::Device("the probe's readback is short".into()));
        }
        self.settled()
    }

    /// The kernel on any state of `2^n` lanes (`n ≥ 1`) and any words, each
    /// gate's index its position: what [`lanes::run_words`] gives, with words
    /// it calls malformed refused as [`WebGpuError::Internal`].
    pub async fn run_lanes(&self, state: &[Lane], gates: &[GateWords]) -> Result<LaneRun, WebGpuError> {
        let len = state.len();
        if len < 2 || !len.is_power_of_two() {
            return Err(WebGpuError::Internal(2));
        }
        let n = len.ilog2();
        if n > u32::from(self.max_qubits) {
            return Err(WebGpuError::TooWide { n_qubits: n.min(255) as u8, max: self.max_qubits });
        }
        let gates: Vec<(GateWords, u32)> = gates.iter().enumerate().map(|(k, w)| (*w, u32::try_from(k).unwrap_or(u32::MAX))).collect();
        self.run(n as u8, Init::Lanes(state), &gates).await
    }

    /// A compiled program ([`plan::compile`]): the lanes and the first op
    /// whose exact result left 32 bits, as [`plan::execute`] gives them.
    pub async fn run_program(&self, p: &Program) -> Result<LaneRun, WebGpuError> {
        let n = p.n_qubits;
        if n == 0 {
            return Ok(LaneRun { amps: vec![[ONE as i32, 0]], out_of_range: None });
        }
        if n > self.max_qubits {
            return Err(WebGpuError::TooWide { n_qubits: n, max: self.max_qubits });
        }
        if n > MAX_QUBITS || p.start >= 1usize << n {
            return Err(WebGpuError::Internal(2));
        }
        let mut gates = Vec::new();
        gates.try_reserve_exact(p.steps.len()).map_err(|e| WebGpuError::Memory(e.to_string()))?;
        for step in &p.steps {
            match *step {
                Step::Gate { gate } => {
                    let g = p.gates.get(gate as usize).ok_or(WebGpuError::Internal(4))?;
                    gates.push((g.words, g.index));
                }
            }
        }
        self.run(n, Init::Basis(p.start), &gates).await
    }

    /// Equal to `c.simulate_from(start)`, or a typed refusal: what the CPU
    /// reference refuses ([`WebGpuError::Circuit`]), more qubits than the
    /// member holds, or a result that leaves 32 bits.
    pub async fn simulate_from(&self, c: &Circuit, start: usize) -> Result<StateVector, WebGpuError> {
        let p = plan::compile(c, start).map_err(WebGpuError::Circuit)?;
        if p.n_qubits > self.max_qubits {
            return Err(WebGpuError::TooWide { n_qubits: p.n_qubits, max: self.max_qubits });
        }
        let run = self.run_program(&p).await?;
        if let Some(gate) = run.out_of_range {
            return Err(WebGpuError::OutOfRange { gate });
        }
        let mut amps = Vec::new();
        amps.try_reserve_exact(run.amps.len()).map_err(|e| WebGpuError::Memory(e.to_string()))?;
        amps.extend(run.amps.iter().map(|l| Amp { re: i64::from(l[0]), im: i64::from(l[1]) }));
        Ok(StateVector { n_qubits: p.n_qubits, amps })
    }

    /// [`Self::simulate_from`] the all-zeros state.
    pub async fn simulate(&self, c: &Circuit) -> Result<StateVector, WebGpuError> {
        self.simulate_from(c, 0).await
    }

    /// The certificate of [`apply_kernel`]`(gs)` as [`Self::implementation`]
    /// on this member, over `corpus`: each circuit run from `|0…0⟩` against the
    /// native simulator. A range report inside a corpus is an error: its
    /// circuits are valid and within the horizon.
    pub async fn certify(&self, gs: Gateset, corpus: Corpus) -> Result<Certificate, WebGpuError> {
        let mut cert = Certification::new();
        for c in corpus.circuits(gs) {
            let reference = lanes::reference_output(&c).map_err(WebGpuError::Circuit)?;
            let run = self.run_program(&plan::compile(&c, 0).map_err(WebGpuError::Circuit)?).await?;
            if let Some(gate) = run.out_of_range {
                return Err(WebGpuError::OutOfRange { gate });
            }
            cert.add(&lanes::circuit_bytes(&c), &reference, &lanes::output(&run.amps));
        }
        Ok(cert.finish(apply_kernel(gs), &self.implementation, &self.id, &corpus.generator(gs)))
    }

    /// A fabric [`Member`]: present as [`Presence::Detected`], with one entry
    /// per gateset, each certified in this call on `corpus`, and no energy
    /// figure.
    pub async fn member(&self, corpus: Corpus) -> Result<Member, WebGpuError> {
        let mut kernels = Vec::new();
        for gs in Gateset::ALL {
            let certificate = self.certify(gs, corpus).await?;
            kernels.push(KernelEntry { kernel: apply_kernel(gs).into(), implementation: self.implementation.clone(), certificate, energy: None });
        }
        Ok(Member { id: self.id.clone(), class: self.class, presence: Presence::Detected, kernels, detail: self.detail.clone() })
    }

    /// The same member with another cap on workgroups along x, so that tests
    /// can force multi-row and ragged grids at small sizes.
    #[doc(hidden)]
    pub fn with_max_x(&self, max_x: u32) -> WebGpuMember {
        WebGpuMember { max_x: max_x.max(1), ..self.clone() }
    }

    /// The same member with a ring of `records` gate records (1 to
    /// [`RING_RECORDS`]), so that tests can make every run rewrite the ring
    /// between many submissions.
    #[doc(hidden)]
    pub fn with_ring_records(&self, records: u32) -> WebGpuMember {
        WebGpuMember { ring_records: records.clamp(1, RING_RECORDS), ..self.clone() }
    }

    /// The same member reading back `bytes` per copy (a multiple of 4, from 4
    /// to [`READBACK_CHUNK`]), so that tests can read a state in many pieces.
    #[doc(hidden)]
    pub fn with_readback_chunk(&self, bytes: u64) -> WebGpuMember {
        WebGpuMember { readback_chunk: (bytes & !3).clamp(4, READBACK_CHUNK), ..self.clone() }
    }

    /// The same device running another kernel text, as implementation
    /// `"mutant"`: for showing that a device running an edited kernel is
    /// caught. Fails if the text does not compile.
    #[doc(hidden)]
    pub async fn with_shader(&self, source: &str) -> Result<WebGpuMember, WebGpuError> {
        self.alive()?;
        let pipelines = kernel_pipelines(&self.device, Cow::Owned(source.to_string())).await?;
        Ok(WebGpuMember { pipelines, implementation: "mutant".into(), ..self.clone() })
    }

    /// An error if the device has been lost.
    fn alive(&self) -> Result<(), WebGpuError> {
        match &lock(&self.sink).lost {
            Some(why) => Err(WebGpuError::Device(why.clone())),
            None => Ok(()),
        }
    }

    /// An error if the device has been lost, or reported anything outside a
    /// scope since the last check (which the error then carries away).
    fn settled(&self) -> Result<(), WebGpuError> {
        let mut s = lock(&self.sink);
        if let Some(why) = &s.lost {
            return Err(WebGpuError::Device(why.clone()));
        }
        if s.messages.is_empty() {
            return Ok(());
        }
        Err(WebGpuError::Device(std::mem::take(&mut s.messages).join("; ")))
    }

    fn buffer(&self, label: &str, size: u64, usage: wgpu::BufferUsages) -> wgpu::Buffer {
        self.device.create_buffer(&wgpu::BufferDescriptor { label: Some(label), size, usage, mapped_at_creation: false })
    }

    /// Copies `bytes` of `src` through `staging`, a piece at a time, handing
    /// each piece and its byte offset to `take`.
    async fn read(&self, src: &wgpu::Buffer, bytes: u64, staging: &wgpu::Buffer, mut take: impl FnMut(u64, &[u8])) -> Result<(), WebGpuError> {
        let piece = staging.size();
        let mut offset = 0;
        while offset < bytes {
            let len = (bytes - offset).min(piece);
            let scopes = Scopes::push(&self.device);
            let mut encoder = self.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("wai-quantum readback") });
            encoder.copy_buffer_to_buffer(src, offset, staging, 0, len);
            let _index = self.queue.submit([encoder.finish()]);
            let slot = Arc::new(Mutex::new(MapSlot::default()));
            let filled = Arc::clone(&slot);
            staging.map_async(wgpu::MapMode::Read, 0..len, move |r| {
                let waker = {
                    let mut s = filled.lock().unwrap_or_else(|p| p.into_inner());
                    s.result = Some(r);
                    s.waker.take()
                };
                if let Some(w) = waker {
                    w.wake();
                }
            });
            scopes.close().await?;
            Mapped {
                slot,
                #[cfg(not(target_arch = "wasm32"))]
                device: self.device.clone(),
                #[cfg(not(target_arch = "wasm32"))]
                sink: Arc::clone(&self.sink),
                #[cfg(not(target_arch = "wasm32"))]
                index: _index,
            }
            .await?;
            {
                let view = staging.get_mapped_range(0..len).map_err(|e| WebGpuError::Device(e.to_string()))?;
                take(offset, &view);
            }
            staging.unmap();
            offset += len;
        }
        Ok(())
    }

    /// Runs `gates` (words and op-log index) over `2^n` lanes from `init`:
    /// one dispatch per gate, in place, in order.
    async fn run(&self, n: u8, init: Init<'_>, gates: &[(GateWords, u32)]) -> Result<LaneRun, WebGpuError> {
        self.alive()?;
        let len = 1usize << n;
        let plane = 4u64 << n;
        let grid = wgsl::grid(1u32 << (n - 1), self.max_x);
        let ring_size = u64::from(self.ring_records) * self.stride;

        // Buffers and the starting state.
        let scopes = Scopes::push(&self.device);
        let state_usage = wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC | wgpu::BufferUsages::COPY_DST;
        let re = self.buffer("wai-quantum re", plane, state_usage);
        let im = self.buffer("wai-quantum im", plane, state_usage);
        let report = self.buffer("wai-quantum report", 8, state_usage);
        let ring = self.buffer("wai-quantum gates", ring_size, wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST);
        let staging = self.buffer("wai-quantum staging", plane.min(self.readback_chunk), wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST);
        let report_staging = self.buffer("wai-quantum report staging", 8, wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST);
        let upload = match init {
            Init::Basis(start) => {
                // Four bytes at `start`; WebGPU zero-fills the rest.
                self.queue.write_buffer(&re, 4 * start as u64, &(ONE as u32).to_le_bytes());
                Ok(())
            }
            Init::Lanes(state) => bytes_of(state.iter().map(|l| l[0] as u32), len).and_then(|r| {
                self.queue.write_buffer(&re, 0, &r);
                drop(r);
                let i = bytes_of(state.iter().map(|l| l[1] as u32), len)?;
                self.queue.write_buffer(&im, 0, &i);
                Ok(())
            }),
        };
        self.queue.write_buffer(&report, 0, &[0, 0, 0, 0, 0xff, 0xff, 0xff, 0xff]);
        let bind = self.device.create_bind_group(&wgpu::BindGroupDescriptor {
            label: Some("wai-quantum apply"),
            layout: &self.pipelines.apply_layout,
            entries: &[
                wgpu::BindGroupEntry { binding: 0, resource: re.as_entire_binding() },
                wgpu::BindGroupEntry { binding: 1, resource: im.as_entire_binding() },
                wgpu::BindGroupEntry { binding: 2, resource: report.as_entire_binding() },
                wgpu::BindGroupEntry {
                    binding: 3,
                    resource: wgpu::BindingResource::Buffer(wgpu::BufferBinding { buffer: &ring, offset: 0, size: NonZeroU64::new(wgsl::GATE_RECORD_BYTES as u64) }),
                },
            ],
        });
        scopes.close().await?;
        upload?;

        // The gates, a chunk per submission, each waited on natively before
        // the next, so that POLL_TIMEOUT bounds one chunk's GPU time.
        let per_submit = (SUBMIT_BUDGET >> (n - 1)).clamp(1, u64::from(self.ring_records)) as usize;
        let stride = self.stride as usize;
        for chunk in gates.chunks(per_submit) {
            let mut records = vec![0u8; chunk.len() * stride];
            for (k, (words, index)) in chunk.iter().enumerate() {
                records[k * stride..k * stride + wgsl::GATE_RECORD_BYTES].copy_from_slice(&wgsl::gate_record(words, grid.items, grid.row_stride, *index));
            }
            let scopes = Scopes::push(&self.device);
            // Ordered on the queue after the submissions that read the ring before.
            self.queue.write_buffer(&ring, 0, &records);
            let mut encoder = self.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("wai-quantum gates") });
            {
                let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor { label: Some("wai-quantum gates"), timestamp_writes: None });
                pass.set_pipeline(&self.pipelines.apply);
                for k in 0..chunk.len() {
                    pass.set_bind_group(0, &bind, &[(k * stride) as u32]);
                    pass.dispatch_workgroups(grid.x, grid.y, 1);
                }
            }
            let _index = self.queue.submit([encoder.finish()]);
            scopes.close().await?;
            #[cfg(not(target_arch = "wasm32"))]
            wait(&self.device, &self.sink, _index)?;
        }

        // The report: a host defect ends the run here.
        let mut words = [0u32; 2];
        self.read(&report, 8, &report_staging, |_, b| {
            for (k, w) in b.chunks_exact(4).take(2).enumerate() {
                words[k] = u32::from_le_bytes([w[0], w[1], w[2], w[3]]);
            }
        })
        .await?;
        self.settled()?;
        if words[0] & 6 != 0 {
            return Err(WebGpuError::Internal(words[0]));
        }
        // The planes.
        let mut amps: Vec<Lane> = Vec::new();
        amps.try_reserve_exact(len).map_err(|e| WebGpuError::Memory(e.to_string()))?;
        amps.resize(len, [0, 0]);
        for (part, plane_buffer) in [(0, &re), (1, &im)] {
            self.read(plane_buffer, plane, &staging, |offset, b| {
                let first = (offset / 4) as usize;
                for (k, w) in b.chunks_exact(4).enumerate() {
                    amps[first + k][part] = u32::from_le_bytes([w[0], w[1], w[2], w[3]]) as i32;
                }
            })
            .await?;
        }
        self.settled()?;
        let out_of_range = (words[0] & 1 != 0).then_some(words[1]);
        Ok(LaneRun { amps, out_of_range })
    }
}

/// The member's calls for a native caller without an async runtime.
#[cfg(not(target_arch = "wasm32"))]
pub mod blocking {
    use super::{Discovery, WebGpuError, WebGpuMember};
    use crate::quantum::{Circuit, Gateset, StateVector};
    use crate::quantum_fabric::{Certificate, Corpus, Member};
    use core::future::Future;
    use core::task::{Context, Poll, Waker};

    /// Polls `f` once with a waker that does nothing. Natively every future
    /// of the GPU interface is already resolved and buffer maps complete
    /// inside the device poll, so one poll suffices; anything else is
    /// [`WebGpuError::Pending`], never a spin.
    pub fn resolve<F: Future>(f: F) -> Result<F::Output, WebGpuError> {
        let mut f = core::pin::pin!(f);
        match f.as_mut().poll(&mut Context::from_waker(Waker::noop())) {
            Poll::Ready(v) => Ok(v),
            Poll::Pending => Err(WebGpuError::Pending),
        }
    }

    /// [`super::discover`].
    pub fn discover() -> Result<Discovery, WebGpuError> {
        resolve(super::discover())
    }

    /// [`WebGpuMember::simulate_from`].
    pub fn simulate_from(m: &WebGpuMember, c: &Circuit, start: usize) -> Result<StateVector, WebGpuError> {
        resolve(m.simulate_from(c, start))?
    }

    /// [`WebGpuMember::certify`].
    pub fn certify(m: &WebGpuMember, gs: Gateset, corpus: Corpus) -> Result<Certificate, WebGpuError> {
        resolve(m.certify(gs, corpus))?
    }

    /// [`WebGpuMember::member`].
    pub fn member(m: &WebGpuMember, corpus: Corpus) -> Result<Member, WebGpuError> {
        resolve(m.member(corpus))?
    }
}

#[cfg(all(test, not(target_arch = "wasm32")))]
mod tests {
    use super::*;

    /// The GPU interface reports a failed native wait, such as a lost device,
    /// by panicking inside its device poll. The member returns it as a typed
    /// error and treats the device as lost from then on; a timeout or a wrong
    /// submission index stays what it is and loses nothing.
    #[test]
    fn a_failed_native_wait_is_a_typed_loss() {
        let sink = Mutex::new(Sink::default());
        assert_eq!(guarded(&sink, || Ok(wgpu::PollStatus::WaitSucceeded)), Ok(()));
        assert_eq!(guarded(&sink, || Err(wgpu::PollError::Timeout)), Err(WebGpuError::Timeout));
        assert!(matches!(guarded(&sink, || Err(wgpu::PollError::WrongSubmissionIndex(2, 1))), Err(WebGpuError::Device(_))));
        assert!(lock(&sink).lost.is_none());
        // A formatted message, as the interface's fatal-error path panics with.
        let r = guarded(&sink, || panic!("Error in {}: {}", "Device::poll", "the device was lost"));
        let Err(WebGpuError::Device(why)) = &r else { panic!("{r:?}") };
        assert_eq!(why, "the device poll failed: Error in Device::poll: the device was lost");
        assert_eq!(lock(&sink).lost.as_deref(), Some(why.as_str()));
        // A literal message too; the first loss is the one kept.
        assert_eq!(guarded(&sink, || panic!("again")), Err(WebGpuError::Device("the device poll failed: again".into())));
        assert_eq!(lock(&sink).lost.as_deref(), Some(why.as_str()));
    }
}