wai-quantum 0.6.0

//! The fabric's npu-cores member: the gate's fixed-point arithmetic on one
//! compute core of a neural processor's tile array, programmed directly
//! rather than through a model runtime. See
//! [`quantum_fabric::npu_cores`](crate::quantum_fabric::npu_cores) for the
//! kernel, the line layout and the argument that it is bit-exact.
//!
//! - **Discovery** ([`discover`]): the device is found in the kernel's
//!   accelerator class (`/sys/class/accel`), by the driver this member's
//!   runtime drives. Its program is built with the tile array's open
//!   toolchain, whose Python the host names (`WAI_QUANTUM_NPU_CORES_PYTHON`),
//!   into a cache directory keyed by the implementation id and the toolchain
//!   (`WAI_QUANTUM_NPU_CORES_CACHE`, else the system's temporary directory).
//!   The host side ([`npu_cores::HOST`]) is compiled with the system's C++
//!   compiler (`CXX`, else `c++`) against the device runtime and loaded. The
//!   device's arithmetic is then probed against [`lanes`] in this process.
//!   A host with no such device finds no member and says why
//!   ([`Discovery::unavailable`]); a device that cannot be built for or
//!   disagrees is refused with the reason.
//! - **Running**: the host gathers the pairs a gate selects, the core forms
//!   their four rows a dispatch of [`npu_cores::CASES`] pairs at a time, and
//!   the host checks each row's range and keeps its low 32 bits.
//! - **Certification, placement and receipts** are as for the other members;
//!   every certificate's provenance is silicon.
//!
//! # Honest boundaries
//!
//! - **One core.** The program uses one compute core of one column; the array
//!   has many. Placement over the columns is not here yet.
//! - **Unmetered**, as every member is by default.

use crate::quantum::{Amp, Circuit, Gateset, MAX_QUBITS, ONE, QuantumError, StateVector};
use crate::quantum_fabric::lanes::{self, GateWords, Lane};
use crate::quantum_fabric::npu_cores::{self, DeviceIdentity, Variant};
use crate::quantum_fabric::plan::{self, Config, LaneRun, Program, Step};
use crate::quantum_fabric::{Certificate, Certification, Corpus, FabricClass, KernelEntry, Member, MemberError, Presence, Provenance, RunFuture, RunMeter, Runner, apply_kernel};
use std::ffi::{CString, c_char, c_int, c_void};
use std::path::{Path, PathBuf};
use std::process::Command;
use std::sync::{Arc, Mutex, OnceLock};

/// One probe case: [`lanes::mul_wide`] and [`lanes::fxmul_wide`] as
/// `(high, low)` words.
pub type ProbeOutput = ((u32, u32), (u32, u32));

/// Why a device is not a member, or a run has no state.
#[derive(Debug, PartialEq, Eq)]
#[non_exhaustive]
pub enum NpuCoresError {
    /// No device, no toolchain, or no runtime, and why.
    Unavailable(String),
    /// The program or the host side did not build: the tool's output.
    Build(String),
    /// The runtime reported an error.
    Device(String),
    /// Exactly what the CPU reference refuses.
    Circuit(QuantumError),
    /// More qubits than this member holds.
    TooWide { n_qubits: u8, max: u8 },
    /// The reference's result leaves 32 bits at this op; the member returns
    /// no state.
    OutOfRange { gate: u32 },
    /// A program the host cannot lay out: a defect of the host, never a result.
    Internal(&'static str),
    /// The device's arithmetic disagreed with the CPU's on this pair.
    Probe { a: i32, b: i32 },
}

impl std::fmt::Display for NpuCoresError {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            NpuCoresError::Unavailable(why) => write!(f, "no tile array to drive: {why}"),
            NpuCoresError::Build(why) => write!(f, "the build failed: {why}"),
            NpuCoresError::Device(why) => write!(f, "device: {why}"),
            NpuCoresError::Circuit(e) => write!(f, "{e}"),
            NpuCoresError::TooWide { n_qubits, max } => write!(f, "{n_qubits} qubits; this member holds at most {max}"),
            NpuCoresError::OutOfRange { gate } => write!(f, "the result of op {gate} leaves 32 bits"),
            NpuCoresError::Internal(why) => write!(f, "host defect: {why}"),
            NpuCoresError::Probe { a, b } => write!(f, "the device's arithmetic differs from the CPU's at ({a}, {b})"),
        }
    }
}

impl std::error::Error for NpuCoresError {}

impl From<NpuCoresError> for MemberError {
    fn from(e: NpuCoresError) -> MemberError {
        match e {
            NpuCoresError::TooWide { n_qubits, max } => MemberError::TooWide { n_qubits, max },
            NpuCoresError::OutOfRange { gate } => MemberError::OutOfRange { gate },
            NpuCoresError::Device(why) => MemberError::Device(why),
            NpuCoresError::Circuit(e) => MemberError::Circuit(e),
            other => MemberError::Other(other.to_string()),
        }
    }
}

/// The driver whose devices this member's runtime drives.
const DRIVER: &str = "amdxdna"; // allow-name: the kernel driver's name, as sysfs gives it

/// A device in the accelerator class, as sysfs describes it.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Found {
    pub path: PathBuf,
    pub vendor: u16,
    pub device: u16,
    pub revision: u8,
    pub firmware: String,
}

/// The accelerator-class devices under `root` (normally `/sys/class/accel`)
/// whose driver is the one this member's runtime drives.
pub fn scan(root: &Path) -> Vec<Found> {
    let read = |p: PathBuf| std::fs::read_to_string(p).map(|s| s.trim().to_string()).unwrap_or_default();
    let num = |s: String| u32::from_str_radix(s.trim_start_matches("0x"), 16).ok();
    let mut out = Vec::new();
    let Ok(entries) = std::fs::read_dir(root) else { return out };
    let mut names: Vec<PathBuf> = entries.filter_map(|e| e.ok().map(|e| e.path())).collect();
    names.sort();
    for p in names {
        let dev = p.join("device");
        let driver = std::fs::read_link(dev.join("driver")).ok().and_then(|l| l.file_name().map(|n| n.to_string_lossy().into_owned())).unwrap_or_default();
        if driver != DRIVER {
            continue;
        }
        let (Some(vendor), Some(device)) = (num(read(dev.join("vendor"))), num(read(dev.join("device")))) else { continue };
        out.push(Found {
            path: p.clone(),
            vendor: vendor as u16,
            device: device as u16,
            revision: num(read(dev.join("revision"))).unwrap_or(0) as u8,
            firmware: read(dev.join("fw_version")),
        });
    }
    out
}

/// The host side's entry points.
struct Host {
    name: unsafe extern "C" fn(*mut c_char, usize, *mut c_char, usize) -> c_int,
    open: unsafe extern "C" fn(*const c_char, *const c_char, *const c_char, *mut *mut c_void, *mut c_char, usize) -> c_int,
    run: unsafe extern "C" fn(*mut c_void, *const i32, usize, *mut i32, usize, *mut c_char, usize) -> c_int,
    close: unsafe extern "C" fn(*mut c_void),
}

#[cfg(target_os = "linux")]
mod dl {
    use std::ffi::{c_char, c_int, c_void};
    #[link(name = "dl")]
    unsafe extern "C" {
        pub fn dlopen(filename: *const c_char, flag: c_int) -> *mut c_void;
        pub fn dlsym(handle: *mut c_void, symbol: *const c_char) -> *mut c_void;
        pub fn dlerror() -> *const c_char;
    }
    pub const RTLD_NOW: c_int = 2;
}

/// `p` as the function pointer type `F`, which the caller names from the
/// symbol's declaration.
///
/// # Safety
///
/// `p` must be a symbol of exactly the signature `F` is.
#[cfg(target_os = "linux")]
unsafe fn as_fn<F: Copy>(p: *mut c_void) -> F {
    assert_eq!(std::mem::size_of::<F>(), std::mem::size_of::<*mut c_void>(), "a function pointer");
    // SAFETY: the sizes agree, and the caller names the symbol's signature.
    unsafe { std::mem::transmute_copy(&p) }
}

#[cfg(target_os = "linux")]
fn load_host(lib: &Path) -> Result<Host, String> {
    let c = CString::new(lib.as_os_str().as_encoded_bytes()).map_err(|_| "a path holding a NUL".to_string())?;
    // SAFETY: dlopen and dlsym with valid C strings; each symbol is cast to the
    // signature the host side declares.
    unsafe {
        let h = dl::dlopen(c.as_ptr(), dl::RTLD_NOW);
        if h.is_null() {
            let e = dl::dlerror();
            return Err(if e.is_null() { format!("{}: cannot load", lib.display()) } else { std::ffi::CStr::from_ptr(e).to_string_lossy().into_owned() });
        }
        let sym = |name: &str| -> Result<*mut c_void, String> {
            let n = CString::new(name).expect("no NUL");
            let p = dl::dlsym(h, n.as_ptr());
            if p.is_null() { Err(format!("the host side has no {name}")) } else { Ok(p) }
        };
        Ok(Host { name: as_fn(sym("wq_npu_name")?), open: as_fn(sym("wq_npu_open")?), run: as_fn(sym("wq_npu_run")?), close: as_fn(sym("wq_npu_close")?) })
    }
}

#[cfg(not(target_os = "linux"))]
fn load_host(_lib: &Path) -> Result<Host, String> {
    Err(format!("the device runtime is loaded on Linux; this is {}", std::env::consts::OS))
}

/// The system's C++ compiler and its version line.
fn cxx() -> Result<(String, String), String> {
    let cxx = std::env::var("CXX").unwrap_or_else(|_| "c++".into());
    let out = Command::new(&cxx).arg("--version").output().map_err(|e| format!("no C++ compiler {cxx}: {e}"))?;
    Ok((cxx, String::from_utf8_lossy(&out.stdout).lines().next().unwrap_or("").to_string()))
}

/// The toolchain's Python, and its versions as it reports them.
fn toolchain() -> Result<(PathBuf, String), String> {
    let py = std::env::var_os("WAI_QUANTUM_NPU_CORES_PYTHON").filter(|s| !s.is_empty()).map(PathBuf::from).ok_or("no toolchain named (WAI_QUANTUM_NPU_CORES_PYTHON, the Python of an environment with the tile array's toolchain)")?;
    let dir = cache_base().join("design");
    std::fs::create_dir_all(&dir).map_err(|e| format!("{}: {e}", dir.display()))?;
    let script = dir.join(format!("design-{}.py", hex(&blake3::hash(npu_cores::DESIGN.as_bytes()).as_bytes()[..6])));
    if !script.exists() {
        std::fs::write(&script, npu_cores::DESIGN).map_err(|e| format!("{}: {e}", script.display()))?;
    }
    let out = Command::new(&py).arg(&script).arg("--versions").output().map_err(|e| format!("{}: {e}", py.display()))?;
    let versions = String::from_utf8_lossy(&out.stdout).trim().to_string();
    if !out.status.success() || versions.is_empty() || versions.contains("absent") {
        return Err(format!("{} does not have the toolchain: {versions} {}", py.display(), String::from_utf8_lossy(&out.stderr).trim()));
    }
    Ok((py, versions))
}

fn hex(bytes: &[u8]) -> String {
    bytes.iter().map(|b| format!("{b:02x}")).collect()
}

fn cache_base() -> PathBuf {
    std::env::var_os("WAI_QUANTUM_NPU_CORES_CACHE").map(PathBuf::from).unwrap_or_else(|| std::env::temp_dir().join("wai-quantum-npu-cores"))
}

/// The cache directory for a variant's program built by a toolchain for a
/// generation.
fn program_dir(v: Variant, generation: &str, toolchain: &str) -> PathBuf {
    let mut o = npu_cores::implementation_id_of(v).into_bytes();
    o.push(0);
    o.extend_from_slice(generation.as_bytes());
    o.push(0);
    o.extend_from_slice(toolchain.as_bytes());
    cache_base().join(format!("program-{}-{}", v.label(), hex(&blake3::hash(&o).as_bytes()[..8])))
}

/// Builds into a staging directory beside `dir` with `build`, then moves it
/// into place, so that a concurrent build never sees half a directory.
fn build_into(dir: &Path, build: impl FnOnce(&Path) -> Result<(), String>) -> Result<(), String> {
    if dir.exists() {
        return Ok(());
    }
    let staging = dir.with_extension(format!("building-{}", std::process::id()));
    let _ = std::fs::remove_dir_all(&staging);
    std::fs::create_dir_all(&staging).map_err(|e| format!("{}: {e}", staging.display()))?;
    if let Err(e) = build(&staging) {
        let _ = std::fs::remove_dir_all(&staging);
        return Err(e);
    }
    match std::fs::rename(&staging, dir) {
        Ok(()) => Ok(()),
        // Another process put it in place first.
        Err(_) if dir.exists() => {
            let _ = std::fs::remove_dir_all(&staging);
            Ok(())
        }
        Err(e) => Err(format!("{} into {}: {e}", staging.display(), dir.display())),
    }
}

/// The variant's program (`final.xclbin`, `insts.bin`), from the cache or
/// built into it.
fn program(py: &Path, toolchain: &str, generation: &str, v: Variant) -> Result<PathBuf, NpuCoresError> {
    let dir = program_dir(v, generation, toolchain);
    let (rnd, sh, last_add) = v.constants();
    build_into(&dir, |s| {
        std::fs::write(s.join("npu_cores.cc"), npu_cores::KERNEL).map_err(|e| e.to_string())?;
        std::fs::write(s.join("design.py"), npu_cores::DESIGN).map_err(|e| e.to_string())?;
        let out = Command::new(py)
            .current_dir(s)
            .args(["design.py", "--kernel", "npu_cores.cc", "--device", generation, "--xclbin", "final.xclbin", "--insts", "insts.bin"])
            .args(["--rnd", &rnd.to_string(), "--sh", &sh.to_string(), "--last-add", &u8::from(last_add).to_string()])
            .args(["--line", &npu_cores::LINE.to_string(), "--lines", &npu_cores::LINES.to_string()])
            .output()
            .map_err(|e| format!("{}: {e}", py.display()))?;
        if !out.status.success() || !s.join("final.xclbin").exists() || !s.join("insts.bin").exists() {
            let mut log = String::from_utf8_lossy(&out.stderr).into_owned();
            log.push_str(&String::from_utf8_lossy(&out.stdout));
            return Err(log.lines().rev().take(20).collect::<Vec<_>>().into_iter().rev().collect::<Vec<_>>().join("\n"));
        }
        Ok(())
    })
    .map_err(NpuCoresError::Build)?;
    Ok(dir)
}

/// The host side, compiled once per process into the cache and loaded.
fn host() -> Result<&'static Host, NpuCoresError> {
    static HOST: OnceLock<Result<Host, String>> = OnceLock::new();
    HOST.get_or_init(|| {
        let (cxx, version) = cxx()?;
        let mut o = npu_cores::HOST.as_bytes().to_vec();
        o.extend_from_slice(version.as_bytes());
        let dir = cache_base().join(format!("host-{}", hex(&blake3::hash(&o).as_bytes()[..8])));
        build_into(&dir, |s| {
            std::fs::write(s.join("host.cpp"), npu_cores::HOST).map_err(|e| e.to_string())?;
            let out = Command::new(&cxx)
                .current_dir(s)
                .args(["-O2", "-shared", "-fPIC", "-std=c++17", "host.cpp", "-o", "libwqnpu.so", "-lxrt_coreutil"])
                .output()
                .map_err(|e| format!("{cxx}: {e}"))?;
            if !out.status.success() {
                return Err(format!("the host side did not compile against the device runtime: {}", String::from_utf8_lossy(&out.stderr).lines().take(12).collect::<Vec<_>>().join("\n")));
            }
            Ok(())
        })?;
        load_host(&dir.join("libwqnpu.so"))
    })
    .as_ref()
    .map_err(|e| NpuCoresError::Unavailable(e.clone()))
}

fn message(buf: &[u8]) -> String {
    let end = buf.iter().position(|&b| b == 0).unwrap_or(buf.len());
    String::from_utf8_lossy(&buf[..end]).into_owned()
}

/// A program open on the device: a hardware context over it, which the
/// runtime holds until it is dropped.
struct Context {
    host: &'static Host,
    handle: Mutex<*mut c_void>,
}

// SAFETY: the handle is only used under the mutex, and the runtime's
// objects behind it may be used from any thread one at a time.
unsafe impl Send for Context {}
// SAFETY: as above.
unsafe impl Sync for Context {}

impl Context {
    fn open(host: &'static Host, dir: &Path) -> Result<Context, NpuCoresError> {
        let c = |p: PathBuf| CString::new(p.as_os_str().as_encoded_bytes()).map_err(|_| NpuCoresError::Internal("a path holding a NUL"));
        let (xclbin, insts) = (c(dir.join("final.xclbin"))?, c(dir.join("insts.bin"))?);
        let kernel = CString::new("MLIR_AIE").expect("no NUL");
        let mut err = [0u8; 1024];
        let mut h = std::ptr::null_mut();
        // SAFETY: valid C strings and an error buffer of its stated length.
        let r = unsafe { (host.open)(xclbin.as_ptr(), insts.as_ptr(), kernel.as_ptr(), &mut h, err.as_mut_ptr().cast(), err.len()) };
        if r != 0 {
            return Err(NpuCoresError::Device(message(&err)));
        }
        Ok(Context { host, handle: Mutex::new(h) })
    }

    /// One dispatch: `input` in, [`npu_cores::OUT_WORDS`] words out.
    fn run(&self, input: &[i32]) -> Result<Vec<i32>, NpuCoresError> {
        let mut out = vec![0i32; npu_cores::OUT_WORDS];
        let mut err = [0u8; 1024];
        let h = self.handle.lock().unwrap_or_else(|p| p.into_inner());
        // SAFETY: the handle is open; the buffers hold the stated words.
        let r = unsafe { (self.host.run)(*h, input.as_ptr(), input.len(), out.as_mut_ptr(), out.len(), err.as_mut_ptr().cast(), err.len()) };
        if r != 0 {
            return Err(NpuCoresError::Device(message(&err)));
        }
        Ok(out)
    }
}

impl Drop for Context {
    fn drop(&mut self) {
        let h = *self.handle.get_mut().unwrap_or_else(|p| p.into_inner());
        // SAFETY: the handle came from open and is closed once.
        unsafe { (self.host.close)(h) };
    }
}

/// A device that is not a member, and why.
#[derive(Debug)]
pub struct Refusal {
    pub device: String,
    pub key: String,
    pub error: NpuCoresError,
}

/// What [`discover`] found.
#[derive(Debug)]
pub struct Discovery {
    pub found: Vec<NpuCoresMember>,
    pub refused: Vec<Refusal>,
    pub unavailable: Option<String>,
}

/// The tile arrays on this host, each built for, opened and probed in this
/// process; or why there is none.
pub fn discover() -> Discovery {
    let mut d = Discovery { found: Vec::new(), refused: Vec::new(), unavailable: None };
    let devices = scan(Path::new("/sys/class/accel"));
    if devices.is_empty() {
        d.unavailable = Some(format!("no accelerator-class device whose driver is {DRIVER}"));
        return d;
    }
    // The runtime opens one device; a second is left alone and said so.
    for (k, f) in devices.into_iter().enumerate() {
        let key = format!("npu-cores-{:04x}-{:04x}", f.vendor, f.device);
        if k > 0 {
            d.refused.push(Refusal { device: f.path.display().to_string(), key, error: NpuCoresError::Unavailable("the runtime opens the first device only".into()) });
            continue;
        }
        match NpuCoresMember::open(&f, Variant::Conforming) {
            Ok(m) => d.found.push(m),
            Err(error) => d.refused.push(Refusal { device: f.path.display().to_string(), key, error }),
        }
    }
    d
}

/// The most qubits a run holds.
pub const MAX_NPU_CORES_QUBITS: u8 = if 22 < MAX_QUBITS { 22 } else { MAX_QUBITS };

/// A neural processor's compute core as a fabric member.
#[derive(Clone)]
pub struct NpuCoresMember {
    generation: &'static str,
    python: PathBuf,
    context: Arc<Context>,
    variant: Variant,
    identity: DeviceIdentity,
    id: String,
    key: String,
    implementation: String,
    detail: String,
    max_qubits: u8,
}

impl std::fmt::Debug for NpuCoresMember {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("NpuCoresMember").field("id", &self.id).field("key", &self.key).field("implementation", &self.implementation).finish()
    }
}

impl NpuCoresMember {
    fn open(found: &Found, variant: Variant) -> Result<NpuCoresMember, NpuCoresError> {
        let generation = npu_cores::generation(found.device).ok_or_else(|| NpuCoresError::Unavailable(format!("no program is written for device {:04x}:{:04x}", found.vendor, found.device)))?;
        let (python, versions) = toolchain().map_err(NpuCoresError::Unavailable)?;
        let host = host()?;
        let mut name = [0u8; 256];
        let mut err = [0u8; 1024];
        // SAFETY: buffers of their stated lengths.
        if unsafe { (host.name)(name.as_mut_ptr().cast(), name.len(), err.as_mut_ptr().cast(), err.len()) } != 0 {
            return Err(NpuCoresError::Device(message(&err)));
        }
        let identity = DeviceIdentity { vendor: found.vendor, device: found.device, revision: found.revision, name: message(&name), firmware: found.firmware.clone(), toolchain: versions.clone() };
        let dir = program(&python, &versions, generation, variant)?;
        let context = Arc::new(Context::open(host, &dir)?);
        let member = NpuCoresMember {
            generation,
            python,
            context,
            variant,
            id: identity.member_id(),
            key: identity.key(),
            implementation: npu_cores::implementation_id_of(variant),
            detail: format!(
                "npu; device {:?} ({:04x}:{:04x} rev {:02x}), the tile array's {generation} generation, firmware {}; one compute core; toolchain {}; up to {MAX_NPU_CORES_QUBITS} qubits; silicon; unmetered",
                identity.name,
                found.vendor,
                found.device,
                found.revision,
                found.firmware,
                versions.replace('\n', ", ")
            ),
            identity,
            max_qubits: MAX_NPU_CORES_QUBITS,
        };
        if variant == Variant::Conforming {
            let pairs = lanes::probe_pairs(4096, 0x5eed);
            let out = member.probe(&pairs)?;
            if let Some((a, b)) = lanes::first_mismatch(&pairs, &out) {
                return Err(NpuCoresError::Probe { a, b });
            }
        }
        Ok(member)
    }

    /// The same device running a mutant's program: for the mutation tests.
    #[doc(hidden)]
    pub fn with_variant(&self, v: Variant) -> Result<NpuCoresMember, NpuCoresError> {
        let dir = program(&self.python, &self.identity.toolchain, self.generation, v)?;
        let context = Arc::new(Context::open(host()?, &dir)?);
        Ok(NpuCoresMember { context, variant: v, implementation: npu_cores::implementation_id_of(v), ..self.clone() })
    }

    #[doc(hidden)]
    pub fn with_max_qubits(&self, max_qubits: u8) -> NpuCoresMember {
        NpuCoresMember { max_qubits: max_qubits.min(self.max_qubits), ..self.clone() }
    }

    pub fn id(&self) -> &str {
        &self.id
    }

    pub fn key(&self) -> &str {
        &self.key
    }

    pub fn class(&self) -> FabricClass {
        FabricClass::Npu
    }

    pub fn detail(&self) -> &str {
        &self.detail
    }

    pub fn implementation(&self) -> &str {
        &self.implementation
    }

    pub fn variant(&self) -> Variant {
        self.variant
    }

    pub fn identity(&self) -> &DeviceIdentity {
        &self.identity
    }

    pub fn provenance(&self) -> Provenance {
        Provenance::Silicon
    }

    pub fn max_qubits(&self) -> u8 {
        self.max_qubits
    }

    pub fn config(&self) -> Config {
        Config::PER_GATE
    }

    /// The device's [`lanes::mul_wide`] and [`lanes::fxmul_wide`] of each pair.
    pub fn probe(&self, pairs: &[(i32, i32)]) -> Result<Vec<ProbeOutput>, NpuCoresError> {
        let mut out = Vec::with_capacity(pairs.len());
        let split = |v: i64| ((v as u64 >> 32) as u32, v as u64 as u32);
        for chunk in pairs.chunks(npu_cores::CASES) {
            let cases: Vec<[i32; 4]> = chunk.iter().map(|&(a, b)| [a, b, 0, 0]).collect();
            let got = npu_cores::unpack(&self.context.run(&npu_cores::pack(npu_cores::MODE_PROBE, &[0; 8], &cases))?);
            out.extend(got[..chunk.len()].iter().map(|v| (split(v[0]), split(v[1]))));
        }
        Ok(out)
    }

    /// The four rows of each pair, from the device.
    fn rows(&self, m: &[i32; 8], cases: &[[i32; 4]]) -> Result<Vec<[i64; 4]>, NpuCoresError> {
        let mut out = Vec::with_capacity(cases.len());
        for chunk in cases.chunks(npu_cores::CASES) {
            let got = npu_cores::unpack(&self.context.run(&npu_cores::pack(npu_cores::MODE_ROWS, m, chunk))?);
            out.extend_from_slice(&got[..chunk.len()]);
        }
        Ok(out)
    }

    /// Gates in order from `state`, as [`lanes::run_words`] runs them.
    pub fn run_lanes(&self, state: &[Lane], gates: &[GateWords]) -> Result<LaneRun, NpuCoresError> {
        let len = state.len();
        if len < 2 || !len.is_power_of_two() {
            return Err(NpuCoresError::Internal("a state that is not 2^n lanes, n at least 1"));
        }
        let n = len.trailing_zeros() as u8;
        if n > self.max_qubits {
            return Err(NpuCoresError::TooWide { n_qubits: n, max: self.max_qubits });
        }
        let planned: Vec<(&GateWords, u32)> = gates.iter().zip(0u32..).collect();
        self.execute(state.to_vec(), &planned)
    }

    fn execute(&self, mut amps: Vec<Lane>, gates: &[(&GateWords, u32)]) -> Result<LaneRun, NpuCoresError> {
        let len = amps.len();
        for (w, _) in gates {
            let (tbit, ctrl) = (w.tbit as usize, w.ctrl as usize);
            if tbit == 0 || !tbit.is_power_of_two() || tbit > len / 2 || ctrl & tbit != 0 || ctrl & !(len - 1) != 0 {
                return Err(NpuCoresError::Internal("malformed gate words for this state"));
            }
        }
        let mut report: Option<u32> = None;
        for &(w, index) in gates {
            let m: [i32; 8] = w.m.map(|x| x as i32);
            let (tbit, ctrl) = (w.tbit as usize, w.ctrl as usize);
            let mut at = Vec::new();
            let mut cases = Vec::new();
            for p in 0..len / 2 {
                let low = p & (tbit - 1);
                let i = ((p - low) << 1) | low;
                if i & ctrl != ctrl {
                    continue;
                }
                at.push(i);
                let (a0, a1) = (amps[i], amps[i | tbit]);
                cases.push([a0[0], a0[1], a1[0], a1[1]]);
            }
            let rows = self.rows(&m, &cases)?;
            for (&i, r) in at.iter().zip(&rows) {
                if r.iter().any(|&v| i32::try_from(v).is_err()) {
                    report = Some(report.map_or(index, |x| x.min(index)));
                }
                let low = |v: i64| v as u64 as u32 as i32;
                amps[i] = [low(r[0]), low(r[1])];
                amps[i | tbit] = [low(r[2]), low(r[3])];
            }
        }
        Ok(LaneRun { amps, out_of_range: report })
    }

    pub fn run_program(&self, p: &Program) -> Result<LaneRun, NpuCoresError> {
        let n = p.n_qubits;
        if n == 0 {
            return Ok(LaneRun { amps: vec![[ONE as i32, 0]], out_of_range: None });
        }
        if n > self.max_qubits {
            return Err(NpuCoresError::TooWide { n_qubits: n, max: self.max_qubits });
        }
        if p.start >= 1usize << n {
            return Err(NpuCoresError::Internal("the start is outside the state"));
        }
        let mut gates = Vec::with_capacity(p.steps.len());
        for step in &p.steps {
            let Step::Gate { gate } = *step else { return Err(NpuCoresError::Internal("a tile step; this member runs gate steps only")) };
            let g = p.gates.get(gate as usize).ok_or(NpuCoresError::Internal("a step names a gate the program does not hold"))?;
            if !g.is_well_formed() {
                return Err(NpuCoresError::Internal("malformed gate words for this state"));
            }
            gates.push((&g.words, g.index));
        }
        let mut amps = vec![[0i32, 0]; 1usize << n];
        amps[p.start] = [ONE as i32, 0];
        self.execute(amps, &gates)
    }

    pub fn simulate_from(&self, c: &Circuit, start: usize) -> Result<StateVector, NpuCoresError> {
        let p = plan::compile_with(c, start, &self.config()).map_err(NpuCoresError::Circuit)?;
        if p.n_qubits > self.max_qubits {
            return Err(NpuCoresError::TooWide { n_qubits: p.n_qubits, max: self.max_qubits });
        }
        let run = self.run_program(&p)?;
        if let Some(gate) = run.out_of_range {
            return Err(NpuCoresError::OutOfRange { gate });
        }
        Ok(StateVector { n_qubits: p.n_qubits, amps: run.amps.iter().map(|l| Amp { re: i64::from(l[0]), im: i64::from(l[1]) }).collect() })
    }

    pub fn simulate(&self, c: &Circuit) -> Result<StateVector, NpuCoresError> {
        self.simulate_from(c, 0)
    }

    /// The certificate of [`apply_kernel`]`(gs)` on this member, with its
    /// provenance.
    pub fn certify(&self, gs: Gateset, corpus: Corpus) -> Result<Certificate, NpuCoresError> {
        let mut cert = Certification::new();
        for c in corpus.circuits(gs) {
            let reference = lanes::reference_output(&c).map_err(NpuCoresError::Circuit)?;
            let run = self.run_program(&plan::compile_with(&c, 0, &self.config()).map_err(NpuCoresError::Circuit)?)?;
            if let Some(gate) = run.out_of_range {
                return Err(NpuCoresError::OutOfRange { gate });
            }
            cert.add(&lanes::circuit_bytes(&c), &reference, &lanes::output(&run.amps));
        }
        Ok(cert.finish(apply_kernel(gs), &self.implementation, &self.id, &corpus.generator(gs)).with_provenance(Provenance::Silicon))
    }

    pub fn member(&self, corpus: Corpus) -> Result<Member, NpuCoresError> {
        let mut kernels = Vec::new();
        for gs in Gateset::ALL {
            kernels.push(KernelEntry { kernel: apply_kernel(gs).into(), implementation: self.implementation.clone(), certificate: self.certify(gs, corpus)?, energy: None });
        }
        Ok(Member { id: self.id.clone(), class: FabricClass::Npu, presence: Presence::Detected, kernels, detail: self.detail.clone() })
    }
}

impl Runner for NpuCoresMember {
    fn member_id(&self) -> String {
        self.id.clone()
    }

    fn run<'r>(&'r self, implementation: &'r str, c: &'r Circuit, start: usize, _meter: &'r mut dyn RunMeter) -> RunFuture<'r> {
        Box::pin(async move {
            if implementation != self.implementation {
                return Err(MemberError::Other(format!("member {} runs implementation {}, not {implementation:?}", self.id, self.implementation)));
            }
            Ok((self.simulate_from(c, start)?, 0, None))
        })
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    /// A sysfs tree under `root`: `accel<k>` entries, each a `device`
    /// directory with the files given and a `driver` link to a directory of
    /// that name.
    /// An entry's name, its driver's name and its files.
    type Entry<'a> = (&'a str, &'a str, &'a [(&'a str, &'a str)]);

    fn tree(root: &Path, entries: &[Entry]) {
        for (name, driver, files) in entries {
            let dev = root.join("class").join(name).join("device");
            std::fs::create_dir_all(&dev).unwrap();
            let target = root.join("drivers").join(driver);
            std::fs::create_dir_all(&target).unwrap();
            #[cfg(unix)]
            std::os::unix::fs::symlink(&target, dev.join("driver")).unwrap();
            for (f, v) in *files {
                std::fs::write(dev.join(f), format!("{v}\n")).unwrap();
            }
        }
    }

    #[test]
    #[cfg(unix)]
    fn the_scan_finds_the_runtimes_devices_and_nothing_else() {
        let root = std::env::temp_dir().join(format!("wq-npu-cores-scan-{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&root);
        tree(
            &root,
            &[
                ("accel0", DRIVER, &[("vendor", "0x1022"), ("device", "0x17f0"), ("revision", "0x11"), ("fw_version", "1.1.2.65")]),
                ("accel1", "another", &[("vendor", "0x1234"), ("device", "0x0001")]),
                ("accel2", DRIVER, &[("vendor", "not hex")]),
            ],
        );
        let found = scan(&root.join("class"));
        assert_eq!(found.len(), 1, "{found:?}");
        assert_eq!((found[0].vendor, found[0].device, found[0].revision, found[0].firmware.as_str()), (0x1022, 0x17f0, 0x11, "1.1.2.65"));
        assert!(scan(&root.join("absent")).is_empty());
        let _ = std::fs::remove_dir_all(&root);
    }

    #[test]
    fn a_program_is_cached_per_variant_generation_and_toolchain() {
        let a = program_dir(Variant::Conforming, "npu2", "t 1");
        assert_ne!(a, program_dir(Variant::M1, "npu2", "t 1"));
        assert_ne!(a, program_dir(Variant::Conforming, "npu1", "t 1"));
        assert_ne!(a, program_dir(Variant::Conforming, "npu2", "t 2"));
        assert_eq!(a, program_dir(Variant::Conforming, "npu2", "t 1"));
    }

    #[test]
    fn the_refusals_that_may_fall_back_do() {
        assert!(MemberError::from(NpuCoresError::OutOfRange { gate: 3 }).falls_back());
        assert!(!MemberError::from(NpuCoresError::Probe { a: 1, b: 2 }).falls_back());
        assert!(!MemberError::from(NpuCoresError::Internal("x")).falls_back());
    }
}