wai-quantum 0.6.0

//! The fabric's neural-processor member: the gate arithmetic of
//! [`quantum_fabric::npu`](crate::quantum_fabric::npu) on the platform's neural
//! processor, through its model runtime. See that module for why every product
//! is exact.
//!
//! - **Discovery** ([`discover`]): on macOS, the limb-product model is written,
//!   compiled by the platform's model runtime and loaded for the CPU and the
//!   neural processor. The runtime's compute plan must place **every**
//!   operation of the model on the neural processor; a model the runtime would
//!   run elsewhere, even in part, is refused with the placement it reported,
//!   so no CPU result is credited to the processor. The products are then
//!   probed against [`lanes`] in this process. Elsewhere, nothing is found, and
//!   [`Discovery::unavailable`] says why.
//! - **Running** ([`NpuMember::run_program`]): the host holds the state; for
//!   each gate, every product the gate needs (sixteen a pair) goes to the
//!   processor in batches, and the host assembles each product and applies
//!   [`lanes::apply_words`]'s fixed-point arithmetic. A result that leaves 32
//!   bits is [`NpuError::OutOfRange`], never a state.
//! - **Certification, placement and receipts** are as for every member.
//!
//! # Honest boundaries
//!
//! - **The processor multiplies; the host assembles.** Every limb product and
//!   every limb sum is formed on the processor; the shifts that join the 15
//!   coefficients of a product, the rounding and the row sums are the host's.
//! - **Placement is the runtime's report.** The compute plan is the runtime's
//!   statement of where each operation runs; it is checked before the member
//!   exists, and named in its detail.
//! - **Unmetered**: the processor's energy counter is not readable unprivileged.

use crate::quantum::{Amp, Circuit, Gateset, MAX_QUBITS, ONE, QuantumError, StateVector};
use crate::quantum_fabric::lanes::{self, GateWords, Lane};
use crate::quantum_fabric::npu::{self, BATCH, LIMBS, SHIFTS};
use crate::quantum_fabric::plan::{self, Config, LaneRun, Program, Step};
use crate::quantum_fabric::{Certificate, Certification, Corpus, KernelEntry, Member, MemberError, Presence, RunFuture, RunMeter, Runner, apply_kernel};

/// One probe case: [`lanes::mul_wide`] and [`lanes::fxmul_wide`], each as
/// `(high, low)` words.
pub type ProbeOutput = ((u32, u32), (u32, u32));

/// Why the processor is not a member, or a run has no state.
#[derive(Debug, PartialEq, Eq)]
#[non_exhaustive]
pub enum NpuError {
    /// No model runtime with a neural processor on this platform, and why.
    Unavailable(String),
    /// The runtime refused or failed the model.
    Runtime(String),
    /// The runtime would run part of the model elsewhere: each operation and
    /// the device it reported.
    Placement(String),
    Circuit(QuantumError),
    TooWide { n_qubits: u8, max: u8 },
    OutOfRange { gate: u32 },
    Internal(&'static str),
    Probe { a: i32, b: i32 },
}

impl std::fmt::Display for NpuError {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            NpuError::Unavailable(why) => write!(f, "no neural processor: {why}"),
            NpuError::Runtime(e) => write!(f, "model runtime: {e}"),
            NpuError::Placement(p) => write!(f, "the runtime would not run the whole model on the neural processor: {p}"),
            NpuError::Circuit(e) => write!(f, "{e}"),
            NpuError::TooWide { n_qubits, max } => write!(f, "{n_qubits} qubits; this member holds at most {max}"),
            NpuError::OutOfRange { gate } => write!(f, "the result of op {gate} leaves 32 bits"),
            NpuError::Internal(why) => write!(f, "host defect: {why}"),
            NpuError::Probe { a, b } => write!(f, "the processor's products differ from the CPU's at ({a}, {b})"),
        }
    }
}

impl std::error::Error for NpuError {}

impl From<NpuError> for MemberError {
    fn from(e: NpuError) -> MemberError {
        match e {
            NpuError::TooWide { n_qubits, max } => MemberError::TooWide { n_qubits, max },
            NpuError::OutOfRange { gate } => MemberError::OutOfRange { gate },
            NpuError::Runtime(why) => MemberError::Device(why),
            NpuError::Circuit(e) => MemberError::Circuit(e),
            other => MemberError::Other(other.to_string()),
        }
    }
}

/// What [`discover`] found: at most one member, the platform's processor.
#[derive(Debug)]
pub struct Discovery {
    pub found: Vec<NpuMember>,
    /// The processor, when it was not a member, with why.
    pub refused: Vec<(String, NpuError)>,
    pub unavailable: Option<String>,
}

/// The class name the runtime gives the neural processor in its compute plan.
pub const NEURAL_PROCESSOR: &str = "MLNeuralEngineComputeDevice"; // allow-name: the runtime's device class name

/// The most qubits a run holds: the host holds the state, and every gate's
/// products cross to the processor, so wide runs are slow rather than refused.
pub const MAX_NPU_QUBITS: u8 = if 20 < MAX_QUBITS { 20 } else { MAX_QUBITS };

/// The sixteen products a pair needs, in the order [`apply_pair`] reads them.
fn pair_products(m: &[i32; 8], a0: Lane, a1: Lane, out: &mut Vec<(i32, i32)>) {
    let (a0r, a0i, a1r, a1i) = (a0[0], a0[1], a1[0], a1[1]);
    out.extend_from_slice(&[
        (m[0], a0r),
        (m[1], a0i),
        (m[2], a1r),
        (m[3], a1i),
        (m[0], a0i),
        (m[1], a0r),
        (m[2], a1i),
        (m[3], a1r),
        (m[4], a0r),
        (m[5], a0i),
        (m[6], a1r),
        (m[7], a1i),
        (m[4], a0i),
        (m[5], a0r),
        (m[6], a1i),
        (m[7], a1r),
    ]);
}

/// `fx` of an exact product: `(p + 2^29) >> 30`.
fn fx(p: i64) -> i64 {
    (p + (1 << 29)) >> 30
}

fn fits32(v: i64) -> bool {
    i32::try_from(v).is_ok()
}

/// A pair's two new lanes from its sixteen products, and whether a sum left
/// 32 bits: row 0 is `(m0 + i·m1)·a0 + (m2 + i·m3)·a1`, row 1 likewise with
/// `m4..m7`, as [`lanes::apply_words`] computes them.
fn apply_pair(p: &[i64]) -> (Lane, Lane, bool) {
    debug_assert!(p.len() >= 16);
    let f: [i64; 16] = std::array::from_fn(|k| fx(p[k]));
    let o0r = f[0] - f[1] + f[2] - f[3];
    let o0i = f[4] + f[5] + f[6] + f[7];
    let o1r = f[8] - f[9] + f[10] - f[11];
    let o1i = f[12] + f[13] + f[14] + f[15];
    let bad = !(fits32(o0r) && fits32(o0i) && fits32(o1r) && fits32(o1i));
    let low = |v: i64| v as u64 as u32 as i32;
    ([low(o0r), low(o0i)], [low(o1r), low(o1i)], bad)
}

#[cfg(target_os = "macos")]
mod coreml {
    use super::NpuError;
    use crate::quantum_fabric::npu::{BATCH, LIMBS, SHIFTS};
    use objc2::AnyThread;
    use objc2::rc::Retained;
    use objc2::runtime::{AnyObject, ProtocolObject};
    use objc2_core_ml::{
        MLComputePlan, MLComputeUnits, MLDictionaryFeatureProvider, MLFeatureProvider, MLFeatureValue, MLModel, MLModelConfiguration, MLMultiArray, MLMultiArrayDataType,
    };
    use objc2_foundation::{NSArray, NSDictionary, NSError, NSNumber, NSString, NSURL};
    use std::path::Path;
    use std::sync::{Arc, Condvar, Mutex};

    fn ns_error(e: &NSError) -> String {
        e.localizedDescription().to_string()
    }

    /// A runtime object handed from the thread the runtime calls back on to the
    /// thread that waits for it, or held under a lock.
    pub struct Handed<T>(pub T);

    // SAFETY: a handed object is moved across threads only through a mutex, so
    // one thread at a time uses it, and its retain count is atomic; the runtime's
    // objects may be used from any thread that way.
    unsafe impl<T> Send for Handed<T> {}

    /// A compiled, loaded model, and where its runtime places each operation.
    pub struct Loaded {
        pub model: Retained<MLModel>,
        /// `(operation, device class)` for every operation but constants.
        pub placement: Vec<(String, String)>,
    }

    fn config() -> Retained<MLModelConfiguration> {
        // SAFETY: a fresh configuration, set before any use.
        unsafe {
            let c = MLModelConfiguration::new();
            c.setComputeUnits(MLComputeUnits::CPUAndNeuralEngine);
            c
        }
    }

    /// Writes `model` under `dir`, compiles it, loads it for the CPU and the
    /// neural processor, and reads the runtime's compute plan for it.
    pub fn load(model: &[u8], dir: &Path) -> Result<Loaded, NpuError> {
        std::fs::create_dir_all(dir).map_err(|e| NpuError::Runtime(format!("{}: {e}", dir.display())))?;
        let compiled = compiled_in(model, dir)?;
        let cfg = config();
        // SAFETY: the compiled model's URL and a configuration.
        let model = unsafe { MLModel::modelWithContentsOfURL_configuration_error(&compiled, &cfg) }.map_err(|e| NpuError::Runtime(format!("loading the model: {}", ns_error(&e))))?;
        let placement = plan(&compiled, &cfg)?;
        Ok(Loaded { model, placement })
    }

    /// The compiled form of `model`, kept in `dir` (`limbs.mlmodelc`): compiled
    /// once, and loaded by later processes without compiling again. Two
    /// processes compiling at once each move their copy in; the first stays.
    fn compiled_in(model: &[u8], dir: &Path) -> Result<Retained<NSURL>, NpuError> {
        let kept = dir.join("limbs.mlmodelc");
        let url_of = |p: &Path| NSURL::fileURLWithPath(&NSString::from_str(&p.to_string_lossy()));
        if kept.is_dir() {
            return Ok(url_of(&kept));
        }
        let io = |e: std::io::Error| NpuError::Runtime(format!("{}: {e}", dir.display()));
        let src = dir.join(format!("limbs-{}.mlmodel", std::process::id()));
        std::fs::write(&src, model).map_err(io)?;
        // SAFETY: a file URL to a model description just written.
        #[allow(deprecated)] // The synchronous form: discovery is synchronous.
        let compiled = unsafe { MLModel::compileModelAtURL_error(&url_of(&src)) };
        let _ = std::fs::remove_file(&src);
        let compiled = compiled.map_err(|e| NpuError::Runtime(format!("compiling the model: {}", ns_error(&e))))?;
        let out = compiled.path().ok_or_else(|| NpuError::Runtime("the compiled model has no path".into()))?.to_string();
        let out = std::path::PathBuf::from(out);
        if std::fs::rename(&out, &kept).is_err() {
            if !kept.is_dir() {
                // Another volume: copy it in under a private name, then move.
                let part = dir.join(format!("limbs.mlmodelc.{}", std::process::id()));
                copy_dir(&out, &part).map_err(io)?;
                if std::fs::rename(&part, &kept).is_err() {
                    let _ = std::fs::remove_dir_all(&part);
                }
            }
            let _ = std::fs::remove_dir_all(&out);
        }
        if kept.is_dir() { Ok(url_of(&kept)) } else { Err(NpuError::Runtime(format!("{}: the compiled model was not kept", kept.display()))) }
    }

    fn copy_dir(from: &Path, to: &Path) -> std::io::Result<()> {
        std::fs::create_dir_all(to)?;
        for e in std::fs::read_dir(from)? {
            let e = e?;
            let t = to.join(e.file_name());
            if e.file_type()?.is_dir() { copy_dir(&e.path(), &t)? } else { std::fs::copy(e.path(), t).map(|_| ())? }
        }
        Ok(())
    }

    /// The runtime's compute plan for the compiled model: each operation's
    /// preferred device.
    fn plan(compiled: &NSURL, cfg: &MLModelConfiguration) -> Result<Vec<(String, String)>, NpuError> {
        type Slot = (Mutex<Option<Result<Handed<Retained<MLComputePlan>>, String>>>, Condvar);
        let slot: Arc<Slot> = Arc::new((Mutex::new(None), Condvar::new()));
        let s = slot.clone();
        let handler = block2::RcBlock::new(move |plan: *mut MLComputePlan, err: *mut NSError| {
            // SAFETY: the runtime hands one of the two, live for the call.
            let r = unsafe {
                match Retained::retain(plan) {
                    Some(p) => Ok(Handed(p)),
                    None => Err(err.as_ref().map_or_else(|| "no plan".to_string(), ns_error)),
                }
            };
            let (lock, cv) = &*s;
            *lock.lock().unwrap_or_else(|p| p.into_inner()) = Some(r);
            cv.notify_all();
        });
        // SAFETY: the URL of a compiled model, a configuration, and a handler
        // that only stores its arguments.
        unsafe { MLComputePlan::loadContentsOfURL_configuration_completionHandler(compiled, cfg, &handler) };
        let (lock, cv) = &*slot;
        let guard = lock.lock().unwrap_or_else(|p| p.into_inner());
        let (mut guard, timeout) = cv.wait_timeout_while(guard, std::time::Duration::from_secs(120), |r| r.is_none()).unwrap_or_else(|p| p.into_inner());
        if timeout.timed_out() {
            return Err(NpuError::Runtime("the compute plan did not arrive".into()));
        }
        let plan = guard.take().expect("set before the notify").map_err(|e| NpuError::Runtime(format!("the compute plan: {e}")))?.0;
        let mut out = Vec::new();
        // SAFETY: reads of the plan's structure, on this thread.
        unsafe {
            let structure = plan.modelStructure();
            let program = structure.program().ok_or_else(|| NpuError::Runtime("the plan has no program".into()))?;
            let functions = program.functions();
            let main = functions.objectForKey(&NSString::from_str("main")).ok_or_else(|| NpuError::Runtime("the program has no main function".into()))?;
            for op in main.block().operations().iter() {
                let name = op.operatorName().to_string();
                if name == "const" {
                    continue;
                }
                let device = match plan.computeDeviceUsageForMLProgramOperation(&op) {
                    Some(usage) => {
                        let d = usage.preferredComputeDevice();
                        let any: &AnyObject = d.as_ref();
                        any.class().name().to_string_lossy().into_owned()
                    }
                    None => "unknown".into(),
                };
                out.push((name, device));
            }
        }
        Ok(out)
    }

    /// The fp16 array `[BATCH, SHIFTS, LIMBS]` holding `bits`, written
    /// through the strides the runtime gives it (which may pad a dimension).
    fn array(bits: &[u16]) -> Result<Retained<MLMultiArray>, NpuError> {
        assert_eq!(bits.len(), BATCH * SHIFTS * LIMBS);
        let shape: Retained<NSArray<NSNumber>> = NSArray::from_retained_slice(&[NSNumber::new_usize(BATCH), NSNumber::new_usize(SHIFTS), NSNumber::new_usize(LIMBS)]);
        // SAFETY: an fp16 array of the model's input shape; every write lands
        // at an index its own strides give, inside the array.
        unsafe {
            let a = MLMultiArray::initWithShape_dataType_error(MLMultiArray::alloc(), &shape, MLMultiArrayDataType::Float16).map_err(|e| NpuError::Runtime(ns_error(&e)))?;
            let strides: Vec<usize> = a.strides().iter().map(|n| n.as_usize()).collect();
            let [s0, s1, s2] = strides[..] else { return Err(NpuError::Runtime(format!("the input has strides {strides:?}"))) };
            if s2 != 1 || s1 < LIMBS || s0 < SHIFTS * s1 {
                return Err(NpuError::Runtime(format!("the input's strides {strides:?} do not hold [{BATCH}, {SHIFTS}, {LIMBS}]")));
            }
            #[allow(deprecated)]
            let p = a.dataPointer().as_ptr().cast::<u16>();
            if (s0, s1) == (SHIFTS * LIMBS, LIMBS) {
                std::ptr::copy_nonoverlapping(bits.as_ptr(), p, BATCH * SHIFTS * LIMBS);
            } else {
                for i in 0..BATCH {
                    for s in 0..SHIFTS {
                        std::ptr::copy_nonoverlapping(bits.as_ptr().add((i * SHIFTS + s) * LIMBS), p.add(i * s0 + s * s1), LIMBS);
                    }
                }
            }
            Ok(a)
        }
    }

    /// One batch through the model: `z`, `[BATCH, SHIFTS]`, as fp16 bits.
    pub fn predict(model: &MLModel, x: &[u16], y: &[u16]) -> Result<Vec<u16>, NpuError> {
        let (ax, ay) = (array(x)?, array(y)?);
        // SAFETY: a feature provider of the model's two inputs, a prediction,
        // and a read of its fp16 output of the shape the model declares.
        unsafe {
            let fx = MLFeatureValue::featureValueWithMultiArray(&ax);
            let fy = MLFeatureValue::featureValueWithMultiArray(&ay);
            let keys = [NSString::from_str("x"), NSString::from_str("y")];
            let values: [Retained<AnyObject>; 2] = [Retained::into_super(Retained::into_super(fx)), Retained::into_super(Retained::into_super(fy))];
            let dict: Retained<NSDictionary<NSString, AnyObject>> = NSDictionary::from_retained_objects(&[&*keys[0], &*keys[1]], &values);
            let provider = MLDictionaryFeatureProvider::initWithDictionary_error(MLDictionaryFeatureProvider::alloc(), &dict).map_err(|e| NpuError::Runtime(ns_error(&e)))?;
            let out = model.predictionFromFeatures_error(ProtocolObject::from_ref(&*provider)).map_err(|e| NpuError::Runtime(format!("predicting: {}", ns_error(&e))))?;
            let z = out.featureValueForName(&NSString::from_str("z")).and_then(|v| v.multiArrayValue()).ok_or_else(|| NpuError::Runtime("no output z".into()))?;
            if z.dataType() != MLMultiArrayDataType::Float16 || z.count() as usize != BATCH * SHIFTS {
                return Err(NpuError::Runtime("the output is not the declared fp16 [batch, shifts]".into()));
            }
            // The runtime may pad each row (strides [32, 1] for 15 values); read
            // through the strides it reports, which must hold each row whole.
            let strides: Vec<usize> = z.strides().iter().map(|n| n.as_usize()).collect();
            let [row, one] = strides[..] else { return Err(NpuError::Runtime(format!("the output has strides {strides:?}"))) };
            if one != 1 || row < SHIFTS {
                return Err(NpuError::Runtime(format!("the output's strides {strides:?} do not hold a row of {SHIFTS}")));
            }
            #[allow(deprecated)]
            let p = z.dataPointer().as_ptr().cast::<u16>();
            let all = std::slice::from_raw_parts(p, (BATCH - 1) * row + SHIFTS);
            let mut out = Vec::with_capacity(BATCH * SHIFTS);
            for r in 0..BATCH {
                out.extend_from_slice(&all[r * row..r * row + SHIFTS]);
            }
            Ok(out)
        }
    }
}

/// Every neural processor on this host: on macOS, the platform's, built,
/// placed and probed in this process.
pub fn discover() -> Discovery {
    let mut d = Discovery { found: Vec::new(), refused: Vec::new(), unavailable: None };
    #[cfg(target_os = "macos")]
    {
        match open(&npu::model()) {
            Ok(m) => d.found.push(m),
            Err(e) => d.refused.push(("npu-0".into(), e)),
        }
    }
    #[cfg(not(target_os = "macos"))]
    {
        d.unavailable = Some("this platform has no neural-processor runtime this member reaches".into());
    }
    d
}

/// The member: the loaded model behind a lock (the runtime's model object is
/// used from one thread at a time), and what discovery established.
pub struct NpuMember {
    #[cfg(target_os = "macos")]
    model: std::sync::Arc<std::sync::Mutex<coreml::Handed<objc2::rc::Retained<objc2_core_ml::MLModel>>>>,
    id: String,
    key: String,
    implementation: String,
    detail: String,
    placement: Vec<(String, String)>,
    max_qubits: u8,
}


impl Clone for NpuMember {
    fn clone(&self) -> Self {
        NpuMember {
            #[cfg(target_os = "macos")]
            model: self.model.clone(),
            id: self.id.clone(),
            key: self.key.clone(),
            implementation: self.implementation.clone(),
            detail: self.detail.clone(),
            placement: self.placement.clone(),
            max_qubits: self.max_qubits,
        }
    }
}

impl std::fmt::Debug for NpuMember {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("NpuMember").field("id", &self.id).field("implementation", &self.implementation).field("placement", &self.placement).finish()
    }
}

#[cfg(target_os = "macos")]
/// Where a model's compiled form is kept: by the model's text and the
/// system's build, whose compiler made it.
fn cache_dir(model: &[u8]) -> std::path::PathBuf {
    let base = std::env::var_os("WAI_QUANTUM_NPU_CACHE").map(std::path::PathBuf::from).unwrap_or_else(|| std::env::temp_dir().join("wai-quantum-npu"));
    let os = std::process::Command::new("sysctl").args(["-n", "kern.osversion"]).output().map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string()).unwrap_or_default();
    let mut h = blake3::Hasher::new();
    h.update(model);
    h.update(os.as_bytes());
    let tag: String = h.finalize().as_bytes()[..8].iter().map(|b| format!("{b:02x}")).collect();
    base.join(tag)
}

/// The placement as text, if the runtime put every operation on the neural
/// processor; otherwise why not.
#[cfg(target_os = "macos")]
fn placed_on_the_processor(placement: &[(String, String)]) -> Result<String, NpuError> {
    let elsewhere: Vec<String> = placement.iter().filter(|(_, d)| d != NEURAL_PROCESSOR).map(|(op, d)| format!("{op} on {d}")).collect();
    if placement.is_empty() || !elsewhere.is_empty() {
        return Err(NpuError::Placement(if elsewhere.is_empty() { "no operation was reported".into() } else { elsewhere.join("; ") }));
    }
    Ok(placement.iter().map(|(op, d)| format!("{op}: {d}")).collect::<Vec<_>>().join(", "))
}

#[cfg(target_os = "macos")]
fn detail_of(placement_text: &str) -> String {
    format!(
        "npu; the platform's neural processor through its model runtime; placement by the runtime's compute plan: {placement_text}; fp16 limb products, the host assembling; up to {MAX_NPU_QUBITS} qubits; unmetered"
    )
}

/// The neural processor running `model`: loaded, its placement checked, and
/// its products probed.
#[cfg(target_os = "macos")]
fn open(model: &[u8]) -> Result<NpuMember, NpuError> {
    let loaded = coreml::load(model, &cache_dir(model))?;
    let placement_text = placed_on_the_processor(&loaded.placement)?;
    let mut o = b"wai:quantum-fabric-npu-member\x01".to_vec();
    o.extend_from_slice(std::env::consts::ARCH.as_bytes());
    o.extend_from_slice(placement_text.as_bytes());
    let hw = std::process::Command::new("sysctl").args(["-n", "machdep.cpu.brand_string"]).output().map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string()).unwrap_or_default();
    o.extend_from_slice(hw.as_bytes());
    let id = format!("local/npu-{}", blake3::hash(&o).as_bytes()[..6].iter().map(|b| format!("{b:02x}")).collect::<String>());
    let member = NpuMember {
        model: std::sync::Arc::new(std::sync::Mutex::new(coreml::Handed(loaded.model))),
        id,
        key: "npu-0".into(),
        implementation: npu::implementation_id_of(model),
        detail: detail_of(&placement_text),
        placement: loaded.placement,
        max_qubits: MAX_NPU_QUBITS,
    };
    let pairs = lanes::probe_pairs(4096, 0x5eed);
    let out = member.probe(&pairs)?;
    match lanes::first_mismatch(&pairs, &out) {
        None => Ok(member),
        Some((a, b)) => Err(NpuError::Probe { a, b }),
    }
}

impl NpuMember {
    pub fn id(&self) -> &str {
        &self.id
    }

    /// The key a runner declares: `"npu-0"`.
    pub fn key(&self) -> &str {
        &self.key
    }

    pub fn class(&self) -> crate::quantum_fabric::FabricClass {
        npu::CLASS
    }

    pub fn detail(&self) -> &str {
        &self.detail
    }

    pub fn implementation(&self) -> &str {
        &self.implementation
    }

    pub fn max_qubits(&self) -> u8 {
        self.max_qubits
    }

    /// Each operation of the model and the device the runtime placed it on.
    pub fn placement(&self) -> &[(String, String)] {
        &self.placement
    }

    /// Exact products `a·b`, every limb product and limb sum formed on the
    /// processor.
    pub fn products(&self, pairs: &[(i32, i32)]) -> Result<Vec<i64>, NpuError> {
        let mut out = Vec::with_capacity(pairs.len());
        let (mut x, mut y) = (vec![0u16; BATCH * SHIFTS * LIMBS], vec![0u16; BATCH * SHIFTS * LIMBS]);
        for chunk in pairs.chunks(BATCH) {
            npu::expand(chunk, &mut x, &mut y);
            let z = self.predict(&x, &y)?;
            for p in 0..chunk.len() {
                let mut c = [0i64; SHIFTS];
                for (s, slot) in c.iter_mut().enumerate() {
                    *slot = npu::int_of_f16(z[p * SHIFTS + s]).ok_or_else(|| NpuError::Runtime("a coefficient that is no integer of fp16's exact range".into()))?;
                }
                out.push(npu::assemble(&c));
            }
        }
        Ok(out)
    }

    #[cfg(target_os = "macos")]
    fn predict(&self, x: &[u16], y: &[u16]) -> Result<Vec<u16>, NpuError> {
        let m = self.model.lock().unwrap_or_else(|p| p.into_inner());
        coreml::predict(&m.0, x, y)
    }

    #[cfg(not(target_os = "macos"))]
    fn predict(&self, _x: &[u16], _y: &[u16]) -> Result<Vec<u16>, NpuError> {
        Err(NpuError::Unavailable("no runtime on this platform".into()))
    }

    /// The processor's [`lanes::mul_wide`] and [`lanes::fxmul_wide`] of each pair.
    pub fn probe(&self, pairs: &[(i32, i32)]) -> Result<Vec<ProbeOutput>, NpuError> {
        let products = self.products(pairs)?;
        let split = |v: i64| ((v as u64 >> 32) as u32, v as u64 as u32);
        Ok(products.iter().map(|&p| (split(p), split(fx(p)))).collect())
    }

    pub fn config(&self) -> Config {
        Config::PER_GATE
    }

    /// Gates in order from `state`, as [`lanes::run_words`] runs them.
    pub fn run_lanes(&self, state: &[Lane], gates: &[GateWords]) -> Result<LaneRun, NpuError> {
        let len = state.len();
        if len < 2 || !len.is_power_of_two() {
            return Err(NpuError::Internal("a state that is not 2^n lanes, n at least 1"));
        }
        let n = len.trailing_zeros() as u8;
        if n > self.max_qubits {
            return Err(NpuError::TooWide { n_qubits: n, max: self.max_qubits });
        }
        let planned: Vec<(&GateWords, u32)> = gates.iter().zip(0u32..).collect();
        self.execute(state.to_vec(), &planned)
    }

    fn execute(&self, mut amps: Vec<Lane>, gates: &[(&GateWords, u32)]) -> Result<LaneRun, NpuError> {
        let len = amps.len();
        for (w, _) in gates {
            let (tbit, ctrl) = (w.tbit as usize, w.ctrl as usize);
            if tbit == 0 || !tbit.is_power_of_two() || tbit > len / 2 || ctrl & tbit != 0 || ctrl & !(len - 1) != 0 {
                return Err(NpuError::Internal("malformed gate words for this state"));
            }
        }
        let mut report: Option<u32> = None;
        let mut products = Vec::new();
        let mut pairs = Vec::new();
        for &(w, index) in gates {
            let m: [i32; 8] = w.m.map(|x| x as i32);
            let (tbit, ctrl) = (w.tbit as usize, w.ctrl as usize);
            pairs.clear();
            products.clear();
            for p in 0..len / 2 {
                let low = p & (tbit - 1);
                let i = ((p - low) << 1) | low;
                if i & ctrl != ctrl {
                    continue;
                }
                pairs.push(i);
                pair_products(&m, amps[i], amps[i | tbit], &mut products);
            }
            let exact = self.products(&products)?;
            for (k, &i) in pairs.iter().enumerate() {
                let (o0, o1, bad) = apply_pair(&exact[16 * k..16 * k + 16]);
                if bad {
                    report = Some(report.map_or(index, |r| r.min(index)));
                }
                amps[i] = o0;
                amps[i | tbit] = o1;
            }
        }
        Ok(LaneRun { amps, out_of_range: report })
    }

    pub fn run_program(&self, p: &Program) -> Result<LaneRun, NpuError> {
        let n = p.n_qubits;
        if n == 0 {
            return Ok(LaneRun { amps: vec![[ONE as i32, 0]], out_of_range: None });
        }
        if n > self.max_qubits {
            return Err(NpuError::TooWide { n_qubits: n, max: self.max_qubits });
        }
        if p.start >= 1usize << n {
            return Err(NpuError::Internal("the start is outside the state"));
        }
        let mut gates = Vec::with_capacity(p.steps.len());
        for step in &p.steps {
            let Step::Gate { gate } = *step else { return Err(NpuError::Internal("a tile step; this member runs gate steps only")) };
            let g = p.gates.get(gate as usize).ok_or(NpuError::Internal("a step names a gate the program does not hold"))?;
            if !g.is_well_formed() {
                return Err(NpuError::Internal("malformed gate words for this state"));
            }
            gates.push((&g.words, g.index));
        }
        let mut amps = vec![[0i32, 0]; 1usize << n];
        amps[p.start] = [ONE as i32, 0];
        self.execute(amps, &gates)
    }

    pub fn simulate_from(&self, c: &Circuit, start: usize) -> Result<StateVector, NpuError> {
        let p = plan::compile_with(c, start, &self.config()).map_err(NpuError::Circuit)?;
        if p.n_qubits > self.max_qubits {
            return Err(NpuError::TooWide { n_qubits: p.n_qubits, max: self.max_qubits });
        }
        let run = self.run_program(&p)?;
        if let Some(gate) = run.out_of_range {
            return Err(NpuError::OutOfRange { gate });
        }
        Ok(StateVector { n_qubits: p.n_qubits, amps: run.amps.iter().map(|l| Amp { re: i64::from(l[0]), im: i64::from(l[1]) }).collect() })
    }

    pub fn simulate(&self, c: &Circuit) -> Result<StateVector, NpuError> {
        self.simulate_from(c, 0)
    }

    /// The certificate of [`apply_kernel`]`(gs)` on this member over `corpus`.
    pub fn certify(&self, gs: Gateset, corpus: Corpus) -> Result<Certificate, NpuError> {
        let mut cert = Certification::new();
        for c in corpus.circuits(gs) {
            let reference = lanes::reference_output(&c).map_err(NpuError::Circuit)?;
            let run = self.run_program(&plan::compile_with(&c, 0, &self.config()).map_err(NpuError::Circuit)?)?;
            if let Some(gate) = run.out_of_range {
                return Err(NpuError::OutOfRange { gate });
            }
            cert.add(&lanes::circuit_bytes(&c), &reference, &lanes::output(&run.amps));
        }
        Ok(cert.finish(apply_kernel(gs), &self.implementation, &self.id, &corpus.generator(gs)))
    }

    /// A fabric [`Member`] of class [`FabricClass::Npu`](crate::quantum_fabric::FabricClass::Npu).
    pub fn member(&self, corpus: Corpus) -> Result<Member, NpuError> {
        let mut kernels = Vec::new();
        for gs in Gateset::ALL {
            kernels.push(KernelEntry { kernel: apply_kernel(gs).into(), implementation: self.implementation.clone(), certificate: self.certify(gs, corpus)?, energy: None });
        }
        Ok(Member { id: self.id.clone(), class: npu::CLASS, presence: Presence::Detected, kernels, detail: self.detail.clone() })
    }

    /// The same member holding at most `max_qubits` qubits.
    #[doc(hidden)]
    pub fn with_max_qubits(&self, max_qubits: u8) -> NpuMember {
        NpuMember { max_qubits: max_qubits.min(self.max_qubits), ..self.clone() }
    }

    /// The same processor running another model text: for the mutation tests.
    /// A mutant the runtime places elsewhere is refused at load, as any model is.
    #[doc(hidden)]
    pub fn with_model(&self, model: &[u8]) -> Result<NpuMember, NpuError> {
        #[cfg(target_os = "macos")]
        {
            let loaded = coreml::load(model, &cache_dir(model))?;
            let placement_text = placed_on_the_processor(&loaded.placement)?;
            Ok(NpuMember {
                model: std::sync::Arc::new(std::sync::Mutex::new(coreml::Handed(loaded.model))),
                implementation: npu::implementation_id_of(model),
                detail: detail_of(&placement_text),
                placement: loaded.placement,
                ..self.clone()
            })
        }
        #[cfg(not(target_os = "macos"))]
        {
            let _ = model;
            Err(NpuError::Unavailable("no runtime on this platform".into()))
        }
    }
}

impl Runner for NpuMember {
    fn member_id(&self) -> String {
        self.id.clone()
    }

    fn run<'r>(&'r self, implementation: &'r str, c: &'r Circuit, start: usize, _meter: &'r mut dyn RunMeter) -> RunFuture<'r> {
        Box::pin(async move {
            if implementation != self.implementation {
                return Err(MemberError::Other(format!("member {} runs implementation {}, not {implementation:?}", self.id, self.implementation)));
            }
            let s = self.simulate_from(c, start)?;
            Ok((s, 0, None))
        })
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    /// The host's half of a gate, with the processor's products replaced by
    /// exact ones: it must be `lanes::run_words`.
    #[test]
    fn the_hosts_arithmetic_is_run_words() {
        let mut s = 0x1234_5678_u64;
        let mut next = || {
            s = s.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
            (s >> 32) as u32
        };
        for case in 0..300 {
            let n = 1 + (next() % 6) as usize;
            let state: Vec<Lane> = (0..1usize << n).map(|_| [next() as i32 >> (case % 3 * 4), next() as i32 >> (case % 3 * 4)]).collect();
            let t = next() as usize % n;
            let ctrl = (0..n).filter(|&b| b != t && next() % 3 == 0).fold(0u32, |a, b| a | 1 << b);
            let w = GateWords { m: [0; 8].map(|_: u32| next() >> (case % 2 * 2)), tbit: 1 << t, ctrl };
            let want = lanes::run_words(&state, std::slice::from_ref(&w)).unwrap();
            let mut amps = state.clone();
            let mut report = None;
            let m: [i32; 8] = w.m.map(|x| x as i32);
            for p in 0..amps.len() / 2 {
                let low = p & ((1 << t) - 1);
                let i = ((p - low) << 1) | low;
                if i & ctrl as usize != ctrl as usize {
                    continue;
                }
                let mut pr = Vec::new();
                pair_products(&m, amps[i], amps[i | 1 << t], &mut pr);
                let exact: Vec<i64> = pr.iter().map(|&(a, b)| i64::from(a) * i64::from(b)).collect();
                let (o0, o1, bad) = apply_pair(&exact);
                if bad {
                    report = Some(0);
                }
                amps[i] = o0;
                amps[i | 1 << t] = o1;
            }
            assert_eq!(report, want.out_of_range, "case {case}");
            assert_eq!(amps, want.amps, "case {case}");
        }
    }

    #[test]
    fn errors_map_onto_the_fabric() {
        assert!(MemberError::from(NpuError::OutOfRange { gate: 1 }).falls_back());
        assert!(!MemberError::from(NpuError::Placement("mul on the CPU".into())).falls_back());
    }
}