wai-quantum 0.5.0

//! The fabric's simulated FPGA member: the gate datapath of
//! [`quantum_fabric::fpga`](crate::quantum_fabric::fpga), compiled at run time
//! by Verilator into a cycle-accurate simulation and driven over a pipe. See
//! that module for the datapath and the argument that it is bit-exact.
//!
//! - **Discovery** ([`discover`]): Verilator is looked for when the process
//!   first asks (`WAI_QUANTUM_VERILATOR`, else `verilator` on the path), so a
//!   build with this feature runs on a host without it and finds no member
//!   there ([`Discovery::unavailable`] says why). The datapath and harness are
//!   compiled into a cache directory keyed by the implementation id and the
//!   simulator (`WAI_QUANTUM_FPGA_CACHE`, else the system's temporary
//!   directory), and the simulation is probed against [`lanes`] in this process
//!   before it is a member.
//! - **Running** ([`FpgaSimMember::run_program`]): the state is loaded into the
//!   datapath's RAM through its host port, each gate runs to completion, and
//!   the report word and the state come back. The datapath reports exactly when
//!   the reference's result leaves 32 bits; the member then returns
//!   [`FpgaError::OutOfRange`], never a state.
//! - **Certification, placement, receipts and metering** are as for the CUDA
//!   member, except that every certificate says it was earned in a simulator
//!   ([`SimIdentity::provenance`]), so placement ranks it after silicon.
//!
//! Every call is synchronous. The simulation is one child process per member,
//! behind a lock, ended when the last clone of the member is dropped.
//!
//! # Honest boundaries
//!
//! - **A simulation, labelled as one.** It proves what the datapath computes
//!   cycle by cycle, not what any board does; no board is claimed.
//! - **Two cycles a pair, in order.** The datapath is written to be read and
//!   checked, not to be fast; [`FpgaSimMember::cycles`] reports what a run took.
//! - **Unmetered by default**, as for every member: a meter reads the host
//!   running the simulation.

use crate::quantum::{Amp, Circuit, Gateset, MAX_QUBITS, ONE, QuantumError, StateVector};
use crate::quantum_energy::EnergyClass;
use crate::quantum_fabric::fpga::{self, SimIdentity};
use crate::quantum_fabric::lanes::{self, GateWords, Lane};
use crate::quantum_fabric::plan::{self, Config, LaneRun, Program, Step};
use crate::quantum_fabric::{
    Certificate, Certification, Corpus, EnergyCalibration, FabricClass, KernelEntry, Member, MemberError, Presence, RunFuture, RunMeter, Runner, Unmetered, Window,
    apply_kernel, pair_updates,
};
use std::io::{Read, Write};
use std::path::{Path, PathBuf};
use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio};
use std::sync::{Arc, Mutex, MutexGuard};

/// One probe case's results: [`lanes::mul_wide`] and [`lanes::fxmul_wide`],
/// each as `(high, low)` words.
pub type ProbeOutput = ((u32, u32), (u32, u32));

/// A run's lanes and report, with the meter's accepted reading (zero and
/// `None` when unmetered).
type MeteredRun = (LaneRun, (u64, Option<EnergyClass>));

/// Why the simulation is not a member, or a run has no state.
#[derive(Debug, PartialEq, Eq)]
#[non_exhaustive]
pub enum FpgaError {
    /// Verilator or a C++ compiler could not be used, and why.
    Unavailable(String),
    /// Verilator refused the datapath or the harness; its output.
    Build(String),
    /// The simulation's process failed, or broke the protocol.
    Simulator(String),
    /// Exactly what the CPU reference refuses.
    Circuit(QuantumError),
    /// More qubits than the datapath's RAM holds.
    TooWide { n_qubits: u8, max: u8 },
    /// The reference's result leaves 32 bits at this op; the member returns
    /// no state.
    OutOfRange { gate: u32 },
    /// A program the host cannot lay out: a defect of the host, never a
    /// result.
    Internal(&'static str),
    /// The datapath's arithmetic disagreed with the CPU's on this pair.
    Probe { a: i32, b: i32 },
}

impl std::fmt::Display for FpgaError {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            FpgaError::Unavailable(why) => write!(f, "the FPGA simulation is unavailable: {why}"),
            FpgaError::Build(log) => write!(f, "the datapath did not compile: {log}"),
            FpgaError::Simulator(e) => write!(f, "simulation: {e}"),
            FpgaError::Circuit(e) => write!(f, "{e}"),
            FpgaError::TooWide { n_qubits, max } => write!(f, "{n_qubits} qubits; this member holds at most {max}"),
            FpgaError::OutOfRange { gate } => write!(f, "the result of op {gate} leaves 32 bits"),
            FpgaError::Internal(why) => write!(f, "host defect: {why}"),
            FpgaError::Probe { a, b } => write!(f, "the datapath's arithmetic differs from the CPU's at ({a}, {b})"),
        }
    }
}

impl std::error::Error for FpgaError {}

impl From<FpgaError> for MemberError {
    fn from(e: FpgaError) -> MemberError {
        match e {
            FpgaError::TooWide { n_qubits, max } => MemberError::TooWide { n_qubits, max },
            FpgaError::OutOfRange { gate } => MemberError::OutOfRange { gate },
            FpgaError::Simulator(why) => MemberError::Device(why),
            FpgaError::Circuit(e) => MemberError::Circuit(e),
            other => MemberError::Other(other.to_string()),
        }
    }
}

fn io(e: std::io::Error) -> FpgaError {
    FpgaError::Simulator(e.to_string())
}

/// A simulation that is not a member, and why.
#[derive(Debug)]
pub struct Refusal {
    pub key: String,
    pub error: FpgaError,
}

/// What [`discover`] found: at most one member, the simulation.
#[derive(Debug)]
pub struct Discovery {
    pub found: Vec<FpgaSimMember>,
    pub refused: Vec<Refusal>,
    /// Why nothing was built: no Verilator, or no C++ compiler. `None` when
    /// the simulator was found.
    pub unavailable: Option<String>,
}

/// The simulator's path and version line, or why there is none.
fn verilator() -> Result<(PathBuf, String), String> {
    let path = std::env::var_os("WAI_QUANTUM_VERILATOR").map(PathBuf::from).unwrap_or_else(|| PathBuf::from("verilator"));
    let out = Command::new(&path).arg("--version").output().map_err(|e| format!("no Verilator at {}: {e}", path.display()))?;
    let line = String::from_utf8_lossy(&out.stdout).lines().next().unwrap_or("").trim().to_string();
    if !out.status.success() || !line.starts_with("Verilator ") {
        return Err(format!("{} --version did not name Verilator", path.display()));
    }
    Ok((path, line))
}

/// The C++ compiler's version line: `$CXX`, else `c++`.
fn compiler() -> Result<String, String> {
    let cxx = std::env::var("CXX").unwrap_or_else(|_| "c++".into());
    let out = Command::new(&cxx).arg("--version").output().map_err(|e| format!("no C++ compiler {cxx}: {e}"))?;
    let line = String::from_utf8_lossy(&out.stdout).lines().next().unwrap_or("").trim().to_string();
    if !out.status.success() || line.is_empty() {
        return Err(format!("{cxx} --version printed nothing"));
    }
    Ok(line)
}

/// Whether the simulator and a C++ compiler can be used, or why not.
pub fn available() -> Result<(), String> {
    verilator()?;
    compiler()?;
    Ok(())
}

/// The simulation, built and probed in this process, or why there is none.
pub fn discover() -> Discovery {
    let mut d = Discovery { found: Vec::new(), refused: Vec::new(), unavailable: None };
    let (path, simulator) = match verilator() {
        Ok(v) => v,
        Err(why) => {
            d.unavailable = Some(why);
            return d;
        }
    };
    let compiler = match compiler() {
        Ok(c) => c,
        Err(why) => {
            d.unavailable = Some(why);
            return d;
        }
    };
    let identity = SimIdentity { simulator, compiler, host: format!("{}-{}", std::env::consts::ARCH, std::env::consts::OS) };
    let key = identity.key();
    match open(&path, identity) {
        Ok(m) => d.found.push(m),
        Err(error) => d.refused.push(Refusal { key, error }),
    }
    d
}

/// The cache directory for a datapath text under a simulator.
fn cache_dir(identity: &SimIdentity, rtl: &str) -> PathBuf {
    let base = std::env::var_os("WAI_QUANTUM_FPGA_CACHE").map(PathBuf::from).unwrap_or_else(|| std::env::temp_dir().join("wai-quantum-fpga"));
    let mut h = blake3::Hasher::new();
    h.update(fpga::implementation_id_of(rtl).as_bytes());
    h.update(identity.member_id().as_bytes());
    let d = h.finalize();
    let tag: String = d.as_bytes()[..8].iter().map(|b| format!("{b:02x}")).collect();
    base.join(tag)
}

/// The simulation's executable for `rtl`: from the cache, or compiled into it.
fn build(verilator: &Path, identity: &SimIdentity, rtl: &str) -> Result<PathBuf, FpgaError> {
    let dir = cache_dir(identity, rtl);
    let exe = dir.join("obj").join("wq_sim");
    if exe.is_file() {
        return Ok(exe);
    }
    // Compile beside the cache entry, then move it into place, so that a
    // concurrent build never sees half a directory.
    let staging = dir.with_extension(format!("building-{}", std::process::id()));
    let _ = std::fs::remove_dir_all(&staging);
    std::fs::create_dir_all(&staging).map_err(|e| FpgaError::Build(format!("{}: {e}", staging.display())))?;
    std::fs::write(staging.join("lanes64.v"), rtl).map_err(|e| FpgaError::Build(e.to_string()))?;
    std::fs::write(staging.join("lanes64_harness.cpp"), fpga::HARNESS).map_err(|e| FpgaError::Build(e.to_string()))?;
    let out = Command::new(verilator)
        .current_dir(&staging)
        .args(fpga::VERILATOR_OPTIONS)
        .args(["--top-module", "wq_lanes64", "--Mdir", "obj", "-o", "wq_sim", "lanes64.v", "lanes64_harness.cpp"])
        .output()
        .map_err(|e| FpgaError::Unavailable(format!("{}: {e}", verilator.display())))?;
    if !out.status.success() {
        let log = String::from_utf8_lossy(&out.stderr);
        let _ = std::fs::remove_dir_all(&staging);
        return Err(FpgaError::Build(log.lines().filter(|l| l.contains("%Error")).take(8).collect::<Vec<_>>().join("\n")));
    }
    match std::fs::rename(&staging, &dir) {
        Ok(()) => {}
        // Another process finished first: keep its build.
        Err(_) if exe.is_file() => {
            let _ = std::fs::remove_dir_all(&staging);
        }
        Err(e) => return Err(FpgaError::Build(format!("{} into {}: {e}", staging.display(), dir.display()))),
    }
    Ok(exe)
}

/// The running simulation: its process and pipes.
struct Sim {
    child: Child,
    stdin: ChildStdin,
    stdout: ChildStdout,
    /// Cycles the last run took.
    cycles: u64,
}

impl Sim {
    fn start(exe: &Path) -> Result<Sim, FpgaError> {
        let mut child = Command::new(exe).stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null()).spawn().map_err(io)?;
        let stdin = child.stdin.take().ok_or(FpgaError::Internal("the simulation has no stdin"))?;
        let stdout = child.stdout.take().ok_or(FpgaError::Internal("the simulation has no stdout"))?;
        Ok(Sim { child, stdin, stdout, cycles: 0 })
    }

    fn read_words(&mut self, n: usize) -> Result<Vec<u32>, FpgaError> {
        let mut bytes = vec![0u8; n * 4];
        self.stdout.read_exact(&mut bytes).map_err(|e| self.died(e))?;
        Ok(bytes.as_chunks::<4>().0.iter().map(|w| u32::from_le_bytes(*w)).collect())
    }

    /// The process's exit status, when a pipe broke because it ended.
    fn died(&mut self, e: std::io::Error) -> FpgaError {
        match self.child.try_wait() {
            Ok(Some(status)) => FpgaError::Simulator(format!("the simulation ended ({status})")),
            _ => io(e),
        }
    }
}

impl Drop for Sim {
    fn drop(&mut self) {
        let _ = self.stdin.write_all(&[0]);
        let _ = self.stdin.flush();
        let _ = self.child.wait();
    }
}

fn open(verilator: &Path, identity: SimIdentity) -> Result<FpgaSimMember, FpgaError> {
    let exe = build(verilator, &identity, fpga::RTL)?;
    let sim = Sim::start(&exe)?;
    let max_qubits = fpga::ADDRESS_BITS.min(MAX_QUBITS);
    let detail = format!(
        "fpga datapath, simulated; {}; compiled by {}; on {}; RAM of 2^{} lanes, up to {max_qubits} qubits; two cycles a pair; a metered run's window covers the simulation on this host",
        identity.simulator,
        identity.compiler,
        identity.host,
        fpga::ADDRESS_BITS,
    );
    let member = FpgaSimMember {
        sim: Arc::new(Mutex::new(sim)),
        verilator: verilator.to_path_buf(),
        id: identity.member_id(),
        key: identity.key(),
        implementation: fpga::implementation_id(),
        detail,
        max_qubits,
        identity,
    };
    let pairs = lanes::probe_pairs(4096, 0x5eed);
    let out = member.probe(&pairs)?;
    match lanes::first_mismatch(&pairs, &out) {
        None => Ok(member),
        Some((a, b)) => Err(FpgaError::Probe { a, b }),
    }
}

/// The simulated datapath as a fabric member.
#[derive(Clone)]
pub struct FpgaSimMember {
    sim: Arc<Mutex<Sim>>,
    verilator: PathBuf,
    identity: SimIdentity,
    id: String,
    key: String,
    implementation: String,
    detail: String,
    max_qubits: u8,
}

impl std::fmt::Debug for FpgaSimMember {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("FpgaSimMember").field("id", &self.id).field("implementation", &self.implementation).field("max_qubits", &self.max_qubits).finish()
    }
}

impl FpgaSimMember {
    /// The member id ([`SimIdentity::member_id`]).
    pub fn id(&self) -> &str {
        &self.id
    }

    /// The key a runner declares for it ([`SimIdentity::key`]).
    pub fn key(&self) -> &str {
        &self.key
    }

    /// Always [`FabricClass::Fpga`].
    pub fn class(&self) -> FabricClass {
        FabricClass::Fpga
    }

    /// What discovery found: the simulator, its compiler, the host and the
    /// datapath's shape. Not signed.
    pub fn detail(&self) -> &str {
        &self.detail
    }

    /// The most qubits a run holds: the RAM's address width.
    pub fn max_qubits(&self) -> u8 {
        self.max_qubits
    }

    /// The implementation id ([`fpga::implementation_id`], or a mutant's).
    pub fn implementation(&self) -> &str {
        &self.implementation
    }

    /// What the member id hashes.
    pub fn identity(&self) -> &SimIdentity {
        &self.identity
    }

    /// The cycles the last run took on the datapath's clock.
    pub fn cycles(&self) -> u64 {
        self.lock().cycles
    }

    fn lock(&self) -> MutexGuard<'_, Sim> {
        self.sim.lock().unwrap_or_else(|p| p.into_inner())
    }

    /// The datapath's [`lanes::mul_wide`] and [`lanes::fxmul_wide`] of each
    /// pair, in order.
    pub fn probe(&self, pairs: &[(i32, i32)]) -> Result<Vec<ProbeOutput>, FpgaError> {
        if pairs.is_empty() {
            return Ok(Vec::new());
        }
        let mut req = Vec::with_capacity(9 + pairs.len() * 8);
        req.push(1u8);
        req.extend_from_slice(&(pairs.len() as u64).to_le_bytes());
        for &(a, b) in pairs {
            req.extend_from_slice(&a.to_le_bytes());
            req.extend_from_slice(&b.to_le_bytes());
        }
        let mut sim = self.lock();
        sim.stdin.write_all(&req).and_then(|()| sim.stdin.flush()).map_err(io)?;
        let words = sim.read_words(pairs.len() * 4)?;
        Ok(words.as_chunks::<4>().0.iter().map(|w| ((w[0], w[1]), (w[2], w[3]))).collect())
    }

    /// The planning this member runs: [`Config::PER_GATE`].
    pub fn config(&self) -> Config {
        Config::PER_GATE
    }

    /// Runs `p` from its basis state: the lanes, and the first op whose exact
    /// result left 32 bits. A program with tile steps is
    /// [`FpgaError::Internal`]: this member runs gate steps only.
    pub fn run_program(&self, p: &Program) -> Result<LaneRun, FpgaError> {
        Ok(self.run_program_metered(p, &mut Unmetered)?.0)
    }

    fn run_program_metered<M: RunMeter + ?Sized>(&self, p: &Program, meter: &mut M) -> Result<MeteredRun, FpgaError> {
        let n = p.n_qubits;
        if n == 0 {
            return Ok((LaneRun { amps: vec![[ONE as i32, 0]], out_of_range: None }, (0, None)));
        }
        if n > self.max_qubits {
            return Err(FpgaError::TooWide { n_qubits: n, max: self.max_qubits });
        }
        if n > MAX_QUBITS || p.start >= 1usize << n {
            return Err(FpgaError::Internal("the start is outside the state"));
        }
        let len = 1usize << n;
        let mut gates = Vec::with_capacity(p.steps.len());
        for step in &p.steps {
            let Step::Gate { gate } = *step else { return Err(FpgaError::Internal("a tile step; this member runs gate steps only")) };
            let g = p.gates.get(gate as usize).ok_or(FpgaError::Internal("a step names a gate the program does not hold"))?;
            if !g.is_well_formed() {
                return Err(FpgaError::Internal("malformed gate words for this state"));
            }
            gates.push((&g.words, g.index));
        }
        let mut init: Vec<Lane> = Vec::new();
        init.try_reserve_exact(len).map_err(|e| FpgaError::Simulator(e.to_string()))?;
        init.resize(len, [0, 0]);
        init[p.start] = [ONE as i32, 0];
        self.execute(&init, &gates, meter)
    }

    /// Gates in order from `state`, as [`lanes::run_words`] runs them. A state
    /// that is not `2^n` lanes (`n ≥ 1`) or words [`lanes::apply_words`]
    /// refuses for it are [`FpgaError::Internal`].
    pub fn run_lanes(&self, state: &[Lane], gates: &[GateWords]) -> Result<LaneRun, FpgaError> {
        let len = state.len();
        if len < 2 || !len.is_power_of_two() {
            return Err(FpgaError::Internal("a state that is not 2^n lanes, n at least 1"));
        }
        let n = len.trailing_zeros() as u8;
        if n > self.max_qubits {
            return Err(FpgaError::TooWide { n_qubits: n, max: self.max_qubits });
        }
        let planned: Vec<(&GateWords, u32)> = gates.iter().zip(0u32..).collect();
        Ok(self.execute(state, &planned, &mut Unmetered)?.0)
    }

    /// Loads `state`, runs `gates` in order inside a window of `meter`, and
    /// reads the report and the state back. Every gate's words are checked
    /// against the state first.
    fn execute<M: RunMeter + ?Sized>(&self, state: &[Lane], gates: &[(&GateWords, u32)], meter: &mut M) -> Result<MeteredRun, FpgaError> {
        let len = state.len();
        for (w, _) in gates {
            let (tbit, ctrl) = (w.tbit as usize, w.ctrl as usize);
            if tbit == 0 || !tbit.is_power_of_two() || tbit > len / 2 || ctrl & tbit != 0 || ctrl & !(len - 1) != 0 {
                return Err(FpgaError::Internal("malformed gate words for this state"));
            }
        }
        let n = len.trailing_zeros();
        let mut req: Vec<u8> = Vec::new();
        req.try_reserve_exact(9 + len * 8 + gates.len() * 44).map_err(|e| FpgaError::Simulator(e.to_string()))?;
        req.push(2u8);
        req.extend_from_slice(&n.to_le_bytes());
        req.extend_from_slice(&(gates.len() as u32).to_le_bytes());
        for l in state {
            req.extend_from_slice(&l[0].to_le_bytes());
            req.extend_from_slice(&l[1].to_le_bytes());
        }
        for &(w, index) in gates {
            for m in w.m {
                req.extend_from_slice(&m.to_le_bytes());
            }
            req.extend_from_slice(&w.tbit.to_le_bytes());
            req.extend_from_slice(&w.ctrl.to_le_bytes());
            req.extend_from_slice(&index.to_le_bytes());
        }
        let mut window = Window::new(meter);
        let mut sim = self.lock();
        window.open();
        sim.stdin.write_all(&req).and_then(|()| sim.stdin.flush()).map_err(io)?;
        drop(req);
        let report = sim.read_words(1)?[0];
        let words = sim.read_words(2 * len)?;
        let cycles = sim.read_words(2)?;
        let reading = window.close();
        sim.cycles = u64::from(cycles[0]) | (u64::from(cycles[1]) << 32);
        let out_of_range = Some(report).filter(|&k| k != u32::MAX);
        let amps: Vec<Lane> = words.as_chunks::<2>().0.iter().map(|w| [w[0] as i32, w[1] as i32]).collect();
        Ok((LaneRun { amps, out_of_range }, reading))
    }

    /// Equal to `c.simulate_from(start)`, or a typed refusal: what the CPU
    /// reference refuses ([`FpgaError::Circuit`]), more qubits than the member
    /// holds, or a result that leaves 32 bits.
    pub fn simulate_from(&self, c: &Circuit, start: usize) -> Result<StateVector, FpgaError> {
        Ok(self.simulate_with(c, start, &mut Unmetered)?.0)
    }

    /// [`Self::simulate_from`] inside a window of `meter`.
    pub fn simulate_metered(&self, c: &Circuit, start: usize, meter: &mut dyn RunMeter) -> Result<(StateVector, u64, Option<EnergyClass>), FpgaError> {
        self.simulate_with(c, start, meter)
    }

    fn simulate_with<M: RunMeter + ?Sized>(&self, c: &Circuit, start: usize, meter: &mut M) -> Result<(StateVector, u64, Option<EnergyClass>), FpgaError> {
        let p = plan::compile_with(c, start, &self.config()).map_err(FpgaError::Circuit)?;
        if p.n_qubits > self.max_qubits {
            return Err(FpgaError::TooWide { n_qubits: p.n_qubits, max: self.max_qubits });
        }
        let (run, (joules_micro, class)) = self.run_program_metered(&p, meter)?;
        if let Some(gate) = run.out_of_range {
            return Err(FpgaError::OutOfRange { gate });
        }
        let amps = run.amps.iter().map(|l| Amp { re: i64::from(l[0]), im: i64::from(l[1]) }).collect();
        Ok((StateVector { n_qubits: p.n_qubits, amps }, joules_micro, class))
    }

    /// [`Self::simulate_from`] the all-zeros state.
    pub fn simulate(&self, c: &Circuit) -> Result<StateVector, FpgaError> {
        self.simulate_from(c, 0)
    }

    /// The certificate of [`apply_kernel`]`(gs)` as [`Self::implementation`]
    /// on this member, over `corpus`, earned in a simulator.
    pub fn certify(&self, gs: Gateset, corpus: Corpus) -> Result<Certificate, FpgaError> {
        Ok(self.certify_metered(gs, corpus, &mut Unmetered)?.0)
    }

    fn certify_metered<M: RunMeter + ?Sized>(&self, gs: Gateset, corpus: Corpus, meter: &mut M) -> Result<(Certificate, Option<(u64, EnergyClass)>), FpgaError> {
        let mut cert = Certification::new();
        let mut calibration = EnergyCalibration::new();
        for c in corpus.circuits(gs) {
            let reference = lanes::reference_output(&c).map_err(FpgaError::Circuit)?;
            let (run, (joules_micro, class)) = self.run_program_metered(&plan::compile_with(&c, 0, &self.config()).map_err(FpgaError::Circuit)?, &mut *meter)?;
            if let Some(gate) = run.out_of_range {
                return Err(FpgaError::OutOfRange { gate });
            }
            cert.add(&lanes::circuit_bytes(&c), &reference, &lanes::output(&run.amps));
            calibration.add(pair_updates(&c), joules_micro, class.as_ref());
        }
        let certificate = cert.finish(apply_kernel(gs), &self.implementation, &self.id, &corpus.generator(gs)).with_provenance(self.identity.provenance());
        Ok((certificate, calibration.finish()))
    }

    /// A fabric [`Member`] of class [`FabricClass::Fpga`]: one entry per
    /// gateset, each certified in this call on `corpus` and earned in a
    /// simulator, and no energy figure.
    pub fn member(&self, corpus: Corpus) -> Result<Member, FpgaError> {
        self.member_with(corpus, &mut Unmetered)
    }

    /// [`Self::member`], each corpus run inside its own window of `meter`.
    pub fn member_metered(&self, corpus: Corpus, meter: &mut dyn RunMeter) -> Result<Member, FpgaError> {
        self.member_with(corpus, meter)
    }

    fn member_with<M: RunMeter + ?Sized>(&self, corpus: Corpus, meter: &mut M) -> Result<Member, FpgaError> {
        let mut kernels = Vec::new();
        for gs in Gateset::ALL {
            let (certificate, energy) = self.certify_metered(gs, corpus, &mut *meter)?;
            kernels.push(KernelEntry { kernel: apply_kernel(gs).into(), implementation: self.implementation.clone(), certificate, energy });
        }
        Ok(Member { id: self.id.clone(), class: FabricClass::Fpga, presence: Presence::Detected, kernels, detail: self.detail.clone() })
    }

    /// The same member holding at most `max_qubits` qubits.
    #[doc(hidden)]
    pub fn with_max_qubits(&self, max_qubits: u8) -> FpgaSimMember {
        FpgaSimMember { max_qubits: max_qubits.min(self.max_qubits), ..self.clone() }
    }

    /// The same simulator running `rtl` in place of the datapath, under the
    /// implementation id that text earns: for the mutation tests.
    #[doc(hidden)]
    pub fn with_rtl(&self, rtl: &str) -> Result<FpgaSimMember, FpgaError> {
        let exe = build(&self.verilator, &self.identity, rtl)?;
        let sim = Sim::start(&exe)?;
        Ok(FpgaSimMember { sim: Arc::new(Mutex::new(sim)), implementation: fpga::implementation_id_of(rtl), ..self.clone() })
    }
}

/// The member on the fabric ([`Fabric`](crate::quantum_fabric::Fabric)).
impl Runner for FpgaSimMember {
    fn member_id(&self) -> String {
        self.id.clone()
    }

    fn run<'r>(&'r self, implementation: &'r str, c: &'r Circuit, start: usize, meter: &'r mut dyn RunMeter) -> RunFuture<'r> {
        Box::pin(async move {
            if implementation != self.implementation {
                return Err(MemberError::Other(format!("member {} runs implementation {}, not {implementation:?}", self.id, self.implementation)));
            }
            Ok(self.simulate_with(c, start, meter)?)
        })
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn errors_map_onto_the_fabric() {
        assert!(MemberError::from(FpgaError::OutOfRange { gate: 3 }).falls_back());
        assert!(!MemberError::from(FpgaError::Probe { a: 1, b: 2 }).falls_back());
        assert!(!MemberError::from(FpgaError::Internal("x")).falls_back());
        assert!(matches!(MemberError::from(FpgaError::Simulator("x".into())), MemberError::Device(_)));
    }

    #[test]
    fn the_cache_keys_on_the_datapath_and_the_simulator() {
        let id = SimIdentity { simulator: "Verilator 5.052".into(), compiler: "c".into(), host: "h".into() };
        let other = SimIdentity { simulator: "Verilator 5.054".into(), ..id.clone() };
        let a = cache_dir(&id, fpga::RTL);
        assert_ne!(a, cache_dir(&other, fpga::RTL));
        assert_ne!(a, cache_dir(&id, &fpga::mutate(&fpga::MUTANTS[0])));
        assert_eq!(a, cache_dir(&id, fpga::RTL));
    }
}