mock_upcloud/

self_check.rs

//! **The calibration: prove the instrument can report the opposite, for every
//! fault it claims to have.**
//!
//! A mock is an instrument, and an instrument nobody has calibrated is a source
//! of greens rather than a source of measurements. The failure this module
//! exists to make impossible is the one this estate committed nine times in one
//! day: a check that could only ever say "clean", run, seen to say "clean", and
//! believed. Behaviour 38 is the same defect written down as a fault —
//! `Estate::detach` had no path that answered `200` without removing the
//! device, so every sweep driven against this mock passed a test the mock could
//! not fail.
//!
//! So: for every variant in [`Fault::ALL`], drive the mock twice — once with
//! the fault DISARMED and once with it ARMED — and compare the two
//! observations.
//!
//! * They differ → the fault is **expressible**. The instrument can report both
//!   the healthy world and the faulty one, and a green from a caller driven
//!   against it means something.
//! * They are identical → the fault is **inexpressible**. The variant exists,
//!   the wire name parses, `--arm` accepts it, and nothing downstream changes.
//!   That is a fault that cannot be provoked, and [`self_check`] NAMES it and
//!   FAILS. It does not skip it: a fault the mock cannot express is worse than
//!   a fault it does not have, because the first one advertises a coverage it
//!   does not possess.
//! * Identical AND [`Fault::seeded_rate_per_mille`] is 0 → reported as a
//!   **hypothesis**: a rate of 0 is this crate's own mark for "asked for by
//!   name, never arrives as weather, not a measurement of the provider"
//!   ([`Fault::WithholdCreatedField`], [`Fault::CloneSyncsLikeImport`]). It is not a
//!   failure — but it is LISTED, with its observation printed, because absent
//!   capability is an answer and never an empty value.
//!
//! Every probe runs IN PROCESS against an [`Estate`] or a [`Mock`]: no socket,
//! no child process, nothing shelled out. The two runs are separate estates
//! built from the same seed, so the only difference between them is the fault.
//! The base is [`Faults::quiet`] rather than [`Faults::none`] on purpose —
//! `none()` arms [`Fault::StaleVncPort`] and [`Fault::UdpInboundDropped`]
//! because they are the provider's normal, and a control that already had the
//! fault armed would compare a thing with itself.

use crate::estate::{BootOrder, Estate, HostKeyPath, Label};
use crate::{Clock, Fault, Faults, Mock};
use serde_json::{json, Value};
use std::sync::Arc;

/// The seed every probe's estate is built from. One number, so a `--self-check`
/// that disagrees between two boxes disagrees about the code and not the draw.
const SEED: u64 = 4242;

/// What one fault's two runs observed.
pub struct Row {
    pub fault: Fault,
    /// What the probe saw with the fault DISARMED.
    pub healthy: String,
    /// What the same probe saw with it ARMED.
    pub faulty: String,
}

/// What a row means.
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub enum Verdict {
    /// The two observations differ: the mock can report both worlds.
    Expressible,
    /// Identical observations, and the fault is a by-name hypothesis
    /// (`seeded_rate_per_mille() == 0`). Listed, not fatal.
    Hypothesis,
    /// Identical observations on a fault that is handed out as WEATHER. Fatal:
    /// a storm arms it, a run reports it fired, and nothing about the run was
    /// different.
    Inexpressible,
}

impl Row {
    pub fn verdict(&self) -> Verdict {
        if self.healthy != self.faulty {
            Verdict::Expressible
        } else if self.fault.seeded_rate_per_mille() == 0 {
            Verdict::Hypothesis
        } else {
            Verdict::Inexpressible
        }
    }

    /// The rows this fault prints in the `--self-check` table: a header line
    /// naming it, then the two observations under each other, because the whole
    /// point is that a reader can see they are DIFFERENT.
    pub fn line(&self) -> String {
        let mark = match self.verdict() {
            Verdict::Expressible => "expressible",
            Verdict::Hypothesis => "HYPOTHESIS",
            Verdict::Inexpressible => "INEXPRESSIBLE",
        };
        let weather = if self.fault.seeded_rate_per_mille() == 0 {
            "by name".to_string()
        } else {
            format!("{}\u{2030}", self.fault.seeded_rate_per_mille())
        };
        let sticky = if self.fault.sticky() { " sticky" } else { "" };
        format!(
            "  {:<31} {:<14} {:>8}{}\n      disarmed  {}\n      ARMED     {}",
            self.fault.name(),
            mark,
            weather,
            sticky,
            self.healthy,
            self.faulty,
        )
    }
}

/// Every fault's calibration, and whether the whole thing passed.
pub struct Report {
    pub rows: Vec<Row>,
}

impl Report {
    /// The run passes when nothing is inexpressible. A hypothesis does not fail
    /// it; being unlisted would.
    pub fn ok(&self) -> bool {
        self.inexpressible().is_empty()
    }

    pub fn inexpressible(&self) -> Vec<Fault> {
        self.pick(Verdict::Inexpressible)
    }

    pub fn hypotheses(&self) -> Vec<Fault> {
        self.pick(Verdict::Hypothesis)
    }

    pub fn expressible(&self) -> Vec<Fault> {
        self.pick(Verdict::Expressible)
    }

    fn pick(&self, v: Verdict) -> Vec<Fault> {
        self.rows.iter().filter(|r| r.verdict() == v).map(|r| r.fault).collect()
    }

    /// The whole table plus its verdict, ready to print.
    pub fn text(&self) -> String {
        let mut out = String::from("mock-upcloud --self-check — can this instrument report the OPPOSITE?\n\n");
        for r in &self.rows {
            out.push_str(&r.line());
            out.push('\n');
        }
        out.push_str(&format!(
            "\n{} faults: {} expressible, {} hypothesis (rate 0, by name), {} INEXPRESSIBLE\n",
            self.rows.len(),
            self.expressible().len(),
            self.hypotheses().len(),
            self.inexpressible().len(),
        ));
        if self.ok() {
            out.push_str("PASS — every fault that is handed out as weather changes what the mock answers.\n");
        } else {
            out.push_str("FAIL — these faults arm, parse and fire, and change NOTHING a caller can see:\n");
            for f in self.inexpressible() {
                out.push_str(&format!("  {}\n", f.name()));
            }
        }
        out
    }
}

/// **Run the calibration.** Every variant in [`Fault::ALL`], twice.
///
/// There is no way to ask for a subset: a calibration that can be narrowed is a
/// calibration somebody will narrow to the faults that pass.
pub fn self_check() -> Report {
    Report { rows: Fault::ALL.iter().map(|f| Row { fault: *f, healthy: observe(*f, false), faulty: observe(*f, true) }).collect() }
}

/// A fresh estate with exactly one fault's worth of difference.
fn estate(f: Fault, armed: bool) -> Estate {
    Estate::new(Clock::virtual_only(), faults(f, armed), SEED)
}

fn faults(f: Fault, armed: bool) -> Faults {
    // `quiet()`, not `none()`: `none()` already arms the two faults that are
    // the provider's normal, and a control with the fault already armed
    // compares a thing with itself.
    let faults = Faults::quiet();
    if armed {
        faults.arm(f);
    }
    faults
}

fn mock(f: Fault, armed: bool) -> Arc<Mock> {
    Mock::new(estate(f, armed))
}

fn site() -> Vec<Label> {
    vec![
        Label { key: "site".into(), value: "gunnar.rs".into() },
        Label { key: "role".into(), value: "twin".into() },
    ]
}

/// Create one server and run it up to `started`. The probes that are about a
/// server all start here.
fn a_server(e: &mut Estate, title: &str) -> String {
    let uuid = e
        .create_server(title, title, "2xCPU-4GB", "se-sto1", site(), &format!("{title}-boot"), 20)
        .expect("the control creates a server");
    e.run_to_quiet();
    uuid
}

fn a_volume(e: &mut Estate, title: &str, gib: u64) -> String {
    let uuid = e.create_storage(title, gib, "maxiops", "se-sto1", site()).expect("create a volume");
    e.run_to_quiet();
    uuid
}

fn status(s: u16) -> String {
    if s == 0 {
        "no reply at all (socket closed)".into()
    } else {
        s.to_string()
    }
}

/// **The one observation per fault.** Each returns a sentence; the only thing
/// that matters is that the armed sentence differs from the disarmed one, and
/// that the sentence says what a CALLER would have seen rather than what a
/// field holds.
fn observe(f: Fault, armed: bool) -> String {
    match f {
        // ── the transport ───────────────────────────────────────────────────
        Fault::PriceTransportReset => {
            let m = mock(f, armed);
            let (s, _) = crate::http::probe(&m, "GET", "/1.3/price", Value::Null);
            format!("GET /1.3/price -> {}", status(s))
        }

        // ── capacity, at both doors ─────────────────────────────────────────
        Fault::OutOfStock => {
            let m = mock(f, armed);
            let (cs, cv) = crate::http::probe(&m, "POST", "/1.3/server", a_server_body());
            let created = cv["server"]["uuid"].as_str().unwrap_or("").to_string();
            let start = if created.is_empty() {
                "never reached".to_string()
            } else {
                m.estate.lock().unwrap().run_to_quiet();
                let (ss, _) = crate::http::probe(&m, "POST", &format!("/1.3/server/{created}/stop"), json!({"stop_server": {"stop_type": "hard"}}));
                let _ = ss;
                m.estate.lock().unwrap().run_to_quiet();
                let (ss, _) = crate::http::probe(&m, "POST", &format!("/1.3/server/{created}/start"), Value::Null);
                status(ss)
            };
            format!("POST /1.3/server -> {} · poweron -> {}", status(cs), start)
        }

        // ── the credential ──────────────────────────────────────────────────
        Fault::RevokedCredential => {
            let m = mock(f, armed);
            {
                let mut e = m.estate.lock().unwrap();
                a_server(&mut e, "gunnar-front");
            }
            let (a, _) = crate::http::probe(&m, "GET", "/1.3/account", Value::Null);
            let (_, l) = crate::http::probe(&m, "GET", "/1.3/server", Value::Null);
            let rows = l["servers"]["server"].as_array().map(|a| a.len()).unwrap_or(0);
            format!("GET /1.3/account -> {} · the list shows {rows} of the 1 server that exists", status(a))
        }

        // ── the console ─────────────────────────────────────────────────────
        Fault::StaleVncPort => {
            let mut e = estate(f, armed);
            let uuid = a_server(&mut e, "gunnar-appliance");
            e.modify_server(&uuid, None, None, None, Some(true), None).expect("remote access on");
            e.stop_server(&uuid, true).expect("stop");
            e.run_to_quiet();
            e.start_server(&uuid).expect("start");
            e.run_to_quiet();
            let s = e.server(&uuid).expect("still there");
            let (_, reported) = s.console().expect("remote access is on");
            if reported == s.vnc_port {
                "after a stop/start the API reports the port the hypervisor is really on".into()
            } else {
                "after a stop/start the API reports the port from BEFORE the restart".into()
            }
        }

        // ── the boot order ──────────────────────────────────────────────────
        Fault::InstallerLoop => {
            let mut e = estate(f, armed);
            let uuid = a_server(&mut e, "gunnar-appliance");
            let medium = a_volume(&mut e, "korp-installer.iso", 1);
            e.stop_server(&uuid, true).expect("stop");
            e.run_to_quiet();
            e.attach(&uuid, &medium, "cdrom").expect("the media go on while it is stopped");
            e.modify_server(&uuid, None, Some(BootOrder::Cdrom), None, None, None).expect("cd first");
            e.start_server(&uuid).expect("start");
            e.run_to_quiet();
            match e.server(&uuid).map(|s| s.guest) {
                Some(crate::estate::Guest::Looping { rounds }) => {
                    format!("the guest is running the installer AGAIN (round {rounds}) with the CD still first")
                }
                Some(g) => format!("the guest installed once and settled as {g:?}"),
                None => "the server vanished".into(),
            }
        }

        // ── behaviour 69: which kind of medium ──────────────────────────────
        Fault::InstallerReboots => {
            let mut e = estate(f, armed);
            let uuid = a_server(&mut e, "gunnar-appliance");
            let medium = a_volume(&mut e, "korp-installer.iso", 1);
            e.stop_server(&uuid, true).expect("stop");
            e.run_to_quiet();
            e.attach(&uuid, &medium, "cdrom").expect("the media go on while it is stopped");
            e.modify_server(&uuid, None, Some(BootOrder::Cdrom), None, None, None).expect("cd first");
            e.start_server(&uuid).expect("start");
            // 200 s after the start: past a power-off pass (130–146 s), well
            // inside a rebooting one (900–1100 s). Stepped, so each
            // transition chains from its own moment.
            for _ in 0..2000 {
                e.clock.advance_ms(100);
                e.settle();
            }
            match e.server(&uuid).map(|s| s.state.clone()) {
                Some(st) => format!("200 s after an installer start the server reads `{st}`"),
                None => "the server vanished".into(),
            }
        }

        // ── the field that is sent ──────────────────────────────────────────
        Fault::WithholdCreatedField => {
            let m = mock(f, armed);
            {
                let mut e = m.estate.lock().unwrap();
                a_volume(&mut e, "twin-data", 44);
            }
            let (_, l) = crate::http::probe(&m, "GET", "/1.3/storage/private", Value::Null);
            let row = &l["storages"]["storage"][0];
            if row["created"].is_null() {
                "the storage row carries NO `created` — a young volume and a six-month orphan are the same row".into()
            } else {
                "the storage row carries `created`, as the account does".into()
            }
        }

        // ── the ordinary wobble ─────────────────────────────────────────────
        Fault::WriteUnavailable => {
            let m = mock(f, armed);
            let (s, _) = crate::http::probe(&m, "POST", "/1.3/storage", json!({"storage": {"title": "twin-data", "size": 44, "tier": "maxiops", "zone": "se-sto1"}}));
            format!("POST /1.3/storage -> {}", status(s))
        }

        // ── what the resize leaves behind ───────────────────────────────────
        Fault::OrphanResizeBackup => {
            let (_, backup, e) = a_resize(f, armed);
            let origin = backup.clone().and_then(|b| e.storage(&b).and_then(|s| s.origin.clone()));
            match origin {
                None => "no backup was minted at all".into(),
                Some(o) if e.storage(&o).is_some() => "the backup's `origin` resolves to the volume it was taken from".into(),
                Some(_) => "the backup's `origin` names a uuid that resolves to NOTHING".into(),
            }
        }

        Fault::ResizeBackupUnlabelled => {
            let (_, backup, e) = a_resize(f, armed);
            match backup.and_then(|b| e.storage(&b).map(|s| s.labels.len())) {
                None => "no backup was minted at all".into(),
                Some(0) => "the backup carries NO labels — only its `origin` ties it to the estate".into(),
                Some(n) => format!("the backup carries the volume's {n} labels"),
            }
        }

        // ── the reply that never arrives ────────────────────────────────────
        Fault::CommitThenDropReply => {
            let m = mock(f, armed);
            let (s, _) = crate::http::probe(&m, "POST", "/1.3/server", a_server_body());
            let committed = m.estate.lock().unwrap().all_servers().count();
            format!("POST /1.3/server -> {} · {committed} server committed at the provider", status(s))
        }

        // ── the two clocks of an import ─────────────────────────────────────
        Fault::ImportFailed => {
            let (uuid, e, _) = an_import(f, armed);
            let st = e.storage(&uuid).expect("the volume");
            let im = st.import.as_ref().map(|i| i.state.clone()).unwrap_or_default();
            format!("after the bytes arrived: storage `{}`, import `{im}`", st.state)
        }

        Fault::SyncExceedsBudget => {
            let (uuid, e, ms) = an_import(f, armed);
            let st = e.storage(&uuid).expect("the volume");
            // The caller's budget is 1200 s. Reported as over or under it
            // rather than as a raw number, because the number is drawn.
            let verdict = if ms > 1_200_000 { "OVER the caller's 1200 s budget" } else { "inside the caller's 1200 s budget" };
            format!("the volume reached `{}` {} s after the upload — {verdict}", st.state, ms / 1000)
        }

        // ── the candidate fix ───────────────────────────────────────────────
        Fault::CloneSyncsLikeImport => {
            let mut e = estate(f, armed);
            let src = a_volume(&mut e, "korp-installer.iso", 1);
            let new = e.clone_storage(&src, "korp-installer.iso (clone)").expect("clone");
            let mut saw_syncing = false;
            for _ in 0..2000 {
                e.tick();
                match e.storage(&new).map(|s| s.state.as_str()) {
                    Some("syncing") => saw_syncing = true,
                    Some("online") => break,
                    _ => {}
                }
            }
            if saw_syncing {
                "the clone waits in `syncing`, exactly as an import does".into()
            } else {
                "the clone goes straight to `online` and never enters `syncing`".into()
            }
        }

        // ── the guest's own software ────────────────────────────────────────
        Fault::GuestReadsRtcAsLocalTime => {
            let mut e = estate(f, armed);
            let uuid = a_server(&mut e, "gunnar-appliance");
            // 2026-09-20, the day the pair was measured. CEST.
            let t = crate::guest_clock::days_from_civil(2026, 9, 20) * 86_400;
            let skew = e.server(&uuid).expect("the server").clock_skew_ms(t);
            format!("the guest's wall clock is {skew} ms from the truth")
        }

        Fault::UdpInboundDropped => {
            // No estate needed: this one is about a packet, and the packet has
            // no state.
            let t = 1_789_000_000i64;
            match crate::guest_clock::ntp_answer(&faults(f, armed), t) {
                Some(_) => "an NTP query is answered".into(),
                None => "an NTP query is never answered — silence, not an error".into(),
            }
        }

        Fault::GuestIgnoresDhcpOption121 => {
            let mut e = estate(f, armed);
            let uuid = a_server(&mut e, "gunnar-appliance");
            // **The destination has to be a box that is really listening, in
            // the OTHER /22.** A made-up far address proves nothing here: it is
            // in neither utility prefix, so the routing branch does not apply
            // and both runs answer `connection refused` — which is how the
            // first draft of this probe reported `guest-ignores-dhcp-option-121`
            // as INEXPRESSIBLE when the fault works perfectly. The calibration
            // caught its own probe, which is the argument for having it.
            //
            // The pool is shuffled per lay, so the second server is not
            // guaranteed to land in the other prefix: create until one does.
            let own = e.server(&uuid).map(|s| s.utility_ip.clone()).unwrap_or_default();
            let own_net = crate::net::net_of(&own);
            let mut dest = String::new();
            for n in 0..24 {
                let peer = a_server(&mut e, &format!("gunnar-front-{n}"));
                let ip = e.server(&peer).map(|s| s.utility_ip.clone()).unwrap_or_default();
                if crate::net::net_of(&ip) != own_net {
                    dest = ip;
                    break;
                }
            }
            assert!(!dest.is_empty(), "the address pool must span both utility prefixes");
            let reach = e.reach(&uuid, &dest, 443).expect("the server is there");
            format!("outbound to a live box in the other utility /22: {}", reach.why())
        }

        // ── the name ────────────────────────────────────────────────────────
        Fault::HijackedName => {
            let mut e = estate(f, armed);
            let uuid = a_server(&mut e, "gunnar-appliance");
            let name = e.host_key_via(&uuid, HostKeyPath::Name).expect("a key on the name");
            let direct = e.host_key_via(&uuid, HostKeyPath::Direct).expect("a key on the direct path");
            if name == direct {
                "the name and the direct path answer the SAME host key".into()
            } else {
                "the name answers a DIFFERENT host key from the direct path".into()
            }
        }

        // ── the account contradicting itself ────────────────────────────────
        Fault::FirewallForbidden => {
            let m = mock(f, armed);
            let uuid = {
                let mut e = m.estate.lock().unwrap();
                a_server(&mut e, "gunnar-front")
            };
            let (d, _) = crate::http::probe(&m, "GET", &format!("/1.3/server/{uuid}"), Value::Null);
            let (fw, _) = crate::http::probe(&m, "GET", &format!("/1.3/server/{uuid}/firewall_rule"), Value::Null);
            format!("a LIVE server: detail -> {} · firewall_rule -> {}", status(d), status(fw))
        }

        Fault::DetailNotFoundForListedServer => {
            let m = mock(f, armed);
            let uuid = {
                let mut e = m.estate.lock().unwrap();
                a_server(&mut e, "gunnar-front")
            };
            let (_, l) = crate::http::probe(&m, "GET", "/1.3/server", Value::Null);
            let rows = l["servers"]["server"].as_array().map(|a| a.len()).unwrap_or(0);
            let (d, _) = crate::http::probe(&m, "GET", &format!("/1.3/server/{uuid}"), Value::Null);
            format!("the list shows {rows} server · its own detail -> {}", status(d))
        }

        // ── the write that lies about itself ────────────────────────────────
        Fault::DetachSaysSuccessButStaysAttached => {
            let mut e = estate(f, armed);
            let uuid = a_server(&mut e, "gunnar-twin");
            let member = a_volume(&mut e, "twin-data", 44);
            e.attach(&uuid, &member, "disk").expect("hot-plug the member");
            e.stop_server(&uuid, true).expect("stop");
            e.run_to_quiet();
            let before = e.server(&uuid).map(|s| s.devices.len()).unwrap_or(0);
            let answer = match e.detach(&uuid, "virtio:1") {
                Ok(()) => "200".to_string(),
                Err(x) => format!("{} {}", x.status, x.code),
            };
            let after = e.server(&uuid).map(|s| s.devices.len()).unwrap_or(0);
            let still = if after < before { "and the device is GONE" } else { "and the device is STILL ATTACHED" };
            format!("detach -> {answer} {still} on the read-back")
        }

        // ── lane T13 ────────────────────────────────────────────────────────
        Fault::DeadToken => {
            let m = mock(f, armed);
            let (a, _) = crate::http::probe(&m, "GET", "/1.3/account", Value::Null);
            let (l, _) = crate::http::probe(&m, "GET", "/1.3/server", Value::Null);
            format!("GET /1.3/account -> {} · GET /1.3/server -> {}", status(a), status(l))
        }
        Fault::ReadBadGateway => {
            let m = mock(f, armed);
            let (l, _) = crate::http::probe(&m, "GET", "/1.3/server", Value::Null);
            format!("GET /1.3/server -> {}", status(l))
        }
        Fault::GuestKernelLacksHotplug => {
            let mut e = estate(f, armed);
            let uuid = a_server(&mut e, "gunnar-appliance");
            let member = a_volume(&mut e, "member", 10);
            match e.attach(&uuid, &member, "disk") {
                Ok(a) => format!("a virtio attach on the running server -> 200 at {a}"),
                Err(x) => format!("a virtio attach on the running server -> {} {}", x.status, x.code),
            }
        }
        Fault::ResizeRequiresDetach => {
            let mut e = estate(f, armed);
            let uuid = a_server(&mut e, "gunnar-twin");
            let member = a_volume(&mut e, "twin-data", 20);
            e.attach(&uuid, &member, "disk").expect("attach");
            e.stop_server(&uuid, true).expect("stop");
            e.run_to_quiet();
            match e.modify_storage(&member, Some(30), None) {
                Ok(()) => "a grow of a volume attached to a STOPPED server -> 200".into(),
                Err(x) => format!("a grow of a volume attached to a STOPPED server -> {} {}", x.status, x.code),
            }
        }
    }
}

/// The body a create sends. One place, so the two runs of a probe cannot differ
/// by a typo in a plan name.
fn a_server_body() -> Value {
    json!({"server": {
        "zone": "se-sto1", "title": "gunnar-front", "hostname": "gunnar-front", "plan": "2xCPU-4GB",
        "storage_devices": {"storage_device": [{"action": "create", "title": "boot", "size": 20, "tier": "maxiops"}]}
    }})
}

/// Grow a volume and resize its filesystem, which is the door behaviour 34
/// found. Returns the volume, the backup it left behind, and the estate to read
/// them out of.
fn a_resize(f: Fault, armed: bool) -> (String, Option<String>, Estate) {
    let mut e = estate(f, armed);
    let vol = a_volume(&mut e, "twin-data", 44);
    let backup = e.resize_filesystem(&vol).ok();
    e.run_to_quiet();
    (vol, backup, e)
}

/// Open a direct upload, PUT some bytes, and run the two clocks out. Returns
/// the volume, the estate, and how long after the upload the volume settled.
fn an_import(f: Fault, armed: bool) -> (String, Estate, u64) {
    let mut e = estate(f, armed);
    let uuid = a_volume(&mut e, "korp-installer.iso", 1);
    e.start_import(&uuid, "direct_upload").expect("open the session");
    // Small on purpose: the upload's own clock is not what either of these two
    // faults is about, and a 43 MB buffer in a self-check is 43 MB of nothing.
    e.upload(&uuid, &[0u8; 4096]).expect("the bytes arrive");
    let t0 = e.clock.now_ms();
    e.run_to_quiet();
    let ms = e.clock.now_ms().saturating_sub(t0);
    (uuid, e, ms)
}

#[cfg(test)]
mod tests {
    use super::*;

    /// **The calibration passes, and it covers everything.** A row per variant,
    /// no variant skipped, and nothing inexpressible.
    #[test]
    fn every_fault_can_be_reported_both_ways() {
        let r = self_check();
        assert_eq!(r.rows.len(), Fault::ALL.len(), "one row per fault, always");
        for row in &r.rows {
            assert_ne!(row.healthy, "", "{} observed nothing disarmed", row.fault.name());
            assert_ne!(row.faulty, "", "{} observed nothing armed", row.fault.name());
        }
        assert!(
            r.ok(),
            "these faults change nothing a caller can see: {:?}\n{}",
            r.inexpressible().iter().map(|f| f.name()).collect::<Vec<_>>(),
            r.text()
        );
    }

    /// **The calibration can FAIL.** The control for the control: a row whose
    /// two observations are identical is reported as a failure when the fault
    /// is weather, and as a listed hypothesis when it is by-name. Without this,
    /// `self_check` would be exactly the kind of check it exists to forbid —
    /// one nobody has seen say no.
    #[test]
    fn a_fault_that_changes_nothing_is_named_and_fails() {
        // `StaleVncPort` is weather (1000‰). Identical observations on it are
        // fatal.
        let weather = Row {
            fault: Fault::StaleVncPort,
            healthy: "the same thing".into(),
            faulty: "the same thing".into(),
        };
        assert_eq!(weather.verdict(), Verdict::Inexpressible);
        let r = Report { rows: vec![weather] };
        assert!(!r.ok());
        assert_eq!(r.inexpressible(), vec![Fault::StaleVncPort]);
        assert!(r.text().contains("FAIL"), "{}", r.text());
        assert!(r.text().contains("stale-vnc-port"), "the failure NAMES it: {}", r.text());

        // A rate-0 fault is a hypothesis: listed, printed, not fatal.
        let by_name = Row {
            fault: Fault::CloneSyncsLikeImport,
            healthy: "the same thing".into(),
            faulty: "the same thing".into(),
        };
        assert_eq!(by_name.verdict(), Verdict::Hypothesis);
        let r = Report { rows: vec![by_name] };
        assert!(r.ok(), "a hypothesis does not fail the run");
        assert_eq!(r.hypotheses(), vec![Fault::CloneSyncsLikeImport]);
        assert!(r.text().contains("clone-syncs-like-import"), "but it is LISTED: {}", r.text());
    }

    /// The table names every fault, so a fault cannot pass by being absent from
    /// the output a person reads.
    #[test]
    fn the_printed_table_names_every_fault() {
        let t = self_check().text();
        for f in Fault::ALL {
            assert!(t.contains(f.name()), "{} is missing from the table:\n{t}", f.name());
        }
    }
}