zc2 0.0.40

P2P compute broker with credit-based billing, WAL, and broker mesh support
//! The machine-readable run.
//!
//! A benchmark whose only output is a drawing cannot be tracked. This is the
//! same run as the report, as one JSON document on stdout, with a schema
//! version so a chart built on it today still parses next year. The human
//! view goes to stderr when `--json` is on, so a pipe never mangles either.

use serde_json::{json, Value};

use super::metrics::{mean_us, percentile_us, stdev_us, Buckets, Placement, MAX_BUCKET_ROWS};
use super::{BenchResults, StopReason};

/// The document's shape. Bump it when a field changes meaning or goes away;
/// adding one is not a break.
pub(crate) const SCHEMA: &str = "zc.bench/1";

fn buckets_json(b: &Buckets, scale: f64, key: &str) -> Value {
    Value::Array(
        b.bounds
            .iter()
            .zip(b.counts.iter())
            .map(|(bound, n)| match *bound {
                u64::MAX => json!({ key: Value::Null, "n": n }),
                v => json!({ key: v as f64 / scale, "n": n }),
            })
            .collect(),
    )
}

fn side(results: &BenchResults, placement: Placement) -> Value {
    let (requests, charged) = results.ledger.totals(placement);
    let lat = results.ledger.latencies(placement);
    let placed = results.ledger.placed();
    json!({
        "requests": requests,
        "share": match placed { 0 => 0.0, p => requests as f64 / p as f64 },
        "p50_us": match lat.is_empty() { true => Value::Null, false => json!(percentile_us(&lat, 50.0)) },
        "charged_zkcr": charged,
    })
}

pub(crate) fn render(results: &BenchResults) -> String {
    let lat = &results.latencies_us;
    let nodes: Vec<Value> = results
        .ledger
        .rows()
        .into_iter()
        .map(|(class, node, stat)| {
            json!({
                "node": node,
                "class": class.as_str(),
                "via": stat.via(),
                "workers": stat.workers.len(),
                "requests": stat.requests,
                "p50_us": stat.p50_us(),
                "charged_zkcr": stat.charged_zkcr,
                "router_expected_ms": stat.expectation.avg_expected_ms(),
                "delivered_ms": stat.expectation.avg_delivered_ms(),
            })
        })
        .collect();

    let doc = json!({
        "schema": SCHEMA,
        "zc": env!("CARGO_PKG_VERSION"),
        "started_at": results.started_at,
        "config": {
            "workload": results.config.workload.name(),
            "strategy": results.config.strategy.as_str(),
            "concurrency": results.config.concurrency,
            "requests": results.total_requests,
            "broker": results.broker_label(),
            "remote": match results.config.remote {
                super::RemoteMode::Off => "off",
                super::RemoteMode::OffThisMachine => "off_this_machine",
                super::RemoteMode::ForcePeerDispatch => "force_peer_dispatch",
            },
            "target_node": results.config.target_node,
            "worker_type": results.config.worker_type,
        },
        "outcome": {
            "completed": results.completed,
            "ok": results.successful,
            "failed": results.failed,
            "duration_s": results.duration.as_secs_f64(),
            "stopped_early": results.stopped.is_some(),
            "stop_reason": match results.stopped {
                Some(StopReason::Interrupt) => Value::String("sigint".into()),
                Some(StopReason::Failing) => Value::String("failing".into()),
                None => Value::Null,
            },
            "failures": results
                .ranked_failure_reasons()
                .into_iter()
                .take(10)
                .map(|(reason, n)| json!({"reason": reason, "n": n}))
                .collect::<Vec<_>>(),
        },
        "throughput_rps": {
            "mean": results.rps,
            "peak": results.rates_per_second.iter().copied().max().unwrap_or(0),
            "per_second": results.rates_per_second,
            "buckets": buckets_json(&Buckets::linear(&results.rates_per_second, 7), 1.0, "le_rps"),
        },
        "latency_us": {
            "min": lat.iter().copied().min().unwrap_or(0),
            "mean": mean_us(lat),
            "stdev": stdev_us(lat),
            "p50": percentile_us(lat, 50.0),
            "p90": percentile_us(lat, 90.0),
            "p99": percentile_us(lat, 99.0),
            "max": lat.iter().copied().max().unwrap_or(0),
            "buckets": buckets_json(&Buckets::of(lat, MAX_BUCKET_ROWS), 1000.0, "le_ms"),
        },
        "placement": {
            "free": side(results, Placement::Free),
            "billed": side(results, Placement::Billed),
            "nodes": nodes,
        },
        "credits": {
            "charged_zkcr": results.total_charged_zkcr,
            "per_request_zkcr": results.avg_charge_zkcr(),
            "billed_requests": results.ledger.totals(Placement::Billed).0,
        },
    });
    serde_json::to_string_pretty(&doc).unwrap_or_else(|_| "{}".to_string())
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::broker::bench::{BenchConfig, BenchResults};

    fn a_run() -> BenchResults {
        let mut r = BenchResults::new();
        r.config = BenchConfig::default();
        r.total_requests = 10;
        r.completed = 10;
        r.successful = 9;
        r.failed = 1;
        r.duration = std::time::Duration::from_secs(2);
        r.rps = 5.0;
        r.latencies_us = vec![1000, 2000, 3000, 4000];
        r.rates_per_second = vec![4, 5];
        r.ledger
            .record("a-w0", Some("local"), Some(0.0), 1000, 0.0, Some(3.0));
        r.ledger
            .record("i9-w0", Some("quic"), Some(3.6), 4000, 0.002, None);
        r
    }

    #[test]
    fn the_document_parses_and_names_its_schema() {
        let v: serde_json::Value = serde_json::from_str(&render(&a_run())).unwrap();
        assert_eq!(v["schema"], SCHEMA);
        assert_eq!(v["outcome"]["ok"], 9);
        assert_eq!(v["outcome"]["failed"], 1);
    }

    #[test]
    fn the_split_is_machine_readable_not_only_drawn() {
        let v: serde_json::Value = serde_json::from_str(&render(&a_run())).unwrap();
        assert_eq!(v["placement"]["free"]["requests"], 1);
        assert_eq!(v["placement"]["billed"]["requests"], 1);
        assert_eq!(v["placement"]["billed"]["charged_zkcr"], 0.002);
        assert_eq!(v["placement"]["free"]["charged_zkcr"], 0.0);
        assert_eq!(v["credits"]["billed_requests"], 1);
    }

    #[test]
    fn each_node_carries_its_class_and_what_the_router_promised() {
        let v: serde_json::Value = serde_json::from_str(&render(&a_run())).unwrap();
        let nodes = v["placement"]["nodes"].as_array().unwrap();
        assert_eq!(nodes.len(), 2);
        let free = nodes.iter().find(|n| n["class"] == "free").unwrap();
        assert_eq!(free["router_expected_ms"], 3.0);
        assert_eq!(free["via"], "local", "the transport is reported too");
        let billed = nodes.iter().find(|n| n["class"] == "billed").unwrap();
        assert!(
            billed["router_expected_ms"].is_null(),
            "nothing was promised"
        );
        assert_eq!(billed["via"], "quic");
    }

    #[test]
    fn an_empty_run_still_produces_a_valid_document() {
        let v: serde_json::Value = serde_json::from_str(&render(&BenchResults::new())).unwrap();
        assert_eq!(v["outcome"]["completed"], 0);
        assert_eq!(v["latency_us"]["p50"], 0);
    }

    #[test]
    fn an_interrupted_run_says_so_where_a_script_can_see_it() {
        let mut r = a_run();
        r.stopped = Some(StopReason::Interrupt);
        let v: serde_json::Value = serde_json::from_str(&render(&r)).unwrap();
        assert_eq!(v["outcome"]["stopped_early"], true);
        assert_eq!(v["outcome"]["stop_reason"], "sigint");
    }
}