//! Benchmark tool for the broker.
//!
//! Measures throughput, latency and — the part no HTTP benchmark has to —
//! where the work actually ran and what it cost. Work on a worker of this
//! broker's own is free; work dispatched to a peer is billed in yen, and
//! a run that reports one number for both cannot answer the question anybody
//! asks first.
//!
//! The pieces live apart so they can be read apart: [`metrics`] measures,
//! [`layout`] draws, [`live`] paints the run as it goes, [`report`] writes
//! what stays in the scrollback, and [`json`] says the same thing to a
//! machine. This file is the orchestration between them.
//!
//! Supports different routing strategies to test worker selection:
//! - best_price: Route to cheapest worker
//! - best_latency: Route to fastest worker
//! - best_availability: Route to most available worker
//! - round_robin: Distribute evenly across workers
//! - random: Random worker selection
//! - weighted_capacity: Weighted by available resources
pub(crate) mod compare;
pub(crate) mod json;
pub(crate) mod layout;
pub(crate) mod live;
pub(crate) mod metrics;
pub(crate) mod report;
pub(crate) mod tui;
use std::io::Write as _;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::Arc;
use std::thread;
use std::time::{Duration, Instant};
use crate::async_exec;
use colored::Colorize;
use layout::{boxed, grouped, pad, term_width, Muted};
use metrics::{parse_expected_ms, Ledger, RateRing};
use super::router::RoutingStrategy;
/// Percentage `num/denom*100`, guarding against a zero denominator
/// (returns 0.0 instead of NaN when there were no requests).
fn pct(num: u64, denom: u64) -> f64 {
if denom == 0 {
0.0
} else {
num as f64 / denom as f64 * 100.0
}
}
/// Concurrency clamped to at least 1 so request distribution never divides by zero.
fn effective_concurrency(c: usize) -> usize {
c.max(1)
}
/// Set by the SIGINT handler; every bench thread checks it before sending the
/// next request.
///
/// A benchmark is the one command people interrupt as a matter of course —
/// they start a hundred thousand requests, watch for twenty seconds, and have
/// seen what they came for. Throwing away those twenty seconds of samples,
/// and leaving the terminal with a half-painted frame and a hidden cursor, is
/// a bug however normal it has become.
static INTERRUPTED: AtomicBool = AtomicBool::new(false);
/// Why a run ended before it sent everything it was asked to.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum StopReason {
/// Ctrl-C.
Interrupt,
/// So much was failing that the run had stopped measuring anything.
Failing,
}
/// Ask the kernel to tell us about Ctrl-C instead of killing us for it.
///
/// The handler does the only thing a signal handler safely can: set a flag.
/// Everything else — draining, restoring the cursor, printing the report —
/// happens on the threads that were already running.
#[cfg(not(unix))]
fn arm_interrupt_handler() {}
#[cfg(unix)]
fn arm_interrupt_handler() {
extern "C" fn on_sigint(_: libc::c_int) {
// A second Ctrl-C after the first is someone saying they meant it:
// restore the terminal and go, rather than waiting on a drain that
// is evidently not draining.
if INTERRUPTED.swap(true, Ordering::SeqCst) {
let _ = std::io::Write::write_all(&mut std::io::stderr(), SHOW_CURSOR.as_bytes());
std::process::exit(130);
}
}
// `sigaction` rather than `signal`: the latter's restart and reset
// semantics differ between platforms, and it needs the handler cast
// straight from a function item to an integer, which clippy rejects.
let handler: extern "C" fn(libc::c_int) = on_sigint;
// SAFETY: `sigaction` is zeroed before use, the handler is a real
// extern "C" fn of the right signature, and the handler itself touches
// only an atomic and, on a second signal, exits.
unsafe {
let mut action: libc::sigaction = std::mem::zeroed();
action.sa_sigaction = handler as usize;
libc::sigemptyset(&mut action.sa_mask);
action.sa_flags = 0;
libc::sigaction(libc::SIGINT, &action, std::ptr::null_mut());
}
}
fn interrupted() -> bool {
INTERRUPTED.load(Ordering::Relaxed)
}
/// Hiding the cursor stops it strobing across a block that repaints ten times
/// a second. Whatever happens next — a finished run, Ctrl-C, a panic — it has
/// to come back.
const HIDE_CURSOR: &str = "\x1b[?25l";
const SHOW_CURSOR: &str = "\x1b[?25h";
/// How many completed requests must be seen before the failure rate is
/// allowed to end a run: small enough to bail out quickly, large enough that
/// a couple of unlucky requests can't.
const FAIL_FAST_MIN_SAMPLE: u64 = 20;
/// The share of failures, over that sample, that means the run is no longer
/// measuring anything.
const FAIL_FAST_RATE: f64 = 0.5;
/// Whether a run has failed enough to be worth stopping. `limit` is the
/// consecutive-failure trip, and 0 disables the guard entirely.
///
/// Two shapes of useless run, so two triggers: everything times out (a long
/// streak), or most things do while a minority succeed. The second is what a
/// broker that routes only some requests down a broken path produces, and a
/// streak counter alone almost never trips on it.
pub(crate) fn run_is_failing(completed: u64, failed: u64, consecutive: u64, limit: u32) -> bool {
if limit == 0 {
return false;
}
if consecutive >= limit as u64 {
return true;
}
completed >= FAIL_FAST_MIN_SAMPLE && (failed as f64) >= (completed as f64) * FAIL_FAST_RATE
}
/// How long to wait for a broker that has no workers yet.
///
/// A broker discovers its workers on a timer, so for a few seconds after it
/// starts it genuinely has none. `zc agent install` returns before that, which
/// is exactly when someone runs a benchmark — and sending a thousand requests
/// into an empty broker produces a thousand failures and no measurement.
const WAIT_FOR_WORKERS: Duration = Duration::from_secs(20);
/// What to do about the worker count before starting.
#[derive(Debug, Clone, Copy, PartialEq)]
pub(crate) enum WorkerCheck {
/// Enough to benchmark.
Ready(u64),
/// None yet, but the broker answered — worth waiting for discovery.
Waiting,
/// None after waiting: there is nothing to measure.
GiveUp,
}
/// Decide from a worker count and how long we have been waiting.
pub(crate) fn worker_check(count: Option<u64>, waited: Duration) -> WorkerCheck {
match count {
Some(n) if n > 0 => WorkerCheck::Ready(n),
// An unreadable answer is not evidence of an empty broker; let the
// run proceed and report whatever it finds.
None => WorkerCheck::Ready(0),
Some(_) if waited < WAIT_FOR_WORKERS => WorkerCheck::Waiting,
Some(_) => WorkerCheck::GiveUp,
}
}
/// Named benchmark workloads
#[derive(Debug, Clone, PartialEq)]
pub enum BenchWorkload {
/// Trivial add(2, 3) — measures pure broker/routing overhead
Add,
/// Recursive fib(28) — CPU-intensive, ~50ms per call
Fib,
}
impl BenchWorkload {
pub fn name(&self) -> &'static str {
match self {
BenchWorkload::Add => "add",
BenchWorkload::Fib => "fib",
}
}
pub fn description(&self) -> &'static str {
match self {
BenchWorkload::Add => "add(2, 3) — routing overhead baseline",
BenchWorkload::Fib => "fib(28) — CPU-intensive recursive Fibonacci",
}
}
/// What a correct reply contains. A benchmark that does not check this is
/// timing error pages.
pub fn expected(&self) -> i64 {
match self {
BenchWorkload::Add => 5,
BenchWorkload::Fib => 317_811,
}
}
}
impl std::str::FromStr for BenchWorkload {
type Err = String;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s.to_lowercase().as_str() {
"add" => Ok(BenchWorkload::Add),
"fib" | "fibonacci" | "cpu" => Ok(BenchWorkload::Fib),
_ => Err(format!(
"Unknown benchmark workload: '{}'. Available: add, fib",
s
)),
}
}
}
/// What was asked for about where the work must NOT run.
///
/// `--remote` is about this machine; `remote_only` on the wire is about the
/// broker. Those are the same thing only when the broker IS this machine, and
/// conflating them is what made `zc bench zc://node-… --remote` scatter its
/// work across the mesh: the flag told the named node to skip its own workers
/// and auction the job off, which is the opposite of naming a node.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum RemoteMode {
/// Not asked for: the broker routes as it sees fit.
#[default]
Off,
/// `--remote` — the work must not run on THIS machine. A broker that is
/// already somewhere else satisfies that by being there.
OffThisMachine,
/// `--remote-only` — the broker must skip its own workers and offer the
/// job to a peer, wherever that broker is. For measuring peer dispatch
/// itself.
ForcePeerDispatch,
}
/// Whether the broker must be told `remote_only` to satisfy [`RemoteMode`].
///
/// The only case where "not on this machine" needs the broker to push work
/// away is when the broker is this machine.
pub(crate) fn wants_remote_only(mode: RemoteMode, broker_is_local: bool) -> bool {
match mode {
RemoteMode::Off => false,
RemoteMode::ForcePeerDispatch => true,
RemoteMode::OffThisMachine => broker_is_local,
}
}
/// The node a `zc://` target names, in canonical `zc://node-<fp>` form.
///
/// Only the key-derived spelling identifies a node. `zc://localhost`, a bare
/// `zc://ip:port` and a legacy hostname label all say which broker to ASK and
/// nothing about where the work has to run, so they are not pins.
pub(crate) fn node_pin_from_uri(uri: &str) -> Option<String> {
let rest = uri.strip_prefix("zc://")?.trim_end_matches('/');
// `zc://node-<fp>:9000` addresses the same node on a stated port.
let rest = match rest.rsplit_once(':') {
Some((head, port)) if port.parse::<u16>().is_ok() => head,
_ => rest,
};
let fp = rest.strip_prefix("node-")?;
let is_fingerprint = fp.len() == 16 && fp.bytes().all(|b| b.is_ascii_hexdigit());
is_fingerprint.then(|| format!("zc://node-{}", fp.to_ascii_lowercase()))
}
/// The first zc version whose broker honours a `target_node` pin.
///
/// Before it, the router matched a pin against the node's *hostname label*
/// rather than its key-derived fingerprint, so no pin ever matched — and
/// `/execute` reads an unmatched pin as "nothing local" and offers the job to
/// the whole mesh. Sending a pin to such a broker therefore pushes work OFF
/// the named node. Bench withholds it and leans on the broker's own
/// local-first rule instead, which keeps the work there on every build.
const NODE_PIN_SINCE: (u64, u64, u64) = (0, 0, 33);
/// `major.minor.patch` from a version string, ignoring any pre-release or
/// build suffix. `None` when it does not start with three numbers.
fn semver3(v: &str) -> Option<(u64, u64, u64)> {
let core = v.trim().split(['-', '+']).next()?;
let mut parts = core.split('.');
let mut next = || parts.next()?.parse::<u64>().ok();
Some((next()?, next()?, next()?))
}
/// Does a broker reporting this version honour a node pin?
///
/// `None` — a broker that discloses no build at all — is a no: disclosure
/// itself only arrived in 0.0.32, so silence means older than that.
pub(crate) fn honours_node_pin(version: Option<&str>) -> bool {
version
.and_then(semver3)
.is_some_and(|v| v >= NODE_PIN_SINCE)
}
/// Benchmark configuration
#[derive(Debug, Clone)]
pub struct BenchConfig {
/// Target broker URL
pub broker_url: String,
/// Number of concurrent workers
pub concurrency: usize,
/// Total number of requests to send
pub requests: u64,
/// Request timeout in seconds
pub timeout_secs: u64,
/// Stop the run after this many failures in a row (0 disables it).
pub fail_fast: u32,
/// User ID for requests
pub user_id: String,
/// API key for Bearer auth (read from ZAKURO_API_KEY env)
pub api_key: Option<String>,
/// CPU requirement per request
pub cpus: f64,
/// Memory requirement per request (bytes)
pub memory_bytes: u64,
/// Routing strategy to use
pub strategy: RoutingStrategy,
/// Run comparison across all strategies
pub compare_strategies: bool,
/// Named benchmark workload
pub workload: BenchWorkload,
/// Filter by worker type (e.g. "standard", "premium") — routes only to matching workers
pub worker_type: Option<String>,
/// What `--remote` / `--remote-only` asked for. See [`RemoteMode`].
pub remote: RemoteMode,
/// Run every request on this node and no other, as `zc://node-<fp>`.
///
/// Set by naming a node as the target. It is sent to the broker as
/// `target_node` only when the broker discloses a build that honours it
/// (see [`honours_node_pin`]) — an older broker reads an unhonoured pin as
/// "nothing local" and offers the job to the whole mesh, which is worse
/// than not pinning at all.
pub target_node: Option<String>,
/// Whether the broker answered the preflight with a build new enough to
/// honour [`target_node`](Self::target_node). Set during preflight, never
/// by a flag.
pub(crate) send_node_pin: bool,
/// Write the run to stdout as one JSON document; the human view moves to
/// stderr so the two can be redirected apart.
pub json: bool,
/// No live view. Implied when the stream we would paint on is not a
/// terminal — repainting cursor moves into a log file helps nobody.
pub no_progress: bool,
/// Whether `-n` was given. `--compare` runs the whole workload once per
/// strategy, so its default is its own: six times the single-run default
/// took two and a half minutes on a mesh with one slow node in it.
pub(crate) requests_explicit: bool,
/// Stay with the streaming line view even on a terminal that could take
/// the interactive one. For a recording, a dumb terminal, or anyone who
/// wants the run to scroll past rather than own the screen.
pub no_tui: bool,
/// No human output at all. Set by `--compare`, which runs six benchmarks
/// and draws its own table from them.
pub(crate) silent: bool,
/// Per-request spend ceiling sent as `budget`.
///
/// This bounds the remote reservation so it cannot reserve against an
/// unpriced (`f64::MAX`) peer worker before the offer resolves. It is a
/// CEILING, not a charge -- a bench job actually costs on the order of
/// ¥0.15. It matters because the broker's precheck rejects the
/// request when the account balance is below the budget, so an
/// unnecessarily large value locks out accounts that can easily afford the
/// work.
pub budget: f64,
}
impl BenchConfig {
/// The node this run holds its work to, if any.
///
/// `--remote-only` is an explicit request for the peer-dispatch path, so
/// it releases the pin: a node named alongside it says which broker to
/// ASK, not where the work has to run. That is the only way to measure a
/// named broker's auction, and refusing the combination would take the
/// measurement away rather than clear anything up.
pub(crate) fn effective_pin(&self) -> Option<&str> {
match self.remote {
RemoteMode::ForcePeerDispatch => None,
_ => self.target_node.as_deref(),
}
}
}
impl Default for BenchConfig {
fn default() -> Self {
Self {
broker_url: "zc://localhost".to_string(),
concurrency: 10,
requests: 1000,
// 15 s, not 30: the slowest legitimate path measured on a mesh
// is a ~0.7 s round trip, and `fib(28)` is seconds at worst. A
// heavier workload can still pass --timeout.
timeout_secs: 15,
fail_fast: 10,
user_id: "bench-user".to_string(),
// Treat a set-but-empty ZAKURO_API_KEY as "no key" so we don't send
// an empty `Authorization: Bearer ` header (which the broker rejects).
// `zc login` writes the key to ~/.zakuro/credentials, and every
// other command reads it from there. Without this a normal shell
// benchmarks as `anonymous` with no balance, and every request
// that routes to a billed worker stalls until it times out.
api_key: {
crate::credentials::load_into_env();
std::env::var("ZAKURO_API_KEY")
}
.ok()
.filter(|k| !k.trim().is_empty()),
cpus: 0.1,
memory_bytes: 1024 * 1024 * 100, // 100 MiB
strategy: RoutingStrategy::BestPrice,
compare_strategies: false,
workload: BenchWorkload::Add,
worker_type: None,
remote: RemoteMode::Off,
target_node: None,
send_node_pin: false,
// Generous next to a real bench job (~¥0.15) while staying
// low enough that the balance precheck is not a gate. The previous
// hardcoded 150,000 meant any account under ¥150,000 failed
// EVERY request with a bare 402 that the summary did not surface --
// which reads as "dispatch is broken" rather than "fund the
// account". Raise it with --budget for genuinely long jobs.
budget: 1500.0,
json: false,
no_progress: false,
no_tui: false,
requests_explicit: false,
silent: false,
}
}
}
/// The live state of a run in flight, shared between the threads sending
/// requests and whichever view is drawing them.
///
/// One struct rather than eight `Arc`s threaded through each call site: the
/// interactive view and the line view both need all of it, and a view that
/// read five of the eight would be drawing a frame that never existed.
#[derive(Clone)]
pub(crate) struct Shared {
pub(crate) completed: Arc<AtomicU64>,
pub(crate) successful: Arc<AtomicU64>,
pub(crate) failed: Arc<AtomicU64>,
pub(crate) stop: Arc<AtomicBool>,
pub(crate) consecutive_failures: Arc<AtomicU64>,
pub(crate) latencies: Arc<std::sync::Mutex<Vec<u64>>>,
pub(crate) ledger: Arc<std::sync::Mutex<Ledger>>,
pub(crate) failure_reasons: Arc<std::sync::Mutex<std::collections::HashMap<String, u64>>>,
pub(crate) total_charged: Arc<std::sync::Mutex<f64>>,
pub(crate) rates: Arc<std::sync::Mutex<RateRing>>,
/// What each worker advertises per hour, asked of the broker once.
pub(crate) prices: std::collections::HashMap<String, f64>,
}
impl Shared {
fn new(capacity: u64) -> Self {
Shared {
completed: Arc::new(AtomicU64::new(0)),
successful: Arc::new(AtomicU64::new(0)),
failed: Arc::new(AtomicU64::new(0)),
stop: Arc::new(AtomicBool::new(false)),
consecutive_failures: Arc::new(AtomicU64::new(0)),
latencies: Arc::new(std::sync::Mutex::new(Vec::with_capacity(
capacity.min(1_000_000) as usize,
))),
ledger: Arc::new(std::sync::Mutex::new(Ledger::new())),
failure_reasons: Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())),
total_charged: Arc::new(std::sync::Mutex::new(0.0)),
rates: Arc::new(std::sync::Mutex::new(RateRing::new())),
prices: std::collections::HashMap::new(),
}
}
fn counters(&self) -> BenchCounters<'_> {
BenchCounters {
successful: &self.successful,
failed: &self.failed,
completed: &self.completed,
stop: &self.stop,
consecutive_failures: &self.consecutive_failures,
latencies: &self.latencies,
ledger: &self.ledger,
failure_reasons: &self.failure_reasons,
total_charged: &self.total_charged,
prices: self.prices.clone(),
}
}
}
/// Everything one run measured.
#[derive(Debug, Clone)]
pub struct BenchResults {
/// How many requests the run was asked for.
pub total_requests: u64,
/// How many it actually sent. Below `total_requests` when the run was
/// interrupted or gave up — and the denominator for every rate, because
/// dividing by what was *asked for* flatters a run that stopped early.
pub completed: u64,
/// Successful requests
pub successful: u64,
/// Failed requests
pub failed: u64,
/// Wall clock
pub duration: Duration,
/// Completed requests per second over the whole run
pub rps: f64,
/// Every successful request's measured time, in microseconds
pub latencies_us: Vec<u64>,
/// Completions in each closed second of the run
pub rates_per_second: Vec<u64>,
/// Where the work ran and what each part of it cost
pub ledger: Ledger,
/// Why requests failed, and how often each way
pub failure_reasons: std::collections::HashMap<String, u64>,
/// What the broker's replies said they charged, summed
pub total_charged: f64,
/// Why the run ended early, if it did
pub stopped: Option<StopReason>,
/// The run's settings, so the report can state its own conditions
pub config: BenchConfig,
/// When it started, as a local timestamp
pub started_at: String,
}
impl Default for BenchResults {
fn default() -> Self {
Self::new()
}
}
impl BenchResults {
pub fn new() -> Self {
BenchResults {
total_requests: 0,
completed: 0,
successful: 0,
failed: 0,
duration: Duration::ZERO,
rps: 0.0,
latencies_us: Vec::new(),
rates_per_second: Vec::new(),
ledger: Ledger::new(),
failure_reasons: std::collections::HashMap::new(),
total_charged: 0.0,
stopped: None,
config: BenchConfig::default(),
started_at: String::new(),
}
}
/// Yen per request, over the requests that actually got a reply.
pub(crate) fn avg_charge(&self) -> f64 {
match self.successful {
0 => 0.0,
n => self.total_charged / n as f64,
}
}
/// Failure reasons, commonest first, ties broken by text so a report of
/// the same run twice reads the same twice.
pub(crate) fn ranked_failure_reasons(&self) -> Vec<(String, u64)> {
let mut v: Vec<(String, u64)> = self
.failure_reasons
.iter()
.map(|(r, c)| (r.clone(), *c))
.collect();
v.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
v
}
/// The broker, named the way the CLI's IP-free rule requires: a host, and
/// never an address.
pub(crate) fn broker_label(&self) -> String {
let url = self
.config
.broker_url
.trim_start_matches("http://")
.trim_start_matches("https://");
format!("broker {}", url.trim_end_matches('/'))
}
pub fn print_report(&self) {
print!("{}", report::render(self, term_width()));
}
/// The same run as one JSON document.
pub fn to_json(&self) -> String {
json::render(self)
}
}
/// The mesh-connectivity line `zc bench` prints. It states the fact and
/// nothing else: the broker's mesh address used to be appended here, and the
/// CLI never shows an IP to an end user (see `main.rs`'s IP-FREE RULE).
fn wireguard_line(connected: bool) -> String {
use colored::Colorize;
match connected {
true => "connected".green().to_string(),
false => "not connected (continuing)".yellow().to_string(),
}
}
/// Query the broker's /health endpoint and return (wireguard_connected, wireguard_ip, node_name).
fn check_wireguard_status(broker_url: &str) -> (bool, Option<String>, Option<String>) {
match crate::vpn::mesh_agent_for(broker_url, std::time::Duration::from_secs(30))
.get(&format!("{}/health", broker_url))
.config()
.timeout_global(Some(Duration::from_secs(5)))
.build()
.call()
{
Ok(resp) => {
if let Ok(body) = resp.into_body().read_to_string() {
if let Ok(v) = serde_json::from_str::<serde_json::Value>(&body) {
let connected = v["wireguard_connected"].as_bool().unwrap_or(false);
let ip = v["wireguard_ip"].as_str().map(|s| s.to_string());
let name = v["node_name"].as_str().map(|s| s.to_string());
return (connected, ip, name);
}
}
(false, None, None)
}
Err(_) => (false, None, None),
}
}
/// Each worker's advertised price, asked of the broker once before the run.
///
/// The reply to an `/execute` does not carry it, and the price is half of
/// what a benchmark is for: a node advertising 3.6 yen/h that bills 0.0000
/// is a finding, and a spend column on its own cannot show it.
fn fleet_prices(config: &BenchConfig) -> std::collections::HashMap<String, f64> {
let mut prices = std::collections::HashMap::new();
let body = crate::vpn::mesh_agent_for(&config.broker_url, Duration::from_secs(10))
.get(&format!("{}/workers", config.broker_url))
.config()
.timeout_global(Some(Duration::from_secs(5)))
.build()
.call()
.ok()
.and_then(|r| r.into_body().read_to_string().ok());
let Some(body) = body else { return prices };
let Ok(json) = serde_json::from_str::<serde_json::Value>(&body) else {
return prices;
};
if let Some(workers) = json["workers"].as_array() {
for w in workers {
if let (Some(name), Some(price)) = (w["name"].as_str(), w["price_per_hour"].as_f64()) {
prices.insert(name.to_string(), price);
}
}
}
prices
}
/// A local timestamp for the report header, without pulling in a formatter.
fn now_local() -> String {
chrono::Local::now().format("%Y-%m-%d %H:%M %Z").to_string()
}
/// Write straight to the terminal, bypassing stdout when `--json` has claimed
/// it. The human view and the machine view must never end up in the same
/// pipe: one of them would be corrupt.
fn paint(config: &BenchConfig, s: &str) {
if config.silent {
return;
}
match config.json {
true => {
let mut err = std::io::stderr();
let _ = err.write_all(s.as_bytes());
let _ = err.flush();
}
false => {
let mut out = std::io::stdout();
let _ = out.write_all(s.as_bytes());
let _ = out.flush();
}
}
}
macro_rules! painted {
($config:expr, $($arg:tt)*) => { paint($config, &format!($($arg)*)) };
}
/// Whether the view can repaint in place: a terminal on the stream we are
/// drawing to, and progress not turned off.
fn can_repaint(config: &BenchConfig) -> bool {
use std::io::IsTerminal;
if config.no_progress || config.silent {
return false;
}
match config.json {
true => std::io::stderr().is_terminal(),
false => std::io::stdout().is_terminal(),
}
}
pub fn run_benchmark(config: BenchConfig) -> BenchResults {
run_benchmark_watched(config, &|_| {})
}
/// A run, with `publish` called once the shared state exists so a caller
/// outside can watch it. `--compare` uses it to draw the strategy it is on
/// while that strategy is still going; every other caller passes a no-op.
pub(crate) fn run_benchmark_watched(
mut config: BenchConfig,
publish: &dyn Fn(&Shared),
) -> BenchResults {
let started_at = now_local();
let mut early = BenchResults::new();
early.config = config.clone();
early.total_requests = config.requests;
early.started_at = started_at.clone();
painted!(&config, "\n");
paint(
&config,
&boxed(&[
format!("{}", "zc bench".bold().white()),
format!(
"{} · {} · {} concurrent · {} requests",
config.workload.name().yellow(),
config.strategy.as_str().yellow(),
config.concurrency,
grouped(config.requests)
),
format!("{} {}", "broker".muted(), config.broker_url.cyan()),
]),
);
painted!(&config, "\n");
painted!(
&config,
" {} {}\n",
pad(
&config.workload.name().yellow().bold().to_string(),
14,
false
),
config.workload.description().muted()
);
painted!(
&config,
" {} {}\n",
pad(
&config.strategy.as_str().yellow().bold().to_string(),
14,
false
),
config.strategy.description().muted()
);
if let Some(ref wt) = config.worker_type {
painted!(&config, " Worker Type: {}\n", wt.yellow().bold());
}
painted!(&config, "\n");
// First check if broker is reachable
painted!(&config, " Connecting to broker... ");
let health =
match crate::vpn::mesh_agent_for(&config.broker_url, std::time::Duration::from_secs(30))
.get(&format!("{}/health", config.broker_url))
.config()
.timeout_global(Some(Duration::from_secs(5)))
.build()
.call()
{
Ok(r) => {
painted!(&config, "{}\n", "OK".green());
r.into_body()
.read_to_string()
.ok()
.and_then(|b| serde_json::from_str::<serde_json::Value>(&b).ok())
}
Err(e) => {
painted!(&config, "{}\n Error: {}\n", "FAILED".red(), e);
return early;
}
};
// The same answer says which build this broker runs (#308), which decides
// whether a node pin can be sent at all.
let broker_version = health
.as_ref()
.and_then(|h| h["build"]["version"].as_str())
.map(str::to_string);
config.send_node_pin = honours_node_pin(broker_version.as_deref());
// Check WireGuard connectivity — required for cross-node billing (skip for localhost)
let loopback = crate::vpn::is_loopback_url(config.broker_url.trim());
if loopback {
painted!(
&config,
" {} (skipping WireGuard check)\n",
"Local broker".muted()
);
} else {
painted!(&config, " Checking WireGuard... ");
let (ts_connected, _, _) = check_wireguard_status(&config.broker_url);
// Not fatal: the broker is already reachable (its /health answered
// above), possibly over another transport such as WireGuard. Warn
// and continue — the workers check below validates the actual fleet.
painted!(&config, "{}\n", wireguard_line(ts_connected));
}
if config.api_key.is_none() {
painted!(
&config,
" {} not signed in: requests run as {} and a billed worker will refuse them — run {}\n",
"Warning:".yellow(),
"anonymous".yellow(),
"zc login".cyan()
);
}
painted!(&config, " Checking workers... ");
let started = Instant::now();
let mut announced_wait = false;
let mut worker_total = 0u64;
loop {
let count = crate::vpn::mesh_agent_for(&config.broker_url, Duration::from_secs(30))
.get(&format!("{}/workers", config.broker_url))
.config()
.timeout_global(Some(Duration::from_secs(5)))
.build()
.call()
.ok()
.and_then(|r| r.into_body().read_to_string().ok())
.and_then(|b| serde_json::from_str::<serde_json::Value>(&b).ok())
.map(|j| j["total"].as_u64().unwrap_or(0));
match worker_check(count, started.elapsed()) {
WorkerCheck::Ready(0) => {
painted!(&config, "{}\n", "UNKNOWN".yellow());
break;
}
WorkerCheck::Ready(n) => {
worker_total = n;
painted!(&config, "{} worker(s)\n", n.to_string().green());
break;
}
WorkerCheck::Waiting => {
if !announced_wait {
// A broker finds its workers on a timer, so this is the
// normal state for a few seconds after it starts.
painted!(&config, "{}", "none yet, waiting… ".yellow());
announced_wait = true;
}
thread::sleep(Duration::from_millis(500));
}
WorkerCheck::GiveUp => {
painted!(
&config,
"{}\n Nothing to benchmark: the broker has no workers after {}s. \n Start some with {} and try again.\n",
"NO WORKERS".red(),
WAIT_FOR_WORKERS.as_secs(),
"zc share".cyan()
);
return early;
}
}
}
// Say what the run will actually do about placement, and what that can
// cost. Both are questions people would rather have answered before the
// run than after it.
let _ = worker_total;
let remote_only = wants_remote_only(config.remote, loopback);
if let (Some(named), None) = (config.target_node.as_deref(), config.effective_pin()) {
painted!(
&config,
" {} {} {}\n",
"--remote-only:".muted(),
named.cyan(),
"says which broker to ask; the work is being sent away from it on purpose".muted()
);
}
if let Some(pin) = config.effective_pin() {
painted!(
&config,
" {} {}{}\n",
"Pinned to".muted(),
pin.cyan(),
match (config.send_node_pin, broker_version.as_deref()) {
(true, _) => String::new(),
(false, Some(v)) => format!(
" — broker runs {v}, which ignores a pin; relying on its local-first routing"
)
.muted()
.to_string(),
(false, None) => " — broker discloses no build; relying on its local-first routing"
.muted()
.to_string(),
}
);
}
if config.remote == RemoteMode::OffThisMachine && !loopback && !remote_only {
painted!(
&config,
" {} the broker is already off this machine, so nothing is forced to a peer\n",
"--remote:".muted()
);
}
painted!(
&config,
" {}\n",
match (loopback, remote_only) {
(_, true) => "every request goes to a peer: this run will be billed"
.yellow()
.to_string(),
(true, _) => "local broker: work that stays here is free"
.muted()
.to_string(),
(false, _) => "work sent to a peer is billed; work that stays local is free"
.muted()
.to_string(),
}
);
painted!(&config, "\n Starting benchmark...\n\n");
arm_interrupt_handler();
let interactive = tui::wanted(&config);
let repaint = !interactive && can_repaint(&config);
if repaint {
paint(&config, HIDE_CURSOR);
}
// Shared counters
let mut shared = Shared::new(config.requests);
shared.prices = fleet_prices(&config);
publish(&shared);
// Calculate requests per worker (clamp concurrency so `-c 0` cannot divide by zero)
let concurrency = effective_concurrency(config.concurrency);
let requests_per_worker = config.requests / concurrency as u64;
let extra_requests = config.requests % concurrency as u64;
let start = Instant::now();
let handles: Vec<_> = (0..concurrency)
.map(|i| {
let config = config.clone();
let shared = shared.clone();
// First few workers get extra requests to handle remainder
let my_requests = if (i as u64) < extra_requests {
requests_per_worker + 1
} else {
requests_per_worker
};
async_exec::spawn_blocking(move || {
run_worker(i, my_requests, &config, &shared.counters());
})
})
.collect();
let total = config.requests;
match interactive {
// The interactive view owns the terminal and the main thread: keys
// come from stdin, so nothing else may be reading it.
true => {
if let Ok(tui::Exit::Quit) = tui::run(&config, &shared, start, total, worker_total) {
INTERRUPTED.store(true, Ordering::SeqCst);
shared.stop.store(true, Ordering::Relaxed);
}
}
false => {
let progress_handle = {
let shared = shared.clone();
let config = config.clone();
async_exec::spawn_blocking(move || {
line_view(&config, &shared, start, total, repaint);
})
};
for handle in handles {
let _ = handle.join();
}
shared.stop.store(true, Ordering::Relaxed);
let _ = progress_handle.join();
if repaint {
paint(&config, SHOW_CURSOR);
}
return finish(config, shared, start, started_at);
}
}
// Interactive path: the view has returned, so the run is over or the
// person ended it. Let the senders notice and wind down.
for handle in handles {
let _ = handle.join();
}
finish(config, shared, start, started_at)
}
/// The streaming line view: one block repainted in place, or a plain line now
/// and then when there is no terminal to repaint on.
fn line_view(config: &BenchConfig, shared: &Shared, start: Instant, total: u64, repaint: bool) {
let spinner = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
let mut i = 0;
let mut drawn = 0usize;
loop {
let done = shared.completed.load(Ordering::Relaxed);
let elapsed = start.elapsed();
if let Ok(mut r) = shared.rates.lock() {
r.sample(done, elapsed);
}
let samples = shared
.latencies
.lock()
.map(|l| l.clone())
.unwrap_or_default();
let led = shared.ledger.lock().map(|l| l.clone()).unwrap_or_default();
let ring = shared.rates.lock().map(|r| r.clone()).unwrap_or_default();
let block = live::LiveView {
spin: spinner[i % spinner.len()],
done,
total,
ok: shared.successful.load(Ordering::Relaxed),
failed: shared.failed.load(Ordering::Relaxed),
elapsed,
samples: &samples,
ledger: &led,
rates: &ring,
width: term_width(),
}
.render();
if repaint {
// Back to the top of the last block, then repaint it. Each line is
// cleared to its end so a shorter line can't leave the tail of a
// longer one behind.
let height = block.lines().count();
let mut out = String::new();
if drawn > 0 {
out.push_str(&format!("\x1b[{drawn}A"));
}
for line in block.lines() {
out.push_str("\x1b[2K");
out.push_str(line);
out.push('\n');
}
// A block that got SHORTER leaves the tail of the taller one on
// screen — the footer printed four times over, once per row the
// node table lost. Blank what the last frame used and no longer
// needs, then come back up to where this frame ends.
if height < drawn {
out.push_str(&"\x1b[2K\n".repeat(drawn - height));
out.push_str(&format!("\x1b[{}A", drawn - height));
}
paint(config, &out);
drawn = height;
} else if !config.no_progress && (done >= total || i % 20 == 0) {
painted!(
config,
" {done}/{total} · {} ok · {} failed · {:.1} req/s\n",
shared.successful.load(Ordering::Relaxed),
shared.failed.load(Ordering::Relaxed),
done as f64 / elapsed.as_secs_f64().max(0.001)
);
}
if done >= total || shared.stop.load(Ordering::Relaxed) {
if repaint && drawn > 0 {
// Wipe the live block; the summary below says all of it again,
// and leaving both on screen reads as a stutter.
let mut out = format!("\x1b[{drawn}A");
out.push_str(&"\x1b[2K\n".repeat(drawn));
out.push_str(&format!("\x1b[{drawn}A"));
paint(config, &out);
}
break;
}
thread::sleep(Duration::from_millis(100));
i += 1;
}
}
/// Gather what the run measured, once the senders have stopped.
fn finish(config: BenchConfig, shared: Shared, start: Instant, started_at: String) -> BenchResults {
let duration = start.elapsed();
let completed_count = shared.completed.load(Ordering::Relaxed);
let stopped = match (interrupted(), completed_count < config.requests) {
(true, _) => Some(StopReason::Interrupt),
(false, true) => Some(StopReason::Failing),
(false, false) => None,
};
if stopped == Some(StopReason::Interrupt) {
painted!(
&config,
"\n {} {}\n",
"^C".red().bold(),
format!(
"stopped after {} — reporting on the {} requests that finished.",
layout::precise_duration(duration),
grouped(completed_count)
)
);
}
let charged = *shared
.total_charged
.lock()
.unwrap_or_else(|e| e.into_inner());
let mut ring = shared.rates.lock().map(|r| r.clone()).unwrap_or_default();
ring.finish(completed_count, duration);
// Rates are per completed request, not per requested one: a run that
// stopped at 786 of 100 000 did 786 requests' worth of work, and dividing
// by 100 000 would report a throughput nobody achieved.
let secs = duration.as_secs_f64();
let rps = match secs > 0.0 {
true => completed_count as f64 / secs,
false => 0.0,
};
BenchResults {
total_requests: config.requests,
completed: completed_count,
successful: shared.successful.load(Ordering::Relaxed),
failed: shared.failed.load(Ordering::Relaxed),
duration,
rps,
latencies_us: shared
.latencies
.lock()
.map(|l| l.clone())
.unwrap_or_default(),
rates_per_second: ring.per_second().to_vec(),
ledger: shared.ledger.lock().map(|l| l.clone()).unwrap_or_default(),
failure_reasons: shared
.failure_reasons
.lock()
.map(|f| f.clone())
.unwrap_or_default(),
total_charged: charged,
stopped,
config,
started_at,
}
}
#[allow(clippy::too_many_arguments)]
/// Build the `X-Zakuro-Requirements` payload for a bench request.
///
/// Split out from `run_worker` so the budget wiring is testable without
/// standing up a broker.
fn build_requirements(config: &BenchConfig) -> serde_json::Value {
let mut requirements = serde_json::json!({
"cpus": config.cpus,
"memory_bytes": config.memory_bytes,
"gpus": 0,
"estimated_duration_secs": 0.1,
"strategy": config.strategy.as_str(),
"remote_only": wants_remote_only(
config.remote,
crate::vpn::is_loopback_url(config.broker_url.trim()),
),
// Tell the broker how long the caller will actually wait. Without it
// the peer-dispatch deadline falls back to a blind 30s and the
// per-peer offer to 310s, so a dispatch kept running — holding a
// worker, still billable — long after the client had given up.
"timeout_secs": config.timeout_secs as f64,
// Bound the remote reservation so it doesn't reserve against an unpriced
// (f64::MAX) peer worker before the offer resolves. Configurable: the
// broker also rejects when balance < budget, so this must not be larger
// than it needs to be. See BenchConfig::budget.
"budget": config.budget
});
if let Some(ref wt) = config.worker_type {
requirements["worker_type"] = serde_json::Value::String(wt.clone());
}
if let (true, Some(tn)) = (config.send_node_pin, config.effective_pin()) {
requirements["target_node"] = serde_json::Value::String(tn.to_string());
}
requirements
}
/// The accumulators every bench worker writes into.
///
/// Grouped rather than passed individually: they are one thing -- the shared
/// tally for a run -- and threading eight references through each call site
/// made run_worker a ten-argument function, which clippy rejects.
struct BenchCounters<'a> {
successful: &'a AtomicU64,
failed: &'a AtomicU64,
completed: &'a AtomicU64,
/// Set once the run has failed enough times in a row to be pointless;
/// every thread checks it before sending the next request.
stop: &'a AtomicBool,
/// Failures in a row across the whole run. Per-thread it would take
/// `limit` failures in one thread, which a concurrent run rarely reaches
/// even when almost everything is failing.
consecutive_failures: &'a AtomicU64,
latencies: &'a std::sync::Mutex<Vec<u64>>,
/// Where each request ran and what it cost.
ledger: &'a std::sync::Mutex<Ledger>,
failure_reasons: &'a std::sync::Mutex<std::collections::HashMap<String, u64>>,
/// What each worker advertises per hour, read from the broker before the
/// run started.
prices: std::collections::HashMap<String, f64>,
/// Running total of `charged` actually returned by `/execute`
/// replies. A reply missing or mangling the field is simply skipped --
/// see `parse_charged`.
total_charged: &'a std::sync::Mutex<f64>,
}
/// Extract `charged` from an `/execute` reply body. `None` when the
/// field is absent or not a number, so a malformed reply cannot silently
/// corrupt the running total as some other number -- it is simply not
/// counted.
fn parse_charged(body: &str) -> Option<f64> {
serde_json::from_str::<serde_json::Value>(body)
.ok()?
.get("charged")?
.as_f64()
}
/// What one reply told us, gathered before the body is consumed.
struct Reply {
worker: String,
transport: Option<String>,
expected_ms: Option<f64>,
header_cost: Option<f64>,
/// What the worker that ran it advertises per hour, when the broker says.
price_per_hour: Option<f64>,
}
/// Whether a reply is the answer the workload asked for.
///
/// `Err` carries the reason, short enough to be a row in the failure table and
/// specific enough to act on: a wrong number means the mesh ran something
/// else, an unreadable body means a proxy or a worker error came back as 200.
fn answer_check(body: &[u8], want: i64) -> Result<(), String> {
match pickled_int(body) {
Some(got) if got == want => Ok(()),
Some(got) => Err(format!("wrong answer: got {got}, want {want}")),
None if body.is_empty() => Err("empty reply".to_string()),
None => Err(format!(
"reply is not a result ({} bytes){}",
body.len(),
error_hint(body)
)),
}
}
/// A few words on what a non-result body actually was, when it is recognisable.
/// "not a result (263 bytes)" sends someone reading a log nowhere; naming the
/// proxy or the traceback ends the investigation.
///
/// `unknown opcode` is called out separately from the generic traceback because
/// it is the one failure here that is nobody's fault locally: a cloudpickled
/// function carries raw CPython bytecode, and opcode numbers are renumbered
/// between minor versions (`LOAD_FAST` is 124 up to 3.12 and 85 from 3.13), so a
/// perfectly healthy worker on a different Python raises `SystemError` on every
/// request. Reported as "the worker raised" it reads as a broken worker.
fn error_hint(body: &[u8]) -> &'static str {
let head = &body[..body.len().min(512)];
let contains = |needle: &str| {
head.windows(needle.len())
.any(|w| w.eq_ignore_ascii_case(needle.as_bytes()))
};
match () {
_ if contains("tinyproxy") || contains("unable to connect") => {
" — proxy could not reach it"
}
_ if contains("<html") => " — an HTML error page",
// Before the generic `Error` arm below, which "SystemError" also
// matches: this one is not a worker fault at all, and sending the
// reader to that worker's logs wastes the afternoon it already cost.
_ if contains("unknown opcode") => {
" — worker runs a different Python than this payload was built for"
}
_ if contains("Traceback") || contains("Error") => " — the worker raised",
_ => "",
}
}
/// Count one failed request and stop the run if it has stopped measuring.
fn record_failure(reason: &str, counters: &BenchCounters<'_>, config: &BenchConfig) {
// Record ALWAYS, so the summary can say why. ZAKURO_BENCH_DEBUG
// now only controls the per-request echo.
if let Ok(mut fr) = counters.failure_reasons.lock() {
*fr.entry(reason.to_string()).or_insert(0) += 1;
}
if std::env::var("ZAKURO_BENCH_DEBUG").is_ok() {
eprintln!("[bench] request error: {}", reason);
}
counters.failed.fetch_add(1, Ordering::Relaxed);
let in_a_row = counters
.consecutive_failures
.fetch_add(1, Ordering::Relaxed)
+ 1;
if run_is_failing(
counters.completed.load(Ordering::Relaxed) + 1,
counters.failed.load(Ordering::Relaxed),
in_a_row,
config.fail_fast,
) {
counters.stop.store(true, Ordering::Relaxed);
}
}
fn run_worker(
_worker_id: usize,
requests: u64,
config: &BenchConfig,
counters: &BenchCounters<'_>,
) {
// `failed`, `failure_reasons` and the fail-fast trip are reached through
// `record_failure`, which both the transport-error arm and the
// wrong-answer arm call.
let BenchCounters {
successful,
completed,
stop,
consecutive_failures,
latencies,
ledger,
total_charged,
prices,
..
} = counters;
let payload = create_test_payload(&config.workload);
let requirements = build_requirements(config);
// One agent for this thread's whole run, not one per request. The agent
// owns the connection pool, so rebuilding it each time threw the
// connection away and made every request pay a fresh handshake -- the
// benchmark was partly measuring its own client.
//
// The agent's own timeout bounds the CONNECT too, which the per-request
// `timeout_global` below does not: a request that stalls reaching the
// broker used to sit here for 310 s whatever --timeout said.
let agent =
crate::vpn::mesh_agent_for(&config.broker_url, Duration::from_secs(config.timeout_secs));
let endpoint = format!("{}/execute", config.broker_url);
for _ in 0..requests {
if stop.load(Ordering::Relaxed) || interrupted() {
return;
}
let start = Instant::now();
let mut req = agent
.post(&endpoint)
.config()
.timeout_global(Some(Duration::from_secs(config.timeout_secs)))
.build()
.header("Content-Type", "application/octet-stream")
.header("X-Zakuro-User", &config.user_id)
.header("X-Zakuro-Requirements", &requirements.to_string());
if let Some(ref key) = config.api_key {
req = req.header("Authorization", &format!("Bearer {}", key));
}
let result = req.send(&payload);
let elapsed_us = start.elapsed().as_micros() as u64;
match result {
Ok(resp) => {
let header = |name: &str| {
resp.headers()
.get(name)
.and_then(|v| v.to_str().ok())
.map(|s| s.to_string())
};
let reply = Reply {
worker: header("X-Zakuro-Worker").unwrap_or_else(|| "unknown".to_string()),
// Which side of the split this request landed on. The
// broker says `local` when it ran the job itself and names
// the peer transport when it sent it away, so this is the
// fact rather than an inference from the price.
transport: header("X-Zakuro-Transport"),
// What the router said it expected of this worker (#267),
// weighed below against what the request actually took.
expected_ms: header("X-Zakuro-Reason")
.as_deref()
.and_then(parse_expected_ms),
header_cost: header("X-Zakuro-Cost").and_then(|c| c.parse::<f64>().ok()),
price_per_hour: header("X-Zakuro-Price-Per-Hour")
.and_then(|p| p.parse::<f64>().ok()),
};
// The body decides whether this was work or an error dressed
// as a 200. Read it as bytes: a reply is a pickle, and reading
// it as a String silently discards every non-UTF-8 one.
let body = resp.into_body().read_to_vec().unwrap_or_default();
// What this request actually cost, per the broker's own reply
// -- never estimated from prices. The body's `charged` is
// the authority; the `X-Zakuro-Cost` header is the fallback
// for a reply whose body we could not read.
let charged = std::str::from_utf8(&body)
.ok()
.and_then(parse_charged)
.or(reply.header_cost)
.unwrap_or(0.0);
match answer_check(&body, config.workload.expected()) {
Err(why) => {
// A 200 carrying the wrong answer is a failed request,
// not a fast one. Counting it as a success is how a
// run of error pages reported excellent throughput.
record_failure(&why, counters, config);
}
Ok(()) => {
successful.fetch_add(1, Ordering::Relaxed);
consecutive_failures.store(0, Ordering::Relaxed);
if let Ok(mut lats) = latencies.lock() {
lats.push(elapsed_us);
}
if let Ok(mut total) = total_charged.lock() {
*total += charged;
}
if let Ok(mut l) = ledger.lock() {
l.record(
&reply.worker,
reply.transport.as_deref(),
reply
.price_per_hour
.or_else(|| prices.get(&reply.worker).copied()),
elapsed_us,
charged,
reply.expected_ms,
);
}
}
}
}
Err(e) => record_failure(&e.to_string(), counters, config),
}
completed.fetch_add(1, Ordering::Relaxed);
}
}
fn create_test_payload(workload: &BenchWorkload) -> Vec<u8> {
match workload {
BenchWorkload::Add => {
// {"func": add, "args": (2, 3), "kwargs": {}} → returns 5
// def add(a, b): return a + b
// cloudpickle.dumps({"func": add, "args": (2, 3), "kwargs": {}})
hex_decode("800595cd010000000000007d94288c0466756e63948c17636c6f75647069636b6c652e636c6f75647069636b6c65948c0e5f6d616b655f66756e6374696f6e9493942868028c0d5f6275696c74696e5f747970659493948c08436f6465547970659485945294284b024b004b004b024b024b03430c97007c007c017a0000005300944e8594298c0161948c01629486948c083c737472696e673e948c03616464948c03616464944b04430b8000d80b0c88718935804c944300942929749452947d94288c0b5f5f7061636b6167655f5f944e8c085f5f6e616d655f5f948c085f5f6d61696e5f5f94754e4e4e7494529468028c125f66756e6374696f6e5f7365747374617465949394681b7d947d942868188c03616464948c0c5f5f7175616c6e616d655f5f948c03616464948c0f5f5f616e6e6f746174696f6e735f5f947d948c0e5f5f6b7764656661756c74735f5f944e8c0c5f5f64656661756c74735f5f944e8c0a5f5f6d6f64756c655f5f9468198c075f5f646f635f5f944e8c0b5f5f636c6f737572655f5f944e8c175f636c6f75647069636b6c655f7375626d6f64756c6573945d948c0b5f5f676c6f62616c735f5f947d947586948652308c0461726773944b024b0386948c066b7761726773947d94752e")
}
BenchWorkload::Fib => {
// {"func": fib, "args": (28,), "kwargs": {}} → returns 317811
// def fib(n): return n if n <= 1 else fib(n-1) + fib(n-2)
// cloudpickle.dumps({"func": fib, "args": (28,), "kwargs": {}})
hex_decode("8005952f020000000000007d94288c0466756e63948c17636c6f75647069636b6c652e636c6f75647069636b6c65948c0e5f6d616b655f66756e6374696f6e9493942868028c0d5f6275696c74696e5f747970659493948c08436f6465547970659485945294284b014b004b004b014b054b03434a97007c0064016b1a000072027c005300740100000000000000007c0064017a0a0000ab01000000000000740100000000000000007c0064027a0a0000ab010000000000007a0000005300944e4b014b0287948c036669629485948c016e9485948c083c737472696e673e948c0366696294680c4b0543298000d8070888418276d80f108808dc0b0e8871903189758b3a9c039841a00199459b0ad10b22d00422944300942929749452947d94288c0b5f5f7061636b6167655f5f944e8c085f5f6e616d655f5f948c085f5f6d61696e5f5f94754e4e4e7494529468028c125f66756e6374696f6e5f7365747374617465949394681b7d947d942868188c03666962948c0c5f5f7175616c6e616d655f5f948c03666962948c0f5f5f616e6e6f746174696f6e735f5f947d948c0e5f5f6b7764656661756c74735f5f944e8c0c5f5f64656661756c74735f5f944e8c0a5f5f6d6f64756c655f5f9468198c075f5f646f635f5f944e8c0b5f5f636c6f737572655f5f944e8c175f636c6f75647069636b6c655f7375626d6f64756c6573945d948c0b5f5f676c6f62616c735f5f947d94680c681b737586948652308c0461726773944b1c85948c066b7761726773947d94752e")
}
}
}
/// The integer a pickled reply carries, when it carries exactly one.
///
/// Not a pickle implementation — a reader for the single shape a bench reply
/// is allowed to have: an optional PROTO/FRAME preamble, one integer, STOP.
/// Anything else is `None`, and `None` is what makes a reply a failure.
///
/// This exists because bench used to check the HTTP status and nothing else.
/// A tinyproxy `500 Unable to connect` HTML page and a pickled Python
/// `SystemError` traceback both arrive as 200 with a worker header, and both
/// were counted as successful requests with excellent latencies — a run that
/// answered nothing scored 336 req/s (measured 2026-09-19). A benchmark that
/// cannot tell work from an error page cannot measure either.
pub(crate) fn pickled_int(body: &[u8]) -> Option<i64> {
let mut i = 0usize;
// Preamble: PROTO <n>, FRAME <u64>. Both optional, either order in practice.
loop {
match body.get(i)? {
0x80 => i += 2, // PROTO + version byte
0x95 => i += 9, // FRAME + 8-byte length
_ => break,
}
}
let value = match body.get(i)? {
// BININT1: one unsigned byte.
0x4b => {
let v = *body.get(i + 1)? as i64;
i += 2;
v
}
// BININT2: two bytes, little-endian, unsigned.
0x4d => {
let b = body.get(i + 1..i + 3)?;
i += 3;
u16::from_le_bytes([b[0], b[1]]) as i64
}
// BININT: four bytes, little-endian, SIGNED.
0x4a => {
let b = body.get(i + 1..i + 5)?;
i += 5;
i32::from_le_bytes([b[0], b[1], b[2], b[3]]) as i64
}
// LONG1: length byte, then that many bytes, little-endian, signed
// two's complement. Empty payload is zero.
0x8a => {
let n = *body.get(i + 1)? as usize;
if n > 8 {
return None;
}
let b = body.get(i + 2..i + 2 + n)?;
i += 2 + n;
let negative = b.last().is_some_and(|last| last & 0x80 != 0);
let mut buf = if negative { [0xffu8; 8] } else { [0u8; 8] };
buf[..n].copy_from_slice(b);
i64::from_le_bytes(buf)
}
_ => return None,
};
// Exactly one value, then the end. A longer pickle is some other object
// that merely begins with an integer.
match body.get(i)? {
0x2e => Some(value),
_ => None,
}
}
/// Decode a hex string to bytes at compile time.
fn hex_decode(s: &str) -> Vec<u8> {
(0..s.len())
.step_by(2)
.map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap())
.collect()
}
/// Parse benchmark arguments and run
pub fn run_from_args(args: &[String]) {
// `zc bench mesh ...` → mesh warmup/calibration (issue #47).
if args.first().map(|a| a == "mesh").unwrap_or(false) {
super::bench_mesh::run_from_args(&args[1..]);
return;
}
// `zc bench node ...` → benchmark this machine's hardware.
if args.first().map(|a| a == "node").unwrap_or(false) {
let code = crate::bench_node::cli::run(&args[1..]);
if code != 0 {
std::process::exit(code);
}
return;
}
let mut config = BenchConfig::default();
if !apply_flags(&mut config, args) {
return;
}
run_configured(config);
}
/// Fold `args` into `config`. Returns false when the command is over —
/// `--help`, `--list-strategies`, or an argument that could not be used.
///
/// Split from [`run_from_args`] so what the flags MEAN can be tested without
/// standing up a broker; `--remote` alone took three live runs to pin down.
fn apply_flags(config: &mut BenchConfig, args: &[String]) -> bool {
let mut i = 0;
while i < args.len() {
match args[i].as_str() {
"-c" | "--concurrency" => {
if i + 1 < args.len() {
config.concurrency = args[i + 1].parse().unwrap_or(10);
i += 1;
}
}
"-n" | "--requests" => {
if i + 1 < args.len() {
config.requests = args[i + 1].parse().unwrap_or(1000);
config.requests_explicit = true;
i += 1;
}
}
"-u" | "--url" => {
if i + 1 < args.len() {
config.target_node = node_pin_from_uri(&args[i + 1]);
config.broker_url = match super::uri::resolve(&args[i + 1]) {
Ok(u) => u,
Err(e) => {
eprintln!("Error: {}", e);
std::process::exit(1);
}
};
i += 1;
}
}
"--user" => {
if i + 1 < args.len() {
config.user_id = args[i + 1].clone();
i += 1;
}
}
"--timeout" => {
if i + 1 < args.len() {
if let Ok(secs) = args[i + 1].parse::<u64>() {
if secs > 0 {
config.timeout_secs = secs;
}
}
i += 1;
}
}
"--fail-fast" => {
if i + 1 < args.len() {
if let Ok(n) = args[i + 1].parse::<u32>() {
config.fail_fast = n;
}
i += 1;
}
}
"-s" | "--strategy" => {
if i + 1 < args.len() {
match args[i + 1].parse::<RoutingStrategy>() {
Ok(s) => config.strategy = s,
Err(e) => {
eprintln!("Error: {}", e);
eprintln!("Available strategies: best_price, best_latency, best_availability, round_robin, random, weighted_capacity");
return false;
}
}
i += 1;
}
}
"-b" | "--benchmark" | "--workload" => {
if i + 1 < args.len() {
match args[i + 1].parse::<BenchWorkload>() {
Ok(w) => config.workload = w,
Err(e) => {
eprintln!("Error: {}", e);
return false;
}
}
i += 1;
}
}
"--worker-type" | "--type" => {
if i + 1 < args.len() {
config.worker_type = Some(args[i + 1].clone());
i += 1;
}
}
// Two different asks, and they were one flag. `--remote` is about
// this machine; `--remote-only` is about the broker.
"--remote" => {
config.remote = RemoteMode::OffThisMachine;
}
"--remote-only" | "--peer-dispatch" => {
config.remote = RemoteMode::ForcePeerDispatch;
}
"--budget" => {
if i + 1 < args.len() {
match args[i + 1].parse::<f64>() {
Ok(b) if b > 0.0 => config.budget = b,
_ => {
eprintln!("Error: --budget requires a positive amount of yen");
return false;
}
}
i += 1;
}
}
"--api-key" | "--key" => {
if i + 1 < args.len() {
config.api_key = Some(args[i + 1].clone());
i += 1;
}
}
"--compare" | "--compare-strategies" => {
config.compare_strategies = true;
}
"--list-strategies" => {
print_strategies();
return false;
}
"--json" => {
// stdout becomes the document, so the human view moves to
// stderr and the two can be redirected apart.
config.json = true;
}
"--no-progress" | "--quiet" | "-q" => {
config.no_progress = true;
}
"--no-tui" | "--plain" => {
config.no_tui = true;
}
"-h" | "--help" => {
print_bench_help();
return false;
}
_ => {
if args[i].starts_with("zc://") {
// Naming a node says two things, and bench used to hear
// only the first: which broker to ask, AND that the work
// belongs on that node. Without the second, `zc bench
// zc://node-X` measured whatever the mesh felt like doing.
config.target_node = node_pin_from_uri(&args[i]);
config.broker_url = match super::uri::resolve(&args[i]) {
Ok(u) => u,
Err(e) => {
eprintln!("Error: {}", e);
std::process::exit(1);
}
};
} else if args[i].starts_with("http://") || args[i].starts_with("https://") {
eprintln!(
"Error: plain HTTP URLs are not allowed — use zc://node-name instead."
);
std::process::exit(1);
}
}
}
i += 1;
}
true
}
/// Run what the flags asked for.
fn run_configured(mut config: BenchConfig) {
// Resolve default zc://localhost if it wasn't replaced by an explicit --url
if config.broker_url.starts_with("zc://") {
config.broker_url = match super::uri::resolve(&config.broker_url) {
Ok(u) => u,
Err(e) => {
eprintln!("Error: {}", e);
std::process::exit(1);
}
};
}
if config.compare_strategies {
run_strategy_comparison(config);
return;
}
let json = config.json;
let results = run_benchmark(config);
match json {
true => println!("{}", results.to_json()),
false => results.print_report(),
}
// An interrupted run is an interrupted run, whatever it managed to
// measure: scripts read the status, and 0 would say it ran to completion.
if results.stopped == Some(StopReason::Interrupt) {
std::process::exit(130);
}
}
fn print_strategies() {
println!();
println!("{}", "Available Routing Strategies:".bold());
println!();
let strategies = [
(
"best_price",
"Route to the cheapest worker",
"Minimizes cost",
),
(
"best_latency",
"Route to the fastest responding worker",
"Minimizes response time",
),
(
"best_availability",
"Route to the most available worker",
"Best for high load",
),
(
"round_robin",
"Distribute evenly across all workers",
"Even distribution",
),
("random", "Random worker selection", "Simple load balancing"),
(
"weighted_capacity",
"Weighted by available resources",
"Capacity-aware",
),
];
for (name, desc, note) in strategies {
println!(
" {:20} - {} {}",
name.cyan(),
desc,
format!("({})", note).muted()
);
}
println!();
}
/// Run every strategy in turn and lay the results side by side.
///
/// Three axes, not one: how fast, how much it cost, and how much of the work
/// stayed on hardware you already own. A strategy that is 5% slower while
/// keeping a fifth more work local and spending a quarter as much is the
/// right choice far more often than the fastest one, and that trade is
/// invisible unless the three sit on the same row.
fn run_strategy_comparison(mut base_config: BenchConfig) {
use compare::{free_share, verdicts, Progress, DEFAULT_REQUESTS_EACH, STRATEGIES};
// A comparison runs the workload once per strategy. Six times the
// single-run default took two and a half minutes on a mesh with one slow
// node in it, and printed one line for the whole of it.
if !base_config.requests_explicit {
base_config.requests = DEFAULT_REQUESTS_EACH;
}
arm_interrupt_handler();
let width = term_width();
let interactive = tui::wanted(&base_config);
let progress = Progress::new();
if !interactive {
println!();
print!(
"{}",
boxed(&[
format!("{}", "zc bench --compare".bold().white()),
format!(
"{} · {} concurrent · {} requests each · {} strategies",
base_config.workload.name().yellow(),
base_config.concurrency,
grouped(base_config.requests),
STRATEGIES.len()
),
format!("{} {}", "broker".muted(), base_config.broker_url.cyan()),
])
);
println!();
}
if !crate::vpn::is_loopback_url(base_config.broker_url.trim()) {
let (connected, _, _) = check_wireguard_status(&base_config.broker_url);
if !connected && !interactive {
println!(" {}", wireguard_line(false));
}
}
// The benchmarks run on their own thread so the drawing thread can keep
// the terminal responsive — a comparison is minutes long, and a view that
// cannot be ended is a view nobody wants to start.
let results = match interactive {
true => {
let driver = {
let base = base_config.clone();
let progress = progress.clone();
std::thread::spawn(move || compare::run_all(base, progress))
};
let _ = tui::run_compare(&base_config, &progress);
driver.join().unwrap_or_default()
}
false => {
let ticker = {
let progress = progress.clone();
let config = base_config.clone();
std::thread::spawn(move || compare_ticker(&config, &progress))
};
let results = compare::run_all(base_config.clone(), progress.clone());
let _ = ticker.join();
results
}
};
if results.is_empty() {
println!(" {}", "nothing was measured".muted());
return;
}
let v = verdicts(&results);
let rule = width.saturating_sub(4).min(116);
println!();
println!(" {}", "═".repeat(rule).muted());
println!(" {}", "strategy comparison".bold());
println!(" {}", "═".repeat(rule).muted());
println!();
println!(
" {}{}{}{} {}{}",
pad(&"strategy".muted(), 20, false),
pad(&"req/s".muted(), 9, true),
pad(&"p50".muted(), 11, true),
pad(&"p99".muted(), 10, true),
pad(&"free / billed".muted(), 26, false),
pad(&"spend ¥".muted(), 13, true),
);
for (i, (strategy, r)) in results.iter().enumerate() {
let share = free_share(r);
let (free, billed) = layout::split_bar(
(share * 100.0).round() as u64,
((1.0 - share) * 100.0).round() as u64,
20,
);
let mark = match (Some(i) == v.fastest, Some(i) == v.cheapest) {
(true, true) => " ◀ fastest and cheapest".green().to_string(),
(true, false) => " ◀ fastest".yellow().to_string(),
(false, true) => " ◀ cheapest".green().to_string(),
_ => String::new(),
};
println!(
" {}{}{}{} {}{}{}{}{}",
pad(strategy.as_str(), 20, false),
pad(&format!("{:.1}", r.rps), 9, true),
pad(
&layout::fmt_us(metrics::percentile_us(&r.latencies_us, 50.0)),
11,
true
),
pad(
&layout::fmt_us(metrics::percentile_us(&r.latencies_us, 99.0)),
10,
true
),
free.green(),
billed.yellow(),
pad(&format!("{:.0}%", share * 100.0).muted(), 6, true),
pad(&format!("{:.4}", r.total_charged), 13, true),
mark,
);
}
println!();
let mut named = 0;
for (name, idx, value) in [
(
"fastest",
v.fastest,
v.fastest.map(|i| format!("{:.1} req/s", results[i].1.rps)),
),
(
"cheapest",
v.cheapest,
v.cheapest
.map(|i| format!("¥{:.4}", results[i].1.total_charged)),
),
(
"most free",
v.freest,
v.freest.map(|i| {
format!(
"{:.0}% of the work cost nothing",
free_share(&results[i].1) * 100.0
)
}),
),
] {
if let (Some(i), Some(value)) = (idx, value) {
named += 1;
println!(
" {}{}{}",
pad(&name.muted(), 14, false),
pad(&results[i].0.as_str().cyan().to_string(), 20, false),
value
);
}
}
// No winner is a result, and a more useful one than a winner invented
// from a one-percent gap. Say what the spread actually was.
if named == 0 && results.len() > 1 {
println!(
" {}",
format!(
"no clear winner — all {} strategies within {:.1}% of each other on speed, \
and none of them spent anything",
results.len(),
v.rps_spread * 100.0
)
.muted()
);
} else if v.fastest.is_none() && results.len() > 1 {
println!(
" {}",
format!(
"on speed, nothing to choose: all {} within {:.1}% of each other",
results.len(),
v.rps_spread * 100.0
)
.muted()
);
}
if results.len() < STRATEGIES.len() {
println!();
println!(
" {}",
format!(
"{} of {} strategies measured — the rest were not run.",
results.len(),
STRATEGIES.len()
)
.muted()
);
}
println!();
}
/// The comparison's progress on a terminal that is not taking the TUI: one
/// line per strategy, rewritten in place, so a two-minute wait says what it
/// is waiting for.
fn compare_ticker(config: &BenchConfig, progress: &std::sync::Arc<compare::Progress>) {
let spinner = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
let tty = can_repaint(config);
let mut i = 0usize;
let mut last: Option<String> = None;
while !progress.is_over() {
if let Some((strategy, shared, started)) = progress.running() {
let done = shared.completed.load(Ordering::Relaxed);
let rate = done as f64 / started.elapsed().as_secs_f64().max(0.001);
let line = format!(
" {} {} {} / {} {:.1} req/s",
spinner[i % spinner.len()].to_string().cyan(),
pad(&strategy.as_str().yellow().to_string(), 20, false),
grouped(done),
grouped(config.requests),
rate
);
match tty {
true => {
print!("\r\x1b[2K{line}");
let _ = std::io::stdout().flush();
}
// Not a terminal: one line per strategy, when it changes, so
// a log gets the shape of the run and not a thousand frames.
false => {
let name = strategy.as_str().to_string();
if last.as_deref() != Some(name.as_str()) {
println!(" running {name}…");
last = Some(name);
}
}
}
}
thread::sleep(Duration::from_millis(100));
i += 1;
}
if tty {
print!("\r\x1b[2K");
let _ = std::io::stdout().flush();
}
}
fn print_bench_help() {
println!();
println!("{}: zc bench [OPTIONS] [zc://node-<fp>]", "Usage".bold());
println!();
println!("Benchmark the broker: how fast, how spread, where it ran, what it cost.");
println!();
println!("Naming a node benchmarks THAT node: the run addresses its broker and asks");
println!("for the work to stay there, so the placement table should show one row.");
println!();
println!("{}", "Options:".bold());
println!(" -c, --concurrency <N> Number of concurrent workers (default: 10)");
println!(" -n, --requests <N> Total requests to send (default: 1000)");
println!(" -u, --url <URL> Broker URL (default: zc://localhost)");
println!(" --user <ID> User ID for requests (default: bench-user)");
println!(" --timeout <SECS> Per-request budget (default: 15)");
println!(" --fail-fast <N> Stop after N failures in a row (default: 10, 0 = never)");
println!(" --api-key <KEY> API key for Bearer auth (required in P2P mode)");
println!(
" --budget <YEN> Per-request spend ceiling [default: 1500]. The broker rejects"
);
println!(
" the request when your balance is below this, so raise it only"
);
println!(" for long jobs — a bench job costs ~¥0.15.");
println!(" Sent as budget: remote dispatch mints its voucher");
println!(" for it, and a broker with vouchers on rejects a remote");
println!(" job without a positive one (400 BUDGET_REQUIRED).");
println!(" -b, --benchmark <NAME> Workload to run (default: add)");
println!(" -s, --strategy <NAME> Routing strategy (default: best_price)");
println!(" --remote Keep the work off THIS machine. A broker that is");
println!(" already elsewhere satisfies it as-is; a broker here");
println!(" is made to offer every request to a peer.");
println!(" --remote-only Make the broker skip its own workers and offer to a");
println!(" peer, wherever that broker is. Measures peer dispatch");
println!(" itself — it sends work AWAY from a node you named.");
println!(" --compare Compare all routing strategies (300 requests each unless");
println!(" -n says otherwise; interactive on a terminal)");
println!(" --json Write the run to stdout as JSON; human view to stderr");
println!(" -q, --no-progress No live view (implied when stdout is not a terminal)");
println!(" --no-tui, --plain Stream the run as lines instead of taking the screen");
println!(" --list-strategies List available routing strategies");
println!(" -h, --help Show this help");
println!();
println!("{}", "Workloads:".bold());
println!(" add add(2, 3) — routing overhead baseline (default)");
println!(" fib fib(28) — CPU-intensive recursive Fibonacci");
println!();
println!("{}", "Routing Strategies:".bold());
println!(" best_price Route to cheapest worker");
println!(" best_latency Route to fastest worker");
println!(" best_availability Route to most available worker");
println!(" round_robin Distribute evenly across workers");
println!(" random Random worker selection");
println!(" weighted_capacity Weighted by available resources");
println!();
println!("{}", "Reading the output:".bold());
println!(
" placement Split by what each request was CHARGED: free, or billed in"
);
println!(" yen. Not by transport — a broker reaches every worker");
println!(" registered with it over its own `local` path, wherever");
println!(" that worker's machine is, so `via` reports the transport");
println!(" separately.");
println!(" router said What the routing strategy predicted for that node against");
println!(" what it delivered. A ⚠ means it was out by 2× or more.");
println!(" ctrl-c Stops the run and reports on what finished (exit 130).");
println!(" interactive On a terminal the run takes the screen: q stops and");
println!(" reports, p freezes the numbers without stopping the run,");
println!(" tab switches panel, arrows scroll it.");
println!();
println!("{}", "Examples:".bold());
println!(" zc bench # Default settings (best_price)");
println!(" zc bench -s best_latency # Use fastest worker strategy");
println!(" zc bench --compare # Compare all routing strategies");
println!(" zc bench -c 50 -n 10000 # 50 concurrent, 10k requests");
println!(" zc bench -n 20000 --json > run.json # Track a run between commits");
println!(" zc bench zc://node-fd29ea8fe9ffa2e0 # Benchmark one node, and only it");
println!(" zc bench --remote # Anywhere but this machine");
println!(" zc bench zc://i9 --remote-only # Measure that broker's peer dispatch");
println!();
println!("{}", "Subcommands:".bold());
println!(
" zc bench mesh --workers a,b,c Warmup/calibrate a mesh (JSON; see `bench mesh -h`)"
);
println!();
}
#[cfg(test)]
mod tests {
use super::*;
/// The `RemoteMode` a set of flags parses to, via the real parser.
fn remote_mode_of(flags: &[&str]) -> RemoteMode {
let mut c = BenchConfig::default();
apply_flags(
&mut c,
&flags.iter().map(|s| s.to_string()).collect::<Vec<_>>(),
);
c.remote
}
fn plain(s: &str) -> String {
let mut out = String::new();
let mut chars = s.chars();
while let Some(c) = chars.next() {
if c == '\u{1b}' {
for e in chars.by_ref() {
if e == 'm' {
break;
}
}
} else {
out.push(c);
}
}
out
}
fn a_run() -> BenchResults {
let mut r = BenchResults::new();
r.total_requests = 100_000;
r.completed = 786;
r.successful = 786;
r.duration = Duration::from_millis(20_900);
r.rps = 37.6;
r.started_at = "2026-09-18 15:34 JST".to_string();
r.latencies_us = (0..786).map(|i| 241_000 + (i * 137) % 161_000).collect();
r.rates_per_second = vec![12, 28, 35, 41, 44, 39, 37, 38, 40, 42];
r.stopped = Some(StopReason::Interrupt);
for i in 0..786u64 {
r.ledger.record(
&format!("dd715b7f-w{}", i % 13),
Some("local"),
Some(28.0),
241_000 + (i * 137) % 161_000,
0.0,
Some(2500.0),
);
}
r
}
// ---- the run's own arithmetic ---------------------------------------
#[test]
fn a_rate_is_per_request_sent_not_per_request_asked_for() {
// The old code divided by config.requests, so a run stopped at 786 of
// 100 000 reported 100 000 / 20.9s ≈ 4 785 req/s. It never did that.
let r = a_run();
assert!(
(r.rps - 37.6).abs() < 0.1,
"rps {} should come from the 786 completed",
r.rps
);
}
#[test]
fn default_budget_is_not_a_balance_gate() {
assert!(BenchConfig::default().budget <= 1500.0);
}
// ---- a reply is the answer, or it is a failure ----------------------
fn hex(s: &str) -> Vec<u8> {
(0..s.len())
.step_by(2)
.map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap())
.collect()
}
/// Real bodies, produced by `pickle.dumps(v, protocol=5)` on the same
/// interpreter the workers run.
#[test]
fn the_reader_reads_the_integers_a_worker_actually_sends() {
// add(2, 3) → 5, as BININT1.
assert_eq!(pickled_int(&hex("80054b052e")), Some(5));
// fib(28) → 317811, framed, as BININT.
assert_eq!(
pickled_int(&hex("80059506000000000000004a73d904002e")),
Some(317_811)
);
assert_eq!(pickled_int(&hex("80054b002e")), Some(0));
// Negative BININT is signed, not a huge positive.
assert_eq!(
pickled_int(&hex("80059506000000000000004af9ffffff2e")),
Some(-7)
);
// LONG1 for a value past 32 bits.
assert_eq!(
pickled_int(&hex("80059509000000000000008a060000000000012e")),
Some(1_099_511_627_776)
);
}
/// Anything that is not exactly one integer is not an answer. A tuple
/// `(5,)` begins with the same BININT1 as `5` — stopping at the first
/// value would accept it.
#[test]
fn the_reader_rejects_everything_that_is_not_one_integer() {
for not_an_int in [
"80059505000000000000004b0585942e", // (5,)
"80059505000000000000008c0135942e", // "5"
] {
assert_eq!(pickled_int(&hex(not_an_int)), None, "{not_an_int}");
}
assert_eq!(pickled_int(b""), None);
assert_eq!(pickled_int(b"<html>nope</html>"), None);
// Truncated mid-value: must be None, never a panic or a partial read.
assert_eq!(pickled_int(&hex("8005950600000000000000")), None);
assert_eq!(pickled_int(&hex("80054b")), None);
}
/// The regression this pairs with: bench checked the HTTP status and
/// nothing else, so a proxy error page and a worker traceback were both
/// successful requests with excellent latency.
#[test]
fn a_two_hundred_carrying_an_error_page_is_a_failure() {
let page = b"<html><head><title>500 Unable to connect</title></head>\
<body>Tinyproxy was unable to connect</body></html>";
let why = answer_check(page, 5).unwrap_err();
assert!(why.contains("not a result"), "{why}");
assert!(why.contains("proxy could not reach it"), "{why}");
}
/// The failure that cost an afternoon: a mesh whose workers do not all run
/// the same CPython answers `add(2, 3)` with a pickled `SystemError`, because
/// a cloudpickled function carries raw bytecode and opcode numbers are
/// renumbered between minor versions (`LOAD_FAST` is 124 through 3.12 and 85
/// from 3.13). "the worker raised" sent the reader to the worker's logs, where
/// nothing is wrong; the worker is healthy and running a different Python.
#[test]
fn an_opcode_mismatch_names_the_python_split_not_the_worker() {
// Verbatim body from a 3.13.9 worker sent a payload built on 3.12.
let raised = b"\x80\x05\x95A\x00\x00\x00\x00\x00\x00\x00\x8c\x08builtins\x94\x8c\x0bSystemError\x94\x93\x94\x8c\x1f<string>:4: unknown opcode 124\x94\x85\x94R\x94.";
let why = answer_check(raised, 5).unwrap_err();
assert!(why.contains("not a result"), "{why}");
assert!(
why.contains("Python"),
"the hint must name the interpreter, not just \"the worker raised\": {why}"
);
}
#[test]
fn the_right_answer_passes_and_a_wrong_one_says_both_numbers() {
assert!(answer_check(&hex("80054b052e"), 5).is_ok());
let why = answer_check(&hex("80054b072e"), 5).unwrap_err();
assert_eq!(why, "wrong answer: got 7, want 5");
}
#[test]
fn an_empty_body_is_named_as_such_rather_than_as_a_bad_pickle() {
assert_eq!(answer_check(b"", 5).unwrap_err(), "empty reply");
}
/// Each workload knows what it is asking for; nothing else does.
#[test]
fn every_workload_states_the_answer_it_expects() {
assert_eq!(BenchWorkload::Add.expected(), 5);
assert_eq!(BenchWorkload::Fib.expected(), 317_811);
}
#[test]
fn budget_is_sent_in_the_requirements_payload() {
let c = BenchConfig {
budget: 3.5,
..Default::default()
};
let v = build_requirements(&c);
assert_eq!(v["budget"].as_f64(), Some(3.5));
}
// ---- what a target means, and what --remote means -------------------
/// `zc bench zc://node-<fp>` names a node, and that is a statement about
/// where the work belongs — not only about which broker to ask.
#[test]
fn a_node_uri_is_a_pin() {
assert_eq!(
node_pin_from_uri("zc://node-fd29ea8fe9ffa2e0").as_deref(),
Some("zc://node-fd29ea8fe9ffa2e0")
);
// An explicit port still names the same node.
assert_eq!(
node_pin_from_uri("zc://node-fd29ea8fe9ffa2e0:9000").as_deref(),
Some("zc://node-fd29ea8fe9ffa2e0")
);
assert_eq!(
node_pin_from_uri("zc://node-FD29EA8FE9FFA2E0").as_deref(),
Some("zc://node-fd29ea8fe9ffa2e0"),
"a fingerprint is hex; its case is not part of the identity"
);
}
/// Everything else says which broker to ASK and nothing about placement.
/// Reading `zc://localhost` as a pin would pin every default run.
#[test]
fn everything_that_is_not_a_node_uri_is_not_a_pin() {
for not_a_pin in [
"zc://localhost",
"zc://10.13.13.18:9000",
"zc://i9",
"zc://node-nothex0123456789",
"zc://node-fd29ea",
"http://10.13.13.18:9000",
] {
assert_eq!(node_pin_from_uri(not_a_pin), None, "{not_a_pin}");
}
}
/// The bug: `--remote` was sent to the broker as `remote_only`, which
/// tells THAT broker to skip its own workers. Against a broker out on the
/// mesh — the case where someone naming a node asks for `--remote` — it
/// pushed the work off the very node the run had named, into a
/// first-accept-wins auction across every peer.
#[test]
fn remote_is_about_this_machine_not_about_the_broker() {
// Broker here: the only way to be off this machine is to leave it.
assert!(wants_remote_only(RemoteMode::OffThisMachine, true));
// Broker already elsewhere: it is already satisfied, force nothing.
assert!(!wants_remote_only(RemoteMode::OffThisMachine, false));
}
/// `--remote-only` keeps the old, broker-relative meaning, for measuring
/// the peer-dispatch path itself.
#[test]
fn remote_only_still_forces_peer_dispatch_anywhere() {
assert!(wants_remote_only(RemoteMode::ForcePeerDispatch, true));
assert!(wants_remote_only(RemoteMode::ForcePeerDispatch, false));
assert!(!wants_remote_only(RemoteMode::Off, true));
assert!(!wants_remote_only(RemoteMode::Off, false));
}
#[test]
fn the_two_remote_flags_parse_to_different_modes() {
assert_eq!(remote_mode_of(&["--remote"]), RemoteMode::OffThisMachine);
assert_eq!(
remote_mode_of(&["--remote-only"]),
RemoteMode::ForcePeerDispatch
);
assert_eq!(remote_mode_of(&[]), RemoteMode::Off);
}
/// A pin only goes on the wire for a broker that honours it. An older one
/// reads an unmatched pin as "nothing local" and offers the job to the
/// whole mesh, so sending it there is worse than not pinning at all.
#[test]
fn a_pin_is_only_sent_to_a_broker_that_honours_it() {
assert!(!honours_node_pin(None), "no build disclosed at all");
assert!(!honours_node_pin(Some("0.0.32")));
assert!(!honours_node_pin(Some("0.0.31")));
assert!(honours_node_pin(Some("0.0.33")));
assert!(honours_node_pin(Some("0.0.33-rc1")));
assert!(honours_node_pin(Some("0.1.0")));
assert!(!honours_node_pin(Some("not-a-version")));
}
/// `--remote-only` is the explicit "measure the auction" flag. Naming a
/// node beside it says which broker to ask, so the pin is released rather
/// than the combination refused — otherwise there is no way to measure a
/// named broker's peer dispatch at all.
#[test]
fn remote_only_releases_the_pin_instead_of_contradicting_it() {
let c = BenchConfig {
target_node: Some("zc://node-fd29ea8fe9ffa2e0".to_string()),
remote: RemoteMode::ForcePeerDispatch,
send_node_pin: true,
..Default::default()
};
assert_eq!(c.effective_pin(), None);
assert!(build_requirements(&c)["target_node"].is_null());
// `--remote` does not release it: it asks for the work to be off this
// machine, which a named node already satisfies.
let kept = BenchConfig {
remote: RemoteMode::OffThisMachine,
..c
};
assert_eq!(kept.effective_pin(), Some("zc://node-fd29ea8fe9ffa2e0"));
}
#[test]
fn the_pin_reaches_the_requirements_only_when_it_is_honoured() {
let pinned = BenchConfig {
target_node: Some("zc://node-fd29ea8fe9ffa2e0".to_string()),
send_node_pin: true,
..Default::default()
};
assert_eq!(
build_requirements(&pinned)["target_node"].as_str(),
Some("zc://node-fd29ea8fe9ffa2e0")
);
let withheld = BenchConfig {
send_node_pin: false,
..pinned.clone()
};
assert!(build_requirements(&withheld)["target_node"].is_null());
}
/// Without this the broker's peer-dispatch deadline falls back to a blind
/// 30s and its per-peer offer to 310s, both of them past the 15s the
/// client will actually wait — so a slow dispatch was guaranteed to be a
/// client-side timeout while the broker kept working on it.
#[test]
fn the_requirements_say_how_long_the_caller_will_wait() {
let c = BenchConfig {
timeout_secs: 15,
..Default::default()
};
assert_eq!(build_requirements(&c)["timeout_secs"].as_f64(), Some(15.0));
}
#[test]
fn failure_reasons_rank_by_count_then_text() {
let mut r = BenchResults::new();
r.failure_reasons.insert("b".into(), 2);
r.failure_reasons.insert("a".into(), 2);
r.failure_reasons.insert("c".into(), 9);
let ranked = r.ranked_failure_reasons();
assert_eq!(ranked[0].0, "c");
assert_eq!(ranked[1].0, "a");
}
#[test]
fn pct_zero_denominator_is_zero_not_nan() {
assert_eq!(pct(0, 0), 0.0);
assert_eq!(pct(1, 4), 25.0);
}
#[test]
fn effective_concurrency_never_zero() {
assert_eq!(effective_concurrency(0), 1);
assert_eq!(effective_concurrency(7), 7);
}
#[test]
fn requests_per_worker_no_divide_by_zero() {
let c = effective_concurrency(0);
assert_eq!(100u64 / c as u64, 100);
}
#[test]
fn a_run_that_only_fails_is_stopped_at_once() {
assert!(run_is_failing(10, 10, 10, 10));
}
#[test]
fn a_run_that_mostly_fails_is_stopped_even_though_some_succeed() {
assert!(run_is_failing(40, 30, 1, 10));
}
#[test]
fn the_guard_can_be_turned_off() {
assert!(!run_is_failing(1000, 1000, 1000, 0));
}
#[test]
fn an_empty_broker_is_waited_for_then_given_up_on() {
assert_eq!(worker_check(Some(0), Duration::ZERO), WorkerCheck::Waiting);
assert_eq!(
worker_check(Some(0), WAIT_FOR_WORKERS + Duration::from_secs(1)),
WorkerCheck::GiveUp
);
}
#[test]
fn a_broker_with_workers_starts_at_once() {
assert_eq!(worker_check(Some(3), Duration::ZERO), WorkerCheck::Ready(3));
}
#[test]
fn an_unreadable_answer_does_not_block_the_run() {
assert_eq!(worker_check(None, Duration::ZERO), WorkerCheck::Ready(0));
}
#[test]
fn the_connectivity_line_carries_no_address() {
for line in [wireguard_line(true), wireguard_line(false)] {
assert!(!line.contains('.'), "{line:?} looks like it names a host");
}
}
#[test]
fn reads_the_charge_the_broker_actually_billed() {
assert_eq!(parse_charged(r#"{"charged":0.004}"#), Some(0.004));
assert_eq!(parse_charged(r#"{"charged":0}"#), Some(0.0));
assert_eq!(parse_charged(r#"{"other":1}"#), None);
assert_eq!(parse_charged("not json"), None);
}
// ---- what the report says --------------------------------------------
#[test]
fn the_report_states_the_conditions_it_ran_under() {
let text = plain(&report::render(&a_run(), 120));
assert!(text.contains("best_price"), "the strategy");
assert!(text.contains("2026-09-18 15:34 JST"), "when");
assert!(text.contains("stopped by ctrl-c"), "how it ended");
assert!(text.contains("786"), "what it actually did");
}
#[test]
fn the_report_separates_free_work_from_billed_work() {
let text = plain(&report::render(&a_run(), 120));
assert!(text.contains("free"), "{text}");
assert!(text.contains("billed"), "{text}");
assert!(
text.contains("cost nothing"),
"the free side is spelled out"
);
}
/// A pinned run that stayed put says so, in one line, rather than making
/// the reader count rows in the placement table.
#[test]
fn a_pinned_run_that_held_says_so() {
let mut r = BenchResults::new();
r.config.target_node = Some("zc://node-fd29ea8fe9ffa2e0".to_string());
r.completed = 3;
r.successful = 3;
for _ in 0..3 {
r.ledger.record(
"worker-zc-worker-x399-2",
Some("local"),
Some(3.6),
3_000_000,
0.001,
None,
);
}
let text = plain(&report::render(&r, 120));
assert!(text.contains("as pinned"), "{text}");
}
/// And a pinned run that scattered says THAT, naming the nodes — which is
/// the whole of the bug report this fix came from: `zc bench
/// zc://node-fd29… --remote` spread 11 requests over five nodes and the
/// report never once said the target had been missed.
#[test]
fn a_pinned_run_that_scattered_is_called_out() {
let mut r = BenchResults::new();
r.config.target_node = Some("zc://node-fd29ea8fe9ffa2e0".to_string());
r.completed = 2;
r.successful = 2;
for worker in ["worker-zc-worker-x399-2", "zc-broker-0"] {
r.ledger.record(
worker,
Some("subscribed"),
Some(3.6),
3_000_000,
0.001,
None,
);
}
let text = plain(&report::render(&r, 120));
assert!(
text.contains("Pinned to zc://node-fd29ea8fe9ffa2e0 but the work ran on 2 nodes"),
"{text}"
);
// The pointer at the build skew survives; `zc brokers` itself may be
// split by the note's own wrapping, so match the sentence around it.
assert!(
text.contains("names each broker's build"),
"point at the build skew: {text}"
);
}
/// A node that ran some work free and some billed is still one node; the
/// ledger keys on (placement, node), so the naive count says two.
#[test]
fn one_node_on_both_sides_of_the_split_is_still_one_node() {
let mut led = metrics::Ledger::new();
led.record("worker-zc-worker-x399-2", None, None, 1_000, 0.0, None);
led.record(
"worker-zc-worker-x399-2",
None,
Some(3.6),
1_000,
0.001,
None,
);
assert_eq!(led.nodes_touched(), vec![("x399".to_string(), 2)]);
}
#[test]
fn a_run_that_cost_nothing_says_so_in_words() {
let text = plain(&report::render(&a_run(), 120));
assert!(
text.contains("Nothing was billed"),
"a free run must say so: {text}"
);
}
#[test]
fn a_run_that_spent_money_reports_the_amount() {
let mut r = a_run();
for i in 0..100u64 {
r.ledger.record(
"i9-w0",
Some("quic"),
Some(3.6),
400_000,
0.0012,
Some(390.0),
);
r.total_charged += 0.0012;
let _ = i;
}
let text = plain(&report::render(&r, 120));
assert!(text.contains("0.1200"), "the total charged: {text}");
assert!(!text.contains("Nothing was billed"));
}
#[test]
fn a_router_that_was_badly_wrong_is_called_out_in_the_notes() {
let text = plain(&report::render(&a_run(), 120));
assert!(text.contains("notes"), "there is a notes section");
assert!(
text.contains("9.5×") || text.contains("pessimistic"),
"the 2.5s-vs-264ms gap must be stated: {text}"
);
}
#[test]
fn the_distribution_shows_a_shape_rather_than_one_full_bar() {
// The bug this redesign exists for: 786 samples between 241ms and
// 402ms used to land in a single ≤500ms bucket.
let text = plain(&report::render(&a_run(), 120));
let rows: Vec<&str> = text
.lines()
.filter(|l| l.trim_start().starts_with('≤') || l.trim_start().starts_with('>'))
.collect();
let with_bars = rows.iter().filter(|l| l.contains('█')).count();
assert!(
with_bars >= 4,
"only {with_bars} bucket(s) drew a bar; the chart says nothing:\n{text}"
);
}
#[test]
fn a_run_of_one_closed_second_draws_no_throughput_distribution() {
// The other half of `one_second_of_samples_is_not_a_distribution`:
// `Buckets` will happily bucket a single number, so declining is the
// view's job. Ten per-second samples are worth a chart; one is the
// 1.3 s run that drew "≤2295 req/s" and "≤2296 req/s" out of one
// number, and it gets no section at all.
let full = plain(&report::render(&a_run(), 120));
assert!(
full.contains("throughput distribution"),
"ten per-second samples are a shape worth drawing:\n{full}"
);
let mut short = a_run();
short.rates_per_second = vec![2295];
let text = plain(&report::render(&short, 120));
assert!(
!text.contains("throughput distribution"),
"one closed second was drawn as a distribution:\n{text}"
);
}
#[test]
fn no_report_line_is_wider_than_the_terminal() {
for width in [60usize, 80, 100, 120, 160] {
let text = plain(&report::render(&a_run(), width));
for line in text.lines() {
assert!(
line.chars().count() <= width,
"at {width} cols this line is {}: {line:?}",
line.chars().count()
);
}
}
}
// ---- what the live view says -----------------------------------------
fn a_live_view<'a>(
samples: &'a [u64],
ledger: &'a Ledger,
rates: &'a RateRing,
width: usize,
) -> live::LiveView<'a> {
live::LiveView {
spin: '⠸',
done: 786,
total: 100_000,
ok: 786,
failed: 0,
elapsed: Duration::from_secs(20),
samples,
ledger,
rates,
width,
}
}
#[test]
fn the_live_block_shows_progress_shape_and_where_the_work_went() {
let r = a_run();
let mut ring = RateRing::new();
for (i, _) in r.rates_per_second.iter().enumerate() {
ring.sample((i as u64 + 1) * 40, Duration::from_secs(i as u64 + 1));
}
let text = plain(&a_live_view(&r.latencies_us, &r.ledger, &ring, 120).render());
assert!(text.contains("786"), "how far");
assert!(text.contains("req/s"), "how fast");
assert!(text.contains("placement"), "where");
assert!(text.contains("free"), "which side of the split");
assert!(text.contains("ctrl-c"), "how to stop");
}
#[test]
fn the_live_block_says_so_before_anything_has_been_routed() {
let ledger = Ledger::new();
let ring = RateRing::new();
let text = plain(&a_live_view(&[], &ledger, &ring, 120).render());
assert!(text.contains("nothing has been routed yet"), "{text}");
}
#[test]
fn a_very_wide_terminal_does_not_scatter_the_block() {
// Every panel has a natural size, so past a point the block stops
// growing — but the progress line right-aligned its rate against the
// full width, leaving the rates eighty columns adrift of a block only
// a hundred wide.
let r = a_run();
let mut ring = RateRing::new();
for i in 1..=20u64 {
ring.sample(i * 40, Duration::from_secs(i));
}
let text = plain(&a_live_view(&r.latencies_us, &r.ledger, &ring, 300).render());
let widest = text.lines().map(|l| l.chars().count()).max().unwrap_or(0);
assert!(widest <= 120, "at 300 columns the block spread to {widest}");
}
#[test]
fn no_live_line_is_wider_than_the_terminal() {
let r = a_run();
let mut ring = RateRing::new();
for i in 0..30u64 {
ring.sample(i * 40, Duration::from_secs(i));
}
for width in [60usize, 72, 80, 96, 104, 120, 160] {
let text = plain(&a_live_view(&r.latencies_us, &r.ledger, &ring, width).render());
for line in text.lines() {
assert!(
line.chars().count() <= width,
"at {width} cols this line is {}: {line:?}",
line.chars().count()
);
}
}
}
#[test]
fn the_live_block_shows_both_sides_of_the_split_when_there_are_both() {
let mut ledger = Ledger::new();
for _ in 0..70 {
ledger.record("a-w0", Some("local"), Some(28.0), 260_000, 0.0, Some(250.0));
}
for _ in 0..30 {
ledger.record(
"i9-w0",
Some("quic"),
Some(3.6),
410_000,
0.0012,
Some(400.0),
);
}
let ring = RateRing::new();
let text = plain(&a_live_view(&[260_000; 70], &ledger, &ring, 120).render());
assert!(text.contains("free"), "{text}");
assert!(text.contains("billed"), "{text}");
assert!(text.contains("yen"), "the billed side carries a price");
assert!(text.contains("0.0360"), "the running spend: {text}");
assert!(text.contains("i9"), "the billed node is named: {text}");
}
#[test]
fn the_live_block_projects_what_the_whole_run_will_cost() {
let mut ledger = Ledger::new();
for _ in 0..786 {
ledger.record("i9-w0", Some("quic"), Some(3.6), 400_000, 0.001, None);
}
let ring = RateRing::new();
let text = plain(&a_live_view(&[400_000; 786], &ledger, &ring, 120).render());
assert!(
text.contains("projected"),
"a 44-minute run's cost is worth knowing at minute one: {text}"
);
}
#[test]
fn the_json_and_the_report_describe_the_same_run() {
let r = a_run();
let doc: serde_json::Value = serde_json::from_str(&r.to_json()).unwrap();
assert_eq!(doc["outcome"]["completed"], r.completed);
assert_eq!(doc["outcome"]["stop_reason"], "sigint");
assert_eq!(doc["placement"]["free"]["requests"], 786);
assert_eq!(doc["placement"]["billed"]["requests"], 0);
}
/// Not an assertion — a way to look at the thing. `cargo test --bin zc
/// preview -- --ignored --nocapture` prints every view at every width.
#[test]
#[ignore]
fn preview() {
let _colour = layout::colour_lock();
colored::control::set_override(true);
let r = a_run();
let mut mixed = a_run();
for i in 0..340u64 {
mixed.ledger.record(
&format!("worker-zc-worker-i9-{}", i % 4),
Some("quic"),
Some(3.6),
410_000 + (i * 971) % 500_000,
0.00048,
Some(408.0),
);
mixed.latencies_us.push(410_000 + (i * 971) % 500_000);
mixed.total_charged += 0.00048;
}
mixed.completed += 340;
mixed.successful += 340;
let mut ring = RateRing::new();
for i in 1..=40u64 {
ring.sample(i * 37 + (i * i) % 11, Duration::from_secs(i));
}
for width in [120usize, 96, 80] {
println!("\n\n╔══ LIVE {width} cols ═══════════════════════════════");
println!(
"{}",
live::LiveView {
spin: '⠸',
done: mixed.completed,
total: 100_000,
ok: mixed.successful,
failed: 17,
elapsed: Duration::from_secs(312),
samples: &mixed.latencies_us,
ledger: &mixed.ledger,
rates: &ring,
width,
}
.render()
);
println!("╔══ REPORT (all local, ctrl-c) {width} cols ══════════");
print!("{}", report::render(&r, width));
println!("╔══ REPORT (mixed) {width} cols ══════════════════════");
print!("{}", report::render(&mixed, width));
}
colored::control::unset_override();
}
#[test]
fn no_view_relies_on_the_faint_attribute() {
// SGR 2 renders as nothing on some terminals (macOS Terminal among
// them), and this view puts every label, unit and column heading in
// secondary weight — so on such a terminal the whole scaffold went
// missing and left bars with no axis.
let _colour = layout::colour_lock();
colored::control::set_override(true);
let r = a_run();
let mut ring = RateRing::new();
for i in 1..=10u64 {
ring.sample(i * 40, Duration::from_secs(i));
}
let views = [
report::render(&r, 120),
a_live_view(&r.latencies_us, &r.ledger, &ring, 120).render(),
];
for v in views {
assert!(
!v.contains("\x1b[2m"),
"a view still emits SGR 2; use .muted() instead of .dimmed()"
);
}
colored::control::unset_override();
}
#[test]
fn every_label_survives_into_the_rendered_report() {
// The failure this guards was silent: the figures were all there and
// every word naming them was gone.
let _colour = layout::colour_lock();
colored::control::set_override(true);
let text = report::render(&a_run(), 120);
for label in [
"min", "p50", "p90", "p99", "max", "mean", "peak", "stdev", "cv", "free", "billed",
"spent", "per req", "node", "req", "share", "spend", "notes",
] {
assert!(
plain(&text).contains(label),
"the report lost the label {label:?}"
);
}
colored::control::unset_override();
}
#[test]
fn the_broker_is_named_without_an_address() {
let mut r = BenchResults::new();
r.config.broker_url = "http://127.0.0.1:9000".into();
assert_eq!(r.broker_label(), "broker 127.0.0.1:9000");
assert!(!r.broker_label().contains("http"));
}
}