zc2 0.0.42

P2P compute broker with credit-based billing, WAL, and broker mesh support
#!/usr/bin/env python3
"""
Local A/B of static vs adaptive routing (ZC-0011, zc#438).

Starts fresh mock workers and a fresh `zc broker` for every arm on
127.0.0.1, drives the broker's `/execute` with a closed loop of concurrent
clients, and prints a markdown table of latency and the per-worker split for
each arm. Nothing leaves the machine: no staging broker, no hub, no mesh.
Stdlib only.

Each mock worker is a single-threaded HTTP server, i.e. a FIFO queue: a
request sent to a busy worker waits behind the one it is running. Service time
is one value per worker with +-25% uniform jitter, done as a sleep so the
workers do not compete for CPU. `--cold-ms` makes the first request on each
fresh worker slower, as a freshly started Python worker is while it imports
what the payload needs: with it, the pre-#438 policy starves identical
workers exactly as the 2026-10-06 staging A/B saw (p50 about 2x static).

Usage (after `cargo build -p zc2 --bin zc`; the broker binary is `$ZC`, else
`$CARGO_TARGET_DIR/debug/zc`, else `target/debug/zc`):

  # homogeneous: the staging A/B's two workloads on 4 identical workers
  ZAKURO_HOME=$(mktemp -d) scripts/bench_adaptive_routing.py --workers 4,4,4,4 \\
      --concurrency 6 --requests 1000 --warmup 20 --rounds 2 --cold-ms 500
  ZAKURO_HOME=$(mktemp -d) scripts/bench_adaptive_routing.py --workers 120,120,120,120 \\
      --concurrency 8 --requests 240 --warmup 16 --rounds 2 --cold-ms 500

  # heterogeneous: two fast workers, two slow ones
  ZAKURO_HOME=$(mktemp -d) scripts/bench_adaptive_routing.py --workers 20,20,200,200 \\
      --concurrency 4 --requests 400 --warmup 20 --rounds 2 --cold-ms 500

`--rounds R` alternates the arms R times (S A A S ...) so drift on the host
hits both. Per-round lines (stderr) show each round's split, the broker's
`route` and `forward` phase times, and the routing reason.
"""

from __future__ import annotations

import argparse
import json
import os
import random
import signal
import socket
import subprocess
import sys
import tempfile
import threading
import time
import urllib.error
import urllib.request
from http.server import BaseHTTPRequestHandler, HTTPServer

HOST = "127.0.0.1"


# ── mock worker ──────────────────────────────────────────────────────────────


def serve_worker(port: int, service_ms: float, name: str, cold_ms: float) -> None:
    """A single-threaded zakuro-shaped worker: /health, /info, /execute.

    The first `/execute` also pays `cold_ms`, as a freshly started Python
    worker does while it imports what the first payload needs."""
    rng = random.Random(port)
    cold = [cold_ms]

    class Handler(BaseHTTPRequestHandler):
        def _send(self, code: int, body: dict) -> None:
            raw = json.dumps(body).encode()
            self.send_response(code)
            self.send_header("Content-Type", "application/json")
            self.send_header("Content-Length", str(len(raw)))
            self.end_headers()
            self.wfile.write(raw)

        def do_GET(self) -> None:
            path = self.path.rstrip("/")
            if path == "/health":
                self._send(200, {"status": "healthy"})
            elif path == "/info":
                self._send(
                    200,
                    {
                        "name": name,
                        "worker_type": "cpu",
                        "resources": {
                            "cpus_total": 1.0,
                            "cpus_available": 1.0,
                            "memory_total": 1 << 30,
                            "memory_available": 1 << 30,
                            "gpus_total": 0,
                            "gpus_available": 0,
                        },
                    },
                )
            else:
                self._send(404, {"error": "not found"})

        def do_POST(self) -> None:
            n = int(self.headers.get("Content-Length", 0) or 0)
            if n:
                self.rfile.read(n)
            took = service_ms * (0.75 + 0.5 * rng.random()) + cold[0]
            cold[0] = 0.0
            time.sleep(took / 1000.0)
            self._send(200, {"worker": name, "service_ms": took})

        def log_message(self, *_args) -> None:
            pass

    HTTPServer((HOST, port), Handler).serve_forever()


# ── helpers ──────────────────────────────────────────────────────────────────


def free_port(avoid: set[int]) -> int:
    """A free TCP port on 127.0.0.1 whose +1 is also free (the broker's QUIC
    port) and that is outside 9000-9010, where IPv6 listeners can shadow it."""
    while True:
        with socket.socket() as s:
            s.bind((HOST, 0))
            p = s.getsockname()[1]
        if 9000 <= p <= 9011 or p in avoid or p + 1 in avoid:
            continue
        try:
            with socket.socket() as s:
                s.bind((HOST, p + 1))
        except OSError:
            continue
        return p


def get_json(url: str, timeout: float = 2.0):
    with urllib.request.urlopen(url, timeout=timeout) as r:
        return json.loads(r.read())


def zc_binary() -> str:
    if os.environ.get("ZC"):
        return os.environ["ZC"]
    target = os.environ.get("CARGO_TARGET_DIR") or os.path.join(
        os.path.dirname(os.path.abspath(__file__)), "..", "target"
    )
    return os.path.join(target, "debug", "zc")


def pct(xs: list[float], p: float) -> float:
    """Nearest-rank percentile, as the policy tests use."""
    s = sorted(xs)
    if not s:
        return float("nan")
    k = max(1, min(len(s), int(-(-p * len(s) // 100))))
    return s[k - 1]


# ── one arm ──────────────────────────────────────────────────────────────────


def start_broker(adaptive: bool, worker_ports: list[int], avoid: set[int]):
    port = free_port(avoid)
    home = tempfile.mkdtemp(prefix="zc-bench-home-")
    # Nothing from the caller's zakuro setup may leak in (a hub URL would make
    # the broker validate the bench key against a real hub).
    env = {k: v for k, v in os.environ.items() if not k.startswith(("ZAKURO_", "ZC_"))}
    env.update(
        {
            "ZAKURO_HOME": home,
            "HOME": home,
            "ZAKURO_ADAPTIVE_ROUTING": "1" if adaptive else "0",
            "ZAKURO_WORKERS": ",".join(f"{HOST}:{p}" for p in worker_ports),
            "ZAKURO_SCAN_RANGE": f"{min(worker_ports)}-{max(worker_ports)}",
            "ZAKURO_SCAN_INTERVAL": "2",
            "ZAKURO_P2P": "false",
            "ZAKURO_PEERS": "",
        }
    )
    log_path = os.path.join(home, "broker.log")
    with open(log_path, "w") as log:  # the child keeps its own copy of the fd
        proc = subprocess.Popen(
            [zc_binary(), "broker", str(port)],
            env=env,
            stdout=log,
            stderr=subprocess.STDOUT,
            start_new_session=True,
        )
    url = f"http://{HOST}:{port}"
    deadline = time.time() + 60
    while time.time() < deadline:
        if proc.poll() is not None:
            raise SystemExit(f"broker exited early, see {log_path}")
        try:
            ws = get_json(url + "/workers")
            ws = ws if isinstance(ws, list) else ws.get("workers", [])
            if sum(1 for w in ws if w.get("status") == "healthy") >= len(worker_ports):
                return proc, url, log_path
        except (OSError, ValueError):
            pass  # not up yet
        time.sleep(0.5)
    proc.kill()
    raise SystemExit(f"broker never saw {len(worker_ports)} healthy workers, see {log_path}")


def drive(url: str, requests: int, concurrency: int) -> list[dict]:
    lock = threading.Lock()
    issued = [0]
    rows: list[dict] = []
    body = json.dumps({"op": "bench"}).encode()

    def client() -> None:
        while True:
            with lock:
                if issued[0] >= requests:
                    return
                issued[0] += 1
            req = urllib.request.Request(
                url + "/execute",
                data=body,
                method="POST",
                headers={
                    "Content-Type": "application/json",
                    # Standalone ledger (no hub): any well-formed zk_ key is
                    # accepted and nothing is billed.
                    "Authorization": "Bearer zk_bench_00000000",
                    # Per-phase timings back in X-Zakuro-Phases.
                    "X-Zakuro-Trace": "1",
                },
            )
            t = time.perf_counter()
            rec: dict = {}
            try:
                with urllib.request.urlopen(req, timeout=60) as r:
                    reply = json.loads(r.read())
                    phases = dict(
                        kv.split("=", 1) for kv in (r.headers.get("X-Zakuro-Phases") or "").split() if "=" in kv
                    )
                    rec.update(
                        ok=True,
                        worker=reply.get("worker"),
                        reason=r.headers.get("X-Zakuro-Reason"),
                        route_ms=float(phases.get("route", "nan")),
                        forward_ms=float(phases.get("forward", "nan")),
                    )
            except urllib.error.HTTPError as e:
                rec.update(ok=False, err=f"{e.code} {e.read()[:120]!r}")
            except Exception as e:  # noqa: BLE001
                rec.update(ok=False, err=str(e)[:120])
            rec["ms"] = (time.perf_counter() - t) * 1000
            with lock:
                rows.append(rec)

    threads = [threading.Thread(target=client) for _ in range(concurrency)]
    for th in threads:
        th.start()
    for th in threads:
        th.join()
    return rows


def start_workers(service: list[float], names: list[str], cold_ms: float, avoid: set[int]):
    ports: list[int] = []
    for _ in service:
        p = free_port(avoid)
        avoid.update({p, p + 1})
        ports.append(p)
    procs = [
        subprocess.Popen(
            [sys.executable, os.path.abspath(__file__), "--serve-worker", str(port), str(ms), name, str(cold_ms)],
            start_new_session=True,
        )
        for port, ms, name in zip(ports, service, names)
    ]
    for port in ports:
        for _ in range(100):
            try:
                get_json(f"http://{HOST}:{port}/health", 0.5)
                break
            except (OSError, ValueError):
                time.sleep(0.1)
    return procs, ports


def stop(procs) -> None:
    for p in procs:
        try:
            os.killpg(p.pid, signal.SIGTERM)
        except ProcessLookupError:
            continue
        try:
            p.wait(timeout=10)
        except subprocess.TimeoutExpired:
            os.killpg(p.pid, signal.SIGKILL)


def run_arm(adaptive: bool, args, service, names, avoid) -> list[dict]:
    """One arm on fresh workers and a fresh broker, as the staging A/B's
    agent restart gave it: nothing learned in one arm carries into the next."""
    workers, ports = start_workers(service, names, args.cold_ms, avoid)
    try:
        broker, url, log = start_broker(adaptive, ports, avoid)
        try:
            drive(url, args.warmup, args.concurrency)  # discarded
            rows = drive(url, args.requests, args.concurrency)
            seen = get_json(url + "/workers")
            seen = seen if isinstance(seen, list) else seen.get("workers", [])
            print(
                "  broker's view: "
                + ", ".join(
                    f"{w.get('name')} avg={w.get('avg_latency_ms', 0):.0f}ms"
                    for w in sorted(seen, key=lambda w: w.get("name") or "")
                ),
                file=sys.stderr,
            )
        finally:
            stop([broker])
    finally:
        stop(workers)
    reasons = sorted({(r.get("reason") or "?").split(":")[0] for r in rows if r["ok"]})
    bad = [r for r in rows if not r["ok"]]
    if bad:
        print(f"  {len(bad)} failed requests, e.g. {bad[0].get('err')} (log {log})", file=sys.stderr)
    ok = [r for r in rows if r["ok"]]
    lat = [r["ms"] for r in ok]
    route = [r["route_ms"] for r in ok if r["route_ms"] == r["route_ms"]]
    fwd = [r["forward_ms"] for r in ok if r["forward_ms"] == r["forward_ms"]]
    print(
        f"  round {'adaptive' if adaptive else 'static  '}: n={len(ok)} p50={pct(lat, 50):.1f} "
        f"p99={pct(lat, 99):.1f} split={split_of(ok, names)} route p50/p99={pct(route, 50):.2f}/"
        f"{pct(route, 99):.2f} forward p50/p99={pct(fwd, 50):.1f}/{pct(fwd, 99):.1f} reasons={reasons}",
        file=sys.stderr,
    )
    return rows


def split_of(ok: list[dict], names: list[str]) -> str:
    return " / ".join(f"{100 * sum(1 for r in ok if r['worker'] == n) / max(1, len(ok)):.0f}" for n in names)


def main() -> None:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--workers", default="4,4,4,4", help="service ms per worker, comma-separated")
    ap.add_argument("--concurrency", type=int, default=6)
    ap.add_argument("--requests", type=int, default=600, help="measured requests per arm per round")
    ap.add_argument("--warmup", type=int, default=60, help="discarded requests per arm per round")
    ap.add_argument("--rounds", type=int, default=2)
    ap.add_argument(
        "--cold-ms",
        type=float,
        default=0.0,
        help="extra time the first request on each fresh worker takes (Python import cost)",
    )
    ap.add_argument("--label", default=None, help="workload label for the table")
    args = ap.parse_args()

    service = [float(x) for x in args.workers.split(",") if x.strip()]
    names = [f"w{i}-{int(ms)}ms" for i, ms in enumerate(service)]
    avoid: set[int] = set()
    order: list[bool] = []
    for r in range(args.rounds):
        order += [False, True] if r % 2 == 0 else [True, False]
    rounds: dict[bool, list[list[dict]]] = {False: [], True: []}
    for adaptive in order:
        rounds[adaptive].append(run_arm(adaptive, args, service, names, avoid))

    def off_uniform(rows: list[dict]) -> float:
        ok = [r for r in rows if r["ok"]]
        return max(abs(sum(1 for r in ok if r["worker"] == n) / max(1, len(ok)) - 1 / len(names)) for n in names)

    label = args.label or f"workers {args.workers} ms, c={args.concurrency}"
    print(
        f"\n| workload | arm | requests | errors | mean ms | p50 ms | p99 ms | max ms "
        f"| split ({', '.join(names)}) | most uneven round |"
    )
    print("|---|---|---|---|---|---|---|---|---|---|")
    for adaptive in (False, True):
        rows = [r for rs in rounds[adaptive] for r in rs]
        ok = [r for r in rows if r["ok"]]
        lat = [r["ms"] for r in ok]
        worst = max(rounds[adaptive], key=off_uniform)
        print(
            f"| {label} | {'adaptive' if adaptive else 'static'} | {len(rows)} | {len(rows) - len(ok)} "
            f"| {sum(lat) / max(1, len(lat)):.1f} | {pct(lat, 50):.1f} | {pct(lat, 99):.1f} "
            f"| {max(lat, default=float('nan')):.1f} | {split_of(ok, names)} % "
            f"| {split_of([r for r in worst if r['ok']], names)} % |"
        )


if __name__ == "__main__":
    if len(sys.argv) == 6 and sys.argv[1] == "--serve-worker":
        serve_worker(int(sys.argv[2]), float(sys.argv[3]), sys.argv[4], float(sys.argv[5]))
    else:
        main()