from __future__ import annotations
import argparse
import json
import os
import random
import signal
import socket
import subprocess
import sys
import tempfile
import threading
import time
import urllib.error
import urllib.request
from http.server import BaseHTTPRequestHandler, HTTPServer
HOST = "127.0.0.1"
def serve_worker(port: int, service_ms: float, name: str, cold_ms: float) -> None:
rng = random.Random(port)
cold = [cold_ms]
class Handler(BaseHTTPRequestHandler):
def _send(self, code: int, body: dict) -> None:
raw = json.dumps(body).encode()
self.send_response(code)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(raw)))
self.end_headers()
self.wfile.write(raw)
def do_GET(self) -> None:
path = self.path.rstrip("/")
if path == "/health":
self._send(200, {"status": "healthy"})
elif path == "/info":
self._send(
200,
{
"name": name,
"worker_type": "cpu",
"resources": {
"cpus_total": 1.0,
"cpus_available": 1.0,
"memory_total": 1 << 30,
"memory_available": 1 << 30,
"gpus_total": 0,
"gpus_available": 0,
},
},
)
else:
self._send(404, {"error": "not found"})
def do_POST(self) -> None:
n = int(self.headers.get("Content-Length", 0) or 0)
if n:
self.rfile.read(n)
took = service_ms * (0.75 + 0.5 * rng.random()) + cold[0]
cold[0] = 0.0
time.sleep(took / 1000.0)
self._send(200, {"worker": name, "service_ms": took})
def log_message(self, *_args) -> None:
pass
HTTPServer((HOST, port), Handler).serve_forever()
def free_port(avoid: set[int]) -> int:
while True:
with socket.socket() as s:
s.bind((HOST, 0))
p = s.getsockname()[1]
if 9000 <= p <= 9011 or p in avoid or p + 1 in avoid:
continue
try:
with socket.socket() as s:
s.bind((HOST, p + 1))
except OSError:
continue
return p
def get_json(url: str, timeout: float = 2.0):
with urllib.request.urlopen(url, timeout=timeout) as r:
return json.loads(r.read())
def zc_binary() -> str:
if os.environ.get("ZC"):
return os.environ["ZC"]
target = os.environ.get("CARGO_TARGET_DIR") or os.path.join(
os.path.dirname(os.path.abspath(__file__)), "..", "target"
)
return os.path.join(target, "debug", "zc")
def pct(xs: list[float], p: float) -> float:
s = sorted(xs)
if not s:
return float("nan")
k = max(1, min(len(s), int(-(-p * len(s) // 100))))
return s[k - 1]
def start_broker(adaptive: bool, worker_ports: list[int], avoid: set[int]):
port = free_port(avoid)
home = tempfile.mkdtemp(prefix="zc-bench-home-")
env = {k: v for k, v in os.environ.items() if not k.startswith(("ZAKURO_", "ZC_"))}
env.update(
{
"ZAKURO_HOME": home,
"HOME": home,
"ZAKURO_ADAPTIVE_ROUTING": "1" if adaptive else "0",
"ZAKURO_WORKERS": ",".join(f"{HOST}:{p}" for p in worker_ports),
"ZAKURO_SCAN_RANGE": f"{min(worker_ports)}-{max(worker_ports)}",
"ZAKURO_SCAN_INTERVAL": "2",
"ZAKURO_P2P": "false",
"ZAKURO_PEERS": "",
}
)
log_path = os.path.join(home, "broker.log")
with open(log_path, "w") as log: proc = subprocess.Popen(
[zc_binary(), "broker", str(port)],
env=env,
stdout=log,
stderr=subprocess.STDOUT,
start_new_session=True,
)
url = f"http://{HOST}:{port}"
deadline = time.time() + 60
while time.time() < deadline:
if proc.poll() is not None:
raise SystemExit(f"broker exited early, see {log_path}")
try:
ws = get_json(url + "/workers")
ws = ws if isinstance(ws, list) else ws.get("workers", [])
if sum(1 for w in ws if w.get("status") == "healthy") >= len(worker_ports):
return proc, url, log_path
except (OSError, ValueError):
pass time.sleep(0.5)
proc.kill()
raise SystemExit(f"broker never saw {len(worker_ports)} healthy workers, see {log_path}")
def drive(url: str, requests: int, concurrency: int) -> list[dict]:
lock = threading.Lock()
issued = [0]
rows: list[dict] = []
body = json.dumps({"op": "bench"}).encode()
def client() -> None:
while True:
with lock:
if issued[0] >= requests:
return
issued[0] += 1
req = urllib.request.Request(
url + "/execute",
data=body,
method="POST",
headers={
"Content-Type": "application/json",
"Authorization": "Bearer zk_bench_00000000",
"X-Zakuro-Trace": "1",
},
)
t = time.perf_counter()
rec: dict = {}
try:
with urllib.request.urlopen(req, timeout=60) as r:
reply = json.loads(r.read())
phases = dict(
kv.split("=", 1) for kv in (r.headers.get("X-Zakuro-Phases") or "").split() if "=" in kv
)
rec.update(
ok=True,
worker=reply.get("worker"),
reason=r.headers.get("X-Zakuro-Reason"),
route_ms=float(phases.get("route", "nan")),
forward_ms=float(phases.get("forward", "nan")),
)
except urllib.error.HTTPError as e:
rec.update(ok=False, err=f"{e.code} {e.read()[:120]!r}")
except Exception as e: rec.update(ok=False, err=str(e)[:120])
rec["ms"] = (time.perf_counter() - t) * 1000
with lock:
rows.append(rec)
threads = [threading.Thread(target=client) for _ in range(concurrency)]
for th in threads:
th.start()
for th in threads:
th.join()
return rows
def start_workers(service: list[float], names: list[str], cold_ms: float, avoid: set[int]):
ports: list[int] = []
for _ in service:
p = free_port(avoid)
avoid.update({p, p + 1})
ports.append(p)
procs = [
subprocess.Popen(
[sys.executable, os.path.abspath(__file__), "--serve-worker", str(port), str(ms), name, str(cold_ms)],
start_new_session=True,
)
for port, ms, name in zip(ports, service, names)
]
for port in ports:
for _ in range(100):
try:
get_json(f"http://{HOST}:{port}/health", 0.5)
break
except (OSError, ValueError):
time.sleep(0.1)
return procs, ports
def stop(procs) -> None:
for p in procs:
try:
os.killpg(p.pid, signal.SIGTERM)
except ProcessLookupError:
continue
try:
p.wait(timeout=10)
except subprocess.TimeoutExpired:
os.killpg(p.pid, signal.SIGKILL)
def run_arm(adaptive: bool, args, service, names, avoid) -> list[dict]:
workers, ports = start_workers(service, names, args.cold_ms, avoid)
try:
broker, url, log = start_broker(adaptive, ports, avoid)
try:
drive(url, args.warmup, args.concurrency) rows = drive(url, args.requests, args.concurrency)
seen = get_json(url + "/workers")
seen = seen if isinstance(seen, list) else seen.get("workers", [])
print(
" broker's view: "
+ ", ".join(
f"{w.get('name')} avg={w.get('avg_latency_ms', 0):.0f}ms"
for w in sorted(seen, key=lambda w: w.get("name") or "")
),
file=sys.stderr,
)
finally:
stop([broker])
finally:
stop(workers)
reasons = sorted({(r.get("reason") or "?").split(":")[0] for r in rows if r["ok"]})
bad = [r for r in rows if not r["ok"]]
if bad:
print(f" {len(bad)} failed requests, e.g. {bad[0].get('err')} (log {log})", file=sys.stderr)
ok = [r for r in rows if r["ok"]]
lat = [r["ms"] for r in ok]
route = [r["route_ms"] for r in ok if r["route_ms"] == r["route_ms"]]
fwd = [r["forward_ms"] for r in ok if r["forward_ms"] == r["forward_ms"]]
print(
f" round {'adaptive' if adaptive else 'static '}: n={len(ok)} p50={pct(lat, 50):.1f} "
f"p99={pct(lat, 99):.1f} split={split_of(ok, names)} route p50/p99={pct(route, 50):.2f}/"
f"{pct(route, 99):.2f} forward p50/p99={pct(fwd, 50):.1f}/{pct(fwd, 99):.1f} reasons={reasons}",
file=sys.stderr,
)
return rows
def split_of(ok: list[dict], names: list[str]) -> str:
return " / ".join(f"{100 * sum(1 for r in ok if r['worker'] == n) / max(1, len(ok)):.0f}" for n in names)
def main() -> None:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--workers", default="4,4,4,4", help="service ms per worker, comma-separated")
ap.add_argument("--concurrency", type=int, default=6)
ap.add_argument("--requests", type=int, default=600, help="measured requests per arm per round")
ap.add_argument("--warmup", type=int, default=60, help="discarded requests per arm per round")
ap.add_argument("--rounds", type=int, default=2)
ap.add_argument(
"--cold-ms",
type=float,
default=0.0,
help="extra time the first request on each fresh worker takes (Python import cost)",
)
ap.add_argument("--label", default=None, help="workload label for the table")
args = ap.parse_args()
service = [float(x) for x in args.workers.split(",") if x.strip()]
names = [f"w{i}-{int(ms)}ms" for i, ms in enumerate(service)]
avoid: set[int] = set()
order: list[bool] = []
for r in range(args.rounds):
order += [False, True] if r % 2 == 0 else [True, False]
rounds: dict[bool, list[list[dict]]] = {False: [], True: []}
for adaptive in order:
rounds[adaptive].append(run_arm(adaptive, args, service, names, avoid))
def off_uniform(rows: list[dict]) -> float:
ok = [r for r in rows if r["ok"]]
return max(abs(sum(1 for r in ok if r["worker"] == n) / max(1, len(ok)) - 1 / len(names)) for n in names)
label = args.label or f"workers {args.workers} ms, c={args.concurrency}"
print(
f"\n| workload | arm | requests | errors | mean ms | p50 ms | p99 ms | max ms "
f"| split ({', '.join(names)}) | most uneven round |"
)
print("|---|---|---|---|---|---|---|---|---|---|")
for adaptive in (False, True):
rows = [r for rs in rounds[adaptive] for r in rs]
ok = [r for r in rows if r["ok"]]
lat = [r["ms"] for r in ok]
worst = max(rounds[adaptive], key=off_uniform)
print(
f"| {label} | {'adaptive' if adaptive else 'static'} | {len(rows)} | {len(rows) - len(ok)} "
f"| {sum(lat) / max(1, len(lat)):.1f} | {pct(lat, 50):.1f} | {pct(lat, 99):.1f} "
f"| {max(lat, default=float('nan')):.1f} | {split_of(ok, names)} % "
f"| {split_of([r for r in worst if r['ok']], names)} % |"
)
if __name__ == "__main__":
if len(sys.argv) == 6 and sys.argv[1] == "--serve-worker":
serve_worker(int(sys.argv[2]), float(sys.argv[3]), sys.argv[4], float(sys.argv[5]))
else:
main()