bench

#!/usr/bin/env bash # Interleaved, feature-matched, repeated. Interleaving is not optional: this machine drifts # thermally by >10% over a few minutes, so measuring all of one server then all of the other # attributes the drift to the software. set -uo pipefail cd "$(dirname "$0")/.." WWW=${WWW:-$PWD/bench/www} TMP=/tmp/faceoff.d; rm -rf $TMP; mkdir -p $TMP/logs REPS=${REPS:-3} nginx_conf() { # $1 = gzip_static on|off cat > $TMP/nginx.conf <bench" '' + "

a paragraph of representative markup.

" * 34 + "\n") (root / "styles.css").write_text( "".join(".c%d{margin:0;padding:%dpx;color:#333;font-family:system-ui}\n" % (i, i % 16) for i in range(240))) FIX # Sidecars, so the "both probing" rows are a real hit on both servers rather than two # failed probes on ours against one failed probe on nginx's. for f in "$WWW/index.html" "$WWW/styles.css"; do gzip -9 -c "$f" > "$f.gz" command -v brotli >/dev/null && brotli -q 11 -c "$f" > "$f.br" done printf 'fixture: index.html %s bytes, styles.css %s bytes\n' \ "$(wc -c < "$WWW/index.html" | tr -d ' ')" "$(wc -c < "$WWW/styles.css" | tr -d ' ')" } # A measured 404 is not a throughput result. Every configuration is checked before its # window, and a miss aborts the run rather than producing a plausible number. assert_200() { local code code=$(curl -s -o /dev/null -w '%{http_code}' "$1") if [ "$code" != "200" ]; then printf 'ABORT: %s answered %s, not 200 — refusing to measure it\n' "$1" "$code" >&2 exit 1 fi } rate() { oha -z 8s -c 50 --no-tui "$1" 2>&1 | awk -F'\t' '/Requests\/sec/ {print $2}'; } warm() { assert_200 "$1"; oha -z 2s -c 50 --no-tui "$1" >/dev/null 2>&1; } fixture declare -a LABELS=() RESULTS=() record() { LABELS+=("$1"); RESULTS+=("$2"); } for path in /index.html /styles.css; do for rep in $(seq 1 $REPS); do # A: mini-static, no sidecar probe vs B: nginx, no gzip_static NO_PRECOMPRESSED=1 PORT=8182 ROOT=$WWW ./target/release/examples/bench_server >/dev/null 2>&1 & p=$!; sleep 0.6; warm "http://127.0.0.1:8182$path" record "static-noprobe$path" "$(rate http://127.0.0.1:8182$path)" kill $p 2>/dev/null; wait $p 2>/dev/null nginx_conf off; nginx -p $TMP -c $TMP/nginx.conf >/dev/null 2>&1 & sleep 0.6; warm "http://127.0.0.1:8181$path" record "nginx-plain$path" "$(rate http://127.0.0.1:8181$path)" pkill -f "nginx: master.*$TMP" 2>/dev/null; sleep 0.4 # C: mini-static with an eager content cache, no filesystem on the hit path CACHE=67108864 NO_PRECOMPRESSED=1 PORT=8182 ROOT=$WWW ./target/release/examples/bench_server >/dev/null 2>&1 & p=$!; sleep 0.6; warm "http://127.0.0.1:8182$path" record "static-cached$path" "$(rate http://127.0.0.1:8182$path)" kill $p 2>/dev/null; wait $p 2>/dev/null # D: mini-static, probing (br+gz) vs E: nginx + gzip_static (gz only) PORT=8182 ROOT=$WWW ./target/release/examples/bench_server >/dev/null 2>&1 & p=$!; sleep 0.6; warm "http://127.0.0.1:8182$path" record "static-probe$path" "$(rate http://127.0.0.1:8182$path)" kill $p 2>/dev/null; wait $p 2>/dev/null nginx_conf on; nginx -p $TMP -c $TMP/nginx.conf >/dev/null 2>&1 & sleep 0.6; warm "http://127.0.0.1:8181$path" record "nginx-gzstatic$path" "$(rate http://127.0.0.1:8181$path)" pkill -f "nginx: master.*$TMP" 2>/dev/null; sleep 0.4 done done rm -rf $TMP python3 - "${LABELS[@]}" --- "${RESULTS[@]}" <<'PY' import sys, statistics args = sys.argv[1:] i = args.index("---") labels, values = args[:i], [float(v) for v in args[i+1:]] groups = {} for l, v in zip(labels, values): groups.setdefault(l, []).append(v) print(f"\n{'configuration':<30}{'mean req/s':>12}{'min':>9}{'max':>9}{'spread':>8}") for l, vs in groups.items(): m = statistics.mean(vs) print(f"{l:<30}{m:>12,.0f}{min(vs):>9,.0f}{max(vs):>9,.0f}{(max(vs)-min(vs))/m*100:>7.1f}%") PY