bench

#!/usr/bin/env bash # throughput.sh — mini-static against nginx, one worker each, identical files. # # The headline claim in README.md is a throughput comparison against nginx. Until this # script existed that claim rested on an ad-hoc run nobody could reproduce or re-point, # which made it unfalsifiable rather than strong. This is the claim, executable. # # The metric that matters is **requests per CPU-second**, not req/s: a server can buy # throughput with cores. Both numbers are reported; the efficiency one is the honest # comparison. CPU time comes from the process's own accounting (`ps -o cputime`), sampled # either side of the run, so it counts the server and not the load generator. # # Usage: ./bench/throughput.sh # both servers # SERVER_ONLY=1 ./bench/throughput.sh # skip nginx (no nginx installed) # # Exit 0 = the run completed. It does not pass or fail: the bar lives in PLAN-layering.md # and moving it is a decision, not a script's job. set -uo pipefail cd "$(dirname "$0")/.." DURATION="${DURATION:-10s}" CONNECTIONS="${CONNECTIONS:-50}" WARMUP="${WARMUP:-3s}" STATIC_PORT=8081 NGINX_PORT=8082 COMPOSED_PORT=8088 WWW="$PWD/bench/www" TMP="$PWD/bench/.run" command -v oha >/dev/null || { echo "oha is required: cargo install oha"; exit 1; } # The fixture: one small file, which is the case a static server is judged on. rm -rf "$TMP" && mkdir -p "$WWW" "$TMP/logs" python3 - "$WWW/index.html" <<'PY' import sys body = "bench\n" body += "

" + ("x" * 400) + "

\n\n" open(sys.argv[1], "w").write(body) PY SIZE=$(wc -c < "$WWW/index.html" | tr -d ' ') echo "== host ==========================================================" printf "machine %s, %s cores\n" "$(sysctl -n hw.model 2>/dev/null || uname -m)" "$(sysctl -n hw.ncpu 2>/dev/null || nproc)" printf "fixture %s bytes\n" "$SIZE" printf "load oha c=%s for %s (after a %s warm-up), one worker per server\n" "$CONNECTIONS" "$DURATION" "$WARMUP" command -v nginx >/dev/null && printf "nginx %s\n" "$(nginx -v 2>&1 | sed 's/.*nginx\///')" printf "mini-static %s\n" "$(grep -m1 '^version' Cargo.toml | cut -d'"' -f2)" echo # cpu_seconds — total CPU time the process has consumed, in seconds. cpu_seconds() { ps -o cputime= -p "$1" 2>/dev/null | tr -d ' ' | awk -F: ' NF==3 { print $1*3600 + $2*60 + $3; next } NF==2 { print $1*60 + $2; next } { print 0 }' } # measure measure() { local label=$1 pid=$2 url=$3 # Warm up and discard. The first measured run of a freshly built binary reported # 14,704 req/s against 60,475 on every run after — cold pages and a cold page cache. # A benchmark whose first number is four times low is worse than no benchmark. oha -z "$WARMUP" -c "$CONNECTIONS" --no-tui "$url" >/dev/null 2>&1 local before after out rps wall_ms wall cpu total per_cpu cpu_pct p50 before=$(cpu_seconds "$pid") out=$(oha -z "$DURATION" -c "$CONNECTIONS" --no-tui "$url" 2>&1) after=$(cpu_seconds "$pid") rps=$(echo "$out" | awk -F'\t' '/Requests\/sec:/ {print $2; exit}') # oha prints this in ms or in secs depending on magnitude; reading the unit matters. # Taking "3000.7195 ms" as seconds is a clean 1000x error that makes CPU% round to nil. wall=$(echo "$out" | awk -F'\t' '/^ Total:/ {print $2; exit}' | awk ' /ms/ { gsub(/[^0-9.]/,""); print $0/1000; next } { gsub(/[^0-9.]/,""); print }') cpu=$(echo "$after - $before" | bc) p50=$(echo "$out" | awk '/50\.00% in/ {print $3; exit}') total=$(echo "$rps * $wall" | bc) if [ -n "$cpu" ] && [ "$(echo "$cpu > 0.05" | bc)" = "1" ]; then per_cpu=$(printf '%.0f' "$(echo "scale=4; $total / $cpu" | bc)") cpu_pct=$(printf '%.0f' "$(echo "scale=4; 100 * $cpu / $wall" | bc)") else per_cpu="n/a"; cpu_pct="n/a" fi printf "| %-18s | %9.0f | %4s%% | %9s | %s ms |\n" "$label" "$rps" "$cpu_pct" "$per_cpu" "${p50:-n/a}" } echo "| server | req/s | CPU | req/CPU-s | p50 |" echo "|--------------------|-----------|------|-----------|-----|" cargo build --release --quiet --example bench_server || exit 1 PORT=$STATIC_PORT ROOT="$WWW" ./target/release/examples/bench_server >/dev/null 2>&1 & STATIC_PID=$! trap 'kill $STATIC_PID 2>/dev/null; [ -n "${COMPOSED_PID:-}" ] && kill $COMPOSED_PID 2>/dev/null; [ -n "${NGINX_PID:-}" ] && kill $NGINX_PID 2>/dev/null; exit' EXIT INT TERM for _ in $(seq 1 50); do nc -z 127.0.0.1 $STATIC_PORT 2>/dev/null && break; sleep 0.1; done measure "mini-static" "$STATIC_PID" "http://127.0.0.1:$STATIC_PORT/index.html" kill $STATIC_PID 2>/dev/null; wait $STATIC_PID 2>/dev/null # The same server with the `.br`/`.gz` lookup off. Browsers send `Accept-Encoding` on every # request, so a root with no sidecars pays two failed `open()`s per request for a lookup # that cannot succeed; this row is what `without_precompressed()` recovers. NO_PRECOMPRESSED=1 PORT=$STATIC_PORT ROOT="$WWW" ./target/release/examples/bench_server >/dev/null 2>&1 & STATIC_PID=$! for _ in $(seq 1 50); do nc -z 127.0.0.1 $STATIC_PORT 2>/dev/null && break; sleep 0.1; done measure " no sidecar probe" "$STATIC_PID" "http://127.0.0.1:$STATIC_PORT/index.html" kill $STATIC_PID 2>/dev/null; wait $STATIC_PID 2>/dev/null # The composed configuration: 16 API routes registered in front of the same file server. # Answers what the router costs the file half, which is the question the layering turns on. cargo build --release --quiet --example bench_composed || exit 1 PORT=$COMPOSED_PORT ROOT="$WWW" ./target/release/examples/bench_composed >/dev/null 2>&1 & COMPOSED_PID=$! for _ in $(seq 1 50); do nc -z 127.0.0.1 $COMPOSED_PORT 2>/dev/null && break; sleep 0.1; done measure "+ 16 api routes" "$COMPOSED_PID" "http://127.0.0.1:$COMPOSED_PORT/index.html" measure " its api route" "$COMPOSED_PID" "http://127.0.0.1:$COMPOSED_PORT/api/v1/resource7/detail" kill $COMPOSED_PID 2>/dev/null; wait $COMPOSED_PID 2>/dev/null if [ -z "${SERVER_ONLY:-}" ] && command -v nginx >/dev/null; then cat > "$TMP/nginx.conf" </dev/null 2>&1 & sleep 0.5 # The worker does the serving; the master only supervises. NGINX_PID=$(pgrep -f "nginx: worker" | head -1) if [ -n "$NGINX_PID" ]; then for _ in $(seq 1 50); do nc -z 127.0.0.1 $NGINX_PORT 2>/dev/null && break; sleep 0.1; done measure "nginx (1 worker)" "$NGINX_PID" "http://127.0.0.1:$NGINX_PORT/index.html" else echo "| nginx | failed to start — see $TMP/logs/error.log |" fi pkill -f "nginx: master.*$TMP" 2>/dev/null fi echo echo "req/CPU-s is the comparison that matters; req/s alone can be bought with cores."