#!/usr/bin/env python3
"""Render the perf-gate pull-request comment from bench-short output directories.
Each directory is one subject's `--output` from bench-short.py. The comment puts
Kache head against its merge base first, and folds every other table away.
"""
import argparse
import html
import json
import sys
from pathlib import Path
PHASES = (
("cold", "Cold"),
("warm_same_tree", "Warm, same path"),
("warm", "Warm, other checkout"),
)
CONTENTION_PHASES = (("cold", "Cold"), ("warm", "Warm"))
COMPARISONS = PHASES + tuple(
(f"contention_{phase}", f"Contention, {label.lower()}")
for phase, label in CONTENTION_PHASES
)
ARM_ORDER = ("head", "kache", "base", "mbx", "sccache")
def number(ms):
s = ms / 1000
if s >= 100:
return f"{s:.0f}"
if s >= 10:
return f"{s:.1f}"
return f"{s:.2f}"
def seconds(ms):
return "—" if ms is None else f"{number(ms)} s"
def count(value):
if value is None:
return "—"
return f"{value:.0f}" if float(value).is_integer() else f"{value:.1f}"
def timing(row):
"""Median, with the range only when there is more than one sample."""
if row["n"] == 1:
return seconds(row["median_ms"])
return f"{seconds(row['median_ms'])} <sub>{number(row['min_ms'])}–{number(row['max_ms'])}</sub>"
def change(comparison):
if comparison is None:
return "—"
text = f"{comparison.get('geometric_pct', comparison['median_pct']):+.1f}%"
if comparison["outcome"] != "inconclusive":
text = f"**{text} {comparison['outcome']}**"
return text
def arms(rows):
present = {row["arm"] for row in rows}
return [arm for arm in ARM_ORDER if arm in present] + sorted(
present - set(ARM_ORDER)
)
def load(directory):
"""One subject: its summary, contention samples, or the reason it is invalid."""
directory = Path(directory)
project = {"name": directory.name}
samples = directory / "samples.json"
if samples.exists():
payload = json.loads(samples.read_text())
project["name"] = payload.get("project", project["name"])
if payload.get("error"):
project["error"] = payload["error"]
return project
project["versions"] = {
r["arm"]: r["result"]["cache_tool_version"]
for r in payload.get("records", [])
if r.get("result", {}).get("cache_tool_version")
}
summary = directory / "summary.json"
if not summary.exists():
report = directory / "perf-gate.md"
lines = report.read_text().splitlines()[1:] if report.exists() else []
project["error"] = (
"\n".join(lines).strip() or "the measurement produced no summary"
)
return project
project["summary"] = json.loads(summary.read_text())
for comparison in project["summary"].get("comparisons", []):
if comparison.get("method") != "exact_order_randomization":
comparison.update(outcome="inconclusive", interval_pct=None,
p_value=None, p_adjusted=None, blocks=None)
growth = comparison.get("duplicate_key_compiles")
if growth is not None:
growth.update(outcome="inconclusive", p_value=None, p_adjusted=None)
# Retain recorded legacy failures as history, without treating
# their bootstrap/sign-test output as new randomization evidence.
project["legacy_statistics"] = True
contention = directory / "contention" / "samples.json"
if "contention" in project["summary"] and contention.exists():
project["contention"] = json.loads(contention.read_text())
return project
def pivot(title, phases, rows):
columns = arms(rows)
by_key = {(row["arm"], row["phase"]): row for row in rows}
lines = [
f"| {title} | " + " | ".join(columns) + " |",
"| --- |" + " ---: |" * len(columns),
]
for phase, label in phases:
cells = [
timing(by_key[(arm, phase)]) if (arm, phase) in by_key else "—"
for arm in columns
]
lines.append(f"| {label} | " + " | ".join(cells) + " |")
return lines
def details(summary, body, open_=False):
return [
"<details open>" if open_ else "<details>",
f"<summary>{summary}</summary>",
"",
*body,
"",
"</details>",
"",
]
def head_vs_base(projects):
compared = [p for p in projects if p["summary"]["comparisons"]]
if not compared:
return []
header = ["Build"]
for p in compared:
header += [f"{p['name']} base", f"{p['name']} head", f"{p['name']} change"]
lines = [
"| " + " | ".join(header) + " |",
"| --- |" + " ---: |" * (len(header) - 1),
]
rows = [(phase, label, "statistics") for phase, label in PHASES]
if any("contention" in p["summary"] for p in compared):
rows += [
(phase, f"Contention, {label.lower()}", "contention")
for phase, label in CONTENTION_PHASES
]
for phase, label, source in rows:
cells = [label]
for p in compared:
summary = p["summary"]
stats = (
summary["statistics"]
if source == "statistics"
else summary.get("contention", {}).get("statistics", [])
)
by_arm = {row["arm"]: row for row in stats if row["phase"] == phase}
key = phase if source == "statistics" else f"contention_{phase}"
comparison = next(
(c for c in summary["comparisons"] if c["phase"] == key), None
)
verdict = change(comparison)
if comparison and comparison.get("duplicate_key_compiles", {}).get("outcome") == "regression":
verdict += "; duplicate compiles regressed"
cells += [
seconds(by_arm["base"]["median_ms"]) if "base" in by_arm else "—",
seconds(by_arm["head"]["median_ms"]) if "head" in by_arm else "—",
verdict,
]
lines.append("| " + " | ".join(cells) + " |")
lines += ["", "Times are medians; paired changes use geometric means across cold-seed blocks. Positive changes are slower. Inconclusive evidence does not establish parity."]
return lines
def tool_versions(projects):
"""Each arm's `--version`, per subject only where the subjects disagree."""
per_arm = {}
for p in projects:
for arm, version in p.get("versions", {}).items():
per_arm.setdefault(arm, {}).setdefault(version, []).append(p["name"])
if not per_arm:
return []
parts = []
for arm in arms([{"arm": arm} for arm in per_arm]):
versions = per_arm[arm]
if len(versions) == 1:
parts.append(f"{arm} `{next(iter(versions))}`")
else:
parts += [
f"{arm} `{version}` ({', '.join(names)})"
for version, names in versions.items()
]
return ["Versions: " + ", ".join(parts) + ".", ""]
def all_tools(projects, open_):
body = []
for p in projects:
body += pivot(p["name"], PHASES, p["summary"]["statistics"]) + [""]
body.append(
"Times exclude setup. Each warm build starts from the cold cache snapshot and an empty build directory. Other tools are context and never decide the verdict."
)
return details("All tools, isolated builds", body, open_)
def contention(projects):
measured = [p for p in projects if "contention" in p["summary"]]
if not measured:
return []
body = []
for p in measured:
if p["summary"]["contention"]["statistics"]:
body += pivot(
p["name"], CONTENTION_PHASES, p["summary"]["contention"]["statistics"]
) + [""]
first = measured[0].get("contention", {})
if "parallelism" in first:
body.append(
f"Six Cargo jobs, {first['parallelism']} at once, each with {first['jobs_per_build']} Cargo jobs and its own empty target directory. Each tool shares one store across the six jobs. Every warm batch starts from its cold seed, and fresh cold seeds are measured {'every sample' if first['cold_every'] == 1 else 'every ' + str(first['cold_every']) + ' samples'}."
)
counters = []
for p in measured:
rows = [
row
for row in p["summary"]["contention"]["statistics"]
if row.get("compiler_runs") is not None
]
if not rows:
continue
counters += [
f"| {p['name']} | Compiler runs | Duplicate keys | Flight wait | Permit wait |",
"| --- | ---: | ---: | ---: | ---: |",
]
for phase, label in CONTENTION_PHASES:
by_arm = {row["arm"]: row for row in rows if row["phase"] == phase}
for arm in arms(by_arm.values()):
row = by_arm[arm]
counters.append(
f"| {arm}, {label.lower()} | {count(row['compiler_runs'])} | {count(row['duplicate_key_compiles'])} | "
f"{seconds(row['flight_wait_ms'])} | {seconds(row['permit_wait_ms'])} |"
)
counters.append("")
if counters:
body += [
"",
*counters,
"Medians per batch. Waits add up overlapping events across jobs, so they are not wall-clock savings.",
]
duplicates = duplicate_compiles_by_reason(measured)
if duplicates:
body += ["", "| Project | Arm | Phase | Skip reason | Extra compiles | Batches | Most repeated crates |",
"| --- | --- | --- | --- | ---: | ---: | --- |",
*duplicates,
"Extra compiles are summed over the batches. Unknown means the wrapper did not report a skip "
"reason, and not skipped means both compiles finished without one. Every key is in the subject's "
"perf-gate artifact, in contention/<batch>/events.jsonl."]
return details("All tools, contention", body)
def duplicate_compiles_by_reason(projects, crates_shown=5):
"""One row per project, arm, phase and skip reason, so a reason shows up once."""
groups = {}
for project in projects:
for record in project.get("contention", {}).get("records", []):
for entry in record.get("events", {}).get("duplicate_compiles", []):
reasons = entry["skip_reasons"] or {"not skipped": entry["compiles"] - 1}
for reason, extra in reasons.items():
group = groups.setdefault(
(project["name"], record["arm"], record["phase"], reason),
{"extra": 0, "batches": set(), "crates": {}},
)
group["extra"] += extra
group["batches"].add(record["sample"])
crates = group["crates"]
crates[entry["crate_name"]] = crates.get(entry["crate_name"], 0) + extra
rows = []
for (name, arm, phase, reason), group in sorted(groups.items()):
crates = sorted(group["crates"].items(), key=lambda item: (-item[1], item[0]))
shown = ", ".join(f"{crate} ({extra})" for crate, extra in crates[:crates_shown])
if len(crates) > crates_shown:
shown += f", {len(crates) - crates_shown} more"
cells = [name, arm, phase, reason, str(group["extra"]), str(len(group["batches"])), shown]
rows.append("| " + " | ".join(
html.escape(cell).replace("|", "|").replace("\n", " ") for cell in cells
) + " |")
return rows
def disk_use(projects):
storage = []
for p in projects:
records = [r for r in p.get("contention", {}).get("records", []) if "storage" in r]
if not records:
continue
columns = arms(records)
storage += [
f"| {p['name']} (GiB) | " + " | ".join(columns) + " |",
"| --- |" + " ---: |" * len(columns),
]
for phase, label in CONTENTION_PHASES:
for scope, scope_label in (
("cache", "cache"),
("targets", "targets"),
("cache_and_targets", "total"),
):
cells = []
for arm in columns:
sizes = [
r["storage"]["groups"][scope]["allocated_bytes"]
for r in records
if r["arm"] == arm and r["phase"] == phase
]
cells.append(
f"{sum(sizes) / len(sizes) / 2**30:.2f}" if sizes else "—"
)
storage.append(
f"| After {label.lower()}, {scope_label} | " + " | ".join(cells) + " |"
)
storage.append("")
if not storage:
return []
storage.append(
"Mean allocated blocks after each contention batch. Hardlinks count once, and the total counts blocks shared by the cache and targets once. Reflinked extents are not resolved, so these are not exclusive disk bytes. Sources, Cargo home, logs and cold snapshots are excluded."
)
return details("All tools, disk use after contention", storage)
def comparison_detail(projects):
compared = [p for p in projects if p["summary"]["comparisons"]]
if not compared:
return []
body = [
"| Build | " + " | ".join(p["name"] for p in compared) + " |",
"| --- |" + " --- |" * len(compared),
]
for phase, label in COMPARISONS:
cells = []
for p in compared:
c = next((c for c in p["summary"]["comparisons"] if c["phase"] == phase), None)
if c is None:
cells.append("—")
continue
interval = c.get("interval_pct")
bounds = (
""
if interval is None
else " (" + " to ".join("unbounded" if x is None else f"{x:+.1f}%" for x in interval) + ")"
)
noun = "pair" if c["n"] == 1 else "pairs"
cells.append(
f"{c.get('geometric_pct', c['median_pct']):+.1f}%{bounds}, {c['n']} {noun}, "
+ (f"{c['blocks']} seed blocks, adjusted p={c['p_adjusted']:.4f}, " if c.get("blocks") is not None else "descriptive, ")
+ c["outcome"]
)
body.append(f"| {label} | " + " | ".join(cells) + " |")
cells = []
for p in compared:
cold = next(
(c for c in p["summary"]["comparisons"] if c["phase"] == "contention_cold"),
{},
)
growth = cold.get("duplicate_key_compiles")
cells.append(
"—"
if growth is None
else f"{growth['positive_pairs']}/{growth['nonzero_pairs']} non-tied pairs grew "
f"({growth['n']} measured), "
+ (f"adjusted p={growth['p_adjusted']:.4f}, " if growth.get("p_adjusted") is not None else "descriptive, ")
+ growth["outcome"]
)
if any(c != "—" for c in cells):
body.append("| Cold contention duplicate compiles | " + " | ".join(cells) + " |")
body += [
"",
"Base/head order is randomized per cold seed. Warm repeats from one seed form one block. Exact two-sided assignment tests compare against zero change; Bonferroni correction covers the predeclared timing/count family. Parentheses show simultaneous family intervals under a constant proportional effect model, conditional on this run. Unbounded intervals and legacy runs stay inconclusive. Nonsignificance does not establish parity. Warm increases in duplicate compiles, misses or passthroughs fail immediately. `samples.json` retains the schedule and raw reports.",
]
return details("Paired comparison", body)
def render(directories, verdict=None):
projects = [load(d) for d in directories]
valid = [p for p in projects if "summary" in p]
invalid = [p for p in projects if "error" in p]
failures = [(p["name"], f) for p in valid for f in p["summary"]["failures"]]
comparisons = [c for p in valid for c in p["summary"]["comparisons"]]
if verdict is None or verdict == "pass":
verdict = ("INVALID MEASUREMENT" if invalid else "FAIL" if failures
else "INCONCLUSIVE" if any(c["outcome"] == "inconclusive" for c in comparisons)
else "NO DETECTED REGRESSION" if comparisons else "DESCRIPTIVE")
names = ", ".join(p["name"] for p in projects)
lines = [f"## Perf gate: {verdict} ({names})", ""]
for p in invalid:
lines += [f"**{p['name']}: invalid measurement.** {p['error']}", ""]
if failures:
lines += ["**Failed checks**", ""]
lines += [f"- {name}: {failure}" for name, failure in failures]
lines.append("")
elif valid and any(p["summary"]["comparisons"] for p in valid):
lines += ["Kache misses and passthroughs did not rise against base.", ""]
if any(p.get("legacy_statistics") for p in valid):
lines += ["Descriptive comparisons carry no statistical verdict. Recorded failures remain visible.", ""]
if valid:
table = head_vs_base(valid)
lines += table + ([""] if table else [])
lines += tool_versions(valid)
lines += all_tools(valid, open_=not table)
lines += contention(valid)
lines += disk_use([p for p in valid if "contention" in p])
lines += comparison_detail(valid)
return "\n".join(lines).rstrip() + "\n"
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("directories", nargs="+", type=Path)
parser.add_argument(
"--verdict", help="headline verdict; derived from the data when omitted"
)
parser.add_argument("--output", type=Path)
args = parser.parse_args()
text = render(args.directories, args.verdict)
if args.output:
args.output.write_text(text)
else:
sys.stdout.write(text)
if __name__ == "__main__":
main()