import argparse
import json
import re
import unicodedata
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
SIZES = ('small', 'medium', 'large')
ISSUES = 'https://github.com/joshka/girt/issues/'
def issue_links(numbers):
return ' '.join(f'[#{n}]({ISSUES}{n})' for n in numbers)
def cell(row, release=None):
if row is None:
return '⏳ pending ' + issue_links([66])
links = issue_links(row.get('issues', []))
status = row.get('status', 'ok')
ratio = row.get('ratio_vs_stock')
if status == 'pending':
return '⏳ pending ' + (links or issue_links([66]))
stamp = row.get('girt_version')
older = f' (girt {stamp})' if stamp and release and stamp != release else ''
if status != 'ok' or ratio is None:
return f'❌ {status if status != "ok" else "unavailable"}{older} {links}'.strip()
if row.get('comparable') is False:
return f'⚪ {ratio:.2f}× (n={row.get("girt_samples", "?")}){older} not like-for-like {links}'.strip()
band = '🔴' if ratio > 2 else '🟠' if ratio > 1.2 else '🟢'
n = row.get('girt_samples', '?')
shown = f'{ratio:.3f}' if abs(ratio - 1.2) < 0.005 or abs(ratio - 2) < 0.005 else f'{ratio:.2f}'
return f'{band} {shown}× (n={n}){older} {links}'.strip()
def format_markdown(text):
def width(value):
return sum(2 if unicodedata.east_asian_width(c) in ('W', 'F') else 1
for c in value)
def wrap_prose(value):
words = re.findall(r'(?:`[^`]+`|\[[^\]]+\]\([^)]*\)|[^\s`\[])+', value)
wrapped, line = [], ''
for word in words:
candidate = (line + ' ' + word).strip()
if line and width(candidate) > 100:
wrapped.append(line)
line = word
else:
line = candidate
return '\n'.join([*wrapped, line])
blocks = text.rstrip().split('\n\n')
result = []
for block in blocks:
lines = block.splitlines()
if lines[0].startswith('|'):
rows = [[c.strip() for c in line.strip('|').split('|')] for line in lines]
widths = [max(width(row[i]) for row in rows) for i in range(len(rows[0]))]
table = []
for n, row in enumerate(rows):
if n == 1:
cells = ['-' * w for w in widths]
else:
cells = [c + ' ' * (w - width(c)) for c, w in zip(row, widths)]
table.append('| ' + ' | '.join(cells) + ' |')
result.append('\n'.join(table))
elif lines[0].startswith(('```', '#', '<!--', '- ')):
result.append(block)
else:
result.append(wrap_prose(' '.join(lines)))
return '\n\n'.join(result) + '\n'
def render(rows, meta):
by_key = {}
operations = {tuple(x) for x in meta['operations']}
for row in rows:
key = (row['op'], row['variant'], row['size'])
if key in by_key:
raise ValueError(f'duplicate matrix cell: {key}')
if row['size'] not in SIZES:
raise ValueError(f'unknown size: {row["size"]}')
by_key[key] = row
operations.add(key[:2])
if (row.get('status', 'ok') != 'ok' and row.get('ratio_vs_stock') is not None):
raise ValueError(f'failure has a successful ratio: {key}')
if (((row.get('ratio_vs_stock') or 0) > 1.2 and row.get('comparable') is not False)
or row.get('status', 'ok') != 'ok') and not row.get('issues'):
raise ValueError(f'gap or failure needs an issue link: {key}')
def severity(op):
cells = [by_key.get((*op, size)) for size in SIZES]
failed = any(r is None or r.get('status', 'ok') != 'ok' for r in cells)
neutral = all(r is not None and r.get('comparable') is False for r in cells)
worst = max((r.get('ratio_vs_stock') or 0 for r in cells
if r and r.get('comparable') is not False), default=0)
return (-int(failed), int(neutral), -worst, op)
lines = [
'# jj performance', '',
'<!-- Generated by scripts/jj-e2e/render-performance.py; edit the JSON inputs. -->', '',
'Compare jj on girt with stock jj at the same base revision. Lower ratios are better.',
'Failures and missing results sort first, followed by the worst measured ratio across sizes.',
'These are warm-cache measurements; provenance below names the release of each cell.', '',
'🟢 ≤1.20×; 🟠 >1.20–2.00×; 🔴 >2.00×; ⚪ not like-for-like; ❌ failure/refusal; ⏳ pending.',
'`n` is the number of successful girt samples; JSON records each tool separately.', '',
'| Operation / workload | Small | Medium | Large |',
'| --- | --- | --- | --- |',
]
for op, variant in sorted(operations, key=severity):
label = f'`{op}` / {variant}'
values = [cell(by_key.get((op, variant, size)), meta.get('girt_release')) for size in SIZES]
lines.append('| ' + ' | '.join([label, *values]) + ' |')
release = meta.get('girt_release')
older = sorted({r['girt_version'] for r in rows
if r.get('girt_version') and release and r['girt_version'] != release})
if older:
lines += ['', 'Cells marked `(girt X)` were measured on that release rather than',
f'girt {release}; each row records its own `girt_commit` and `measured_on` date.']
lines += [
'', '## Limits and interpretation', '',
'A timeout has a recorded limit and no successful girt ratio; a refusal has no timed girt',
'sample. Neither is presented as a fast result.',
'Stock jj taking more than twice Git is flagged separately in the JSON as `jj_level_gap`.',
'Git equivalents sometimes perform different work; read each row’s workload notes.',
'Git has no timed counterpart for colocated initialization (the input is already a Git',
'repository), import (Git already sees its own refs), or jj undo/operation restore',
'(Git has no equivalent repository-wide operation restoration). Reflogs approximate jj',
'evolution and operation history but contain different records.',
'Ranges, peak RSS, sample counts, per-row load, issue mappings, and limitations are in',
'[the results JSON](benchmarks/jj-matrix.json).',
'`util gc` rows are not like-for-like: girt\'s gc packs loose objects and prunes, but',
'doesn\'t rewrite existing packs as `git gc` does. They stay timed as a collection-cost',
'signal but are marked ⚪, are never shown as a girt win, and carry `comparable: false`.',
'Peak RSS counts resident pages of memory-mapped pack and index files, so it is not heap',
'use alone.', '',
]
load = meta.get('load_1m')
if load:
lines += [
'Machine load was not controlled. The one-minute load average at sample start ranged',
f'from {load["min"]} to {load["max"]} (median {load["median"]}) on {load["cores"]} cores,',
'so other work competed with many samples. Tool order alternates within each cell, which',
'spreads that contention across tools but does not remove it. Treat ratios within about',
'0.2 of a band boundary on sub-second cells as provisional.', '',
]
lines += [
'## Provenance', '',
f'- Measurement dates: {meta["dates"]}.',
f'- Machine: {meta["machine"]}.',
f'- girt: {"release " + release + ", " if release else ""}`{meta["girt_commit"]}`.',
f'- jj adapter: `{meta["jj_adapter_commit"]}`.',
f'- Stock jj base: `{meta["jj_stock_base"]}`.',
f'- Git: {meta["git_version"]}; release commit `{meta["git_commit"]}`.',
f'- Build: {meta["build"]}.', '',
*[line for note in meta.get('notes', []) for line in ('Note: ' + note, '')],
('Every cell was measured on girt ' + release + '.' if release and not older
else 'Most cells were measured on girt ' + str(release) + '; marked cells name their own release.'),
'Each row records the girt release, girt commit, jj adapter commit, and date it was',
'measured, so cells can be re-measured selectively after a change. Binary fingerprints',
'are per row; [the JSON provenance block](benchmarks/jj-matrix.json) also holds',
'per-repository object, ref, tag, worktree, and pack facts. The Git commit identifies',
'release source; distribution packaging may add patches. Profiles and raw checkpoints are',
'retained locally at the evidence paths; they are not bundled in this repository. The',
'[original 53 small rows](jj-performance-original-small.json) from the first matrix are kept',
'as a historical baseline.', '',
'| Fixture | Packed / loose objects | Refs | Tags | Worktree files | Pack bytes | Loose / packed refs |',
'| --- | --- | --- | --- | --- | --- | --- |',
]
for size in SIZES:
f = meta['fixtures'][size]
counts = dict(line.split(': ',1) for line in f.get('object_counts','').splitlines() if ': ' in line)
objects = f'{counts.get("in-pack",f["objects"])} / {counts.get("count","unknown")}'
values = [size, objects, str(f['refs']), str(f['tags']),
str(f['worktree_files']), str(f['pack_bytes']),
f'{f["loose_refs"]} / {f["packed_refs"] or 0}']
lines.append('| ' + ' | '.join(values) + ' |')
fsck = meta.get('fsck', {})
lines += [
'', '## Method', '',
'Build both binaries before taking the shared `/tmp/girt-timing.lock`. Prepare inputs',
'outside the lock. Use fresh APFS `cp -cR` copies for mutating samples, one large copy at',
'a time; checkpoint timing and validation before deleting it. Reuse clean read-only',
'baselines and verify unchanged jj operation heads. Automatic Git maintenance is disabled;',
'explicit GC remains measured.', '',
'Linux has tracked paths differing only by case. On case-insensitive APFS, Git rebase',
'setup verifies that every unstaged path is such an alias and marks only those paths',
'assume-unchanged before timing. Other unstaged edits fail setup. This platform limitation',
'means the checkout cannot represent both versions of those files faithfully.', '',
'Full runs take six samples per tool on small and medium cells in Git, girt, stock, stock,',
'girt, Git blocks. Linux cells, and targeted refreshes that release the timing lock between',
'samples, use adaptive calibration in Git, stock, girt order: if the slowest',
'first sample exceeds 60 seconds, take one sample per tool; if it exceeds 10 seconds, take',
'three; otherwise take six. Further samples alternate girt and stock positions, so',
'slow-cell ratios have lower precision. Counts in the JSON describe the samples retained.',
'',
'Kill each timed girt process group at `max(2 × first stock-jj seconds, 5 seconds)`.',
'Stock jj and Git each have a 600-second cap. Timeouts fail the cell and record the limit.',
'Report medians, full ranges, and peak RSS using `/usr/bin/time -l`. Release the lock',
'between timed samples in bounded-disk mode. Caches are warm and are not flushed.', '',
f'Full `git fsck` follows the `{fsck.get("scope", "operation-type")}` scope: each mutating',
'operation type is checked once per size, and clone and explicit GC are checked at each',
f'size. This run recorded {fsck.get("checks", "?")} checks and {fsck.get("failures", "?")}',
'failures. Semantic checks cover imported/exported refs and actual garbage removal.',
'Profiles diagnose gaps without replacing timings; adaptive diagnostic recordings are',
'capped at five seconds.', '',
'Fetch cells perform an untimed fetch before introducing the measured commit/tag delta.',
'This imports existing tags and makes noop a genuine second fetch with no remote changes.',
'Stale-push setup prepares remote-tracking refs with untimed Git fetch plus jj import.',
'Limited log uses `::@ --limit 100` against Git `-n 100`.', '',
'## Refresh before a release', '',
'On a quiet macOS machine, build the intended frozen jj/girt revisions and prepare all',
'fixtures under `~/tmp/girt-bench` or `~/tmp/girt-fixtures`. Follow',
'[the matrix runner instructions](../scripts/jj-e2e/README.md), recording exact commits,',
'binary hashes, fixture facts, failures, and issue links. Keep the raw checkpoint and logs.',
'Publication is a manual normalization step: the runner emits a timing list and raw checkpoint,',
'not this publication schema. Merge selected reruns by repository/operation, retain replaced',
'records, classify statuses, attach reviewed issue/profile mappings, and audit sample counts,',
'ranges, fingerprints, and validation epochs before updating the publication envelope.',
'Update the results and provenance in `docs/benchmarks/jj-matrix.json` from that run,',
'then regenerate and check the page:', '',
'```sh',
'python3 scripts/jj-e2e/render-performance.py',
'python3 scripts/jj-e2e/render-performance.py --check',
'```', '',
'CI runs only the freshness check. It never runs benchmarks.', '',
]
return format_markdown('\n'.join(lines))
def main():
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument('--matrix', type=Path, default=ROOT/'docs/benchmarks/jj-matrix.json')
ap.add_argument('--provenance', type=Path,
help='separate provenance for legacy list-only inputs')
ap.add_argument('--output', type=Path, default=ROOT/'docs/jj-performance.md')
ap.add_argument('--check', action='store_true')
args = ap.parse_args()
matrix = json.loads(args.matrix.read_text())
if isinstance(matrix, dict):
if matrix.get('schema_version') != 1:
ap.error('unsupported matrix schema_version')
rows, meta = matrix['results'], matrix['provenance']
if args.check and args.matrix == ROOT/'docs/benchmarks/jj-matrix.json':
mirrors = ((ROOT/'docs/jj-performance.json', rows),
(ROOT/'docs/jj-performance-provenance.json', meta))
for path, expected in mirrors:
if not path.exists() or json.loads(path.read_text()) != expected:
ap.exit(1, f'{path.name} differs from the matrix envelope\n')
else:
if args.provenance is None:
ap.error('legacy list inputs require --provenance')
rows, meta = matrix, json.loads(args.provenance.read_text())
result = render(rows, meta)
if args.check:
if not args.output.exists() or args.output.read_text() != result:
ap.exit(1, 'jj performance page is stale; run scripts/jj-e2e/render-performance.py\n')
else:
args.output.write_text(result)
if __name__ == '__main__':
main()