rx4 0.7.0

The agent harness engine — loop, tools, providers, sessions, permissions, computer-use
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73

#!/usr/bin/env python3
"""Write REPORT.md + results.json and print a resolve-rate table."""
from __future__ import annotations

import argparse
import json
import sys
from collections import defaultdict
from pathlib import Path


def rate(rows: list[dict]) -> str:
    scored = [r for r in rows if r.get("resolved") is True or r.get("resolved") is False]
    if not scored:
        return "n/a (no official eval)"
    ok = sum(1 for r in scored if r["resolved"] is True)
    return f"{ok}/{len(scored)} ({ok / len(scored):.0%})"


def main() -> int:
    p = argparse.ArgumentParser()
    p.add_argument("--cells", required=True, help="JSONL of per-instance cells")
    p.add_argument("--out-dir", required=True)
    p.add_argument("--meta", default="{}")
    args = p.parse_args()
    cells = [json.loads(l) for l in Path(args.cells).read_text().splitlines() if l.strip()]
    meta = json.loads(args.meta)
    out_dir = Path(args.out_dir)
    out_dir.mkdir(parents=True, exist_ok=True)

    by = defaultdict(list)
    for c in cells:
        by[(c["harness"], c["model"])].append(c)

    lines = [
        "# SWE-bench Verified accuracy",
        "",
        f"- Dataset: `{meta.get('dataset', 'princeton-nlp/SWE-bench_Verified')}`",
        f"- Metric: resolve rate (fail-to-pass tests), not wall-clock on toy tasks",
        f"- n={meta.get('n')}  sample-seed={meta.get('sample_seed')}",
        f"- Driver: {meta.get('driver', 'auto')}",
        f"- Docker: {meta.get('docker', 'unknown')}",
        "",
        "## Summary",
        "",
        "| harness | model | resolve | instances |",
        "|---|---|---|---|",
    ]
    for (h, m) in sorted(by):
        rows = by[(h, m)]
        lines.append(f"| {h} | {m} | {rate(rows)} | {len(rows)} |")
    lines += ["", "## Cells", "", "| harness | model | instance_id | resolved | seconds |", "|---|---|---|---|---|"]
    for c in cells:
        r = c.get("resolved")
        if r is True:
            rs = "true"
        elif r is False:
            rs = "false"
        else:
            rs = "null"
        lines.append(
            f"| {c['harness']} | {c['model']} | {c['instance_id']} | {rs} | {c.get('seconds', '')} |"
        )
    lines.append("")
    (out_dir / "REPORT.md").write_text("\n".join(lines))
    payload = {"meta": meta, "cells": cells}
    (out_dir / "results.json").write_text(json.dumps(payload, indent=2) + "\n")
    sys.stdout.write("\n".join(lines) + "\n")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())