{
"schema_version": 1,
"status": "isolated_baseline_complete",
"issue": "#385",
"generated_at": "2026-06-25T19:16:11+08:00",
"runner": {
"name": "codex",
"version": "codex-cli 0.142.1",
"model": "gpt-5.5",
"reasoning_effort": "medium"
},
"fixture": {
"path": "eval/coding-bench/fixtures/tasks.json",
"tasks": 5,
"runs_per_condition": 3,
"total_runs": 45,
"task_timeout_ms": 180000
},
"command": "cargo run -- eval-coding-bench --fixture eval/coding-bench/fixtures/tasks.json --runs-per-condition 3 --runner codex --model gpt-5.5 --reasoning-effort medium --ignore-budget --json-out eval/coding-bench/reports/baseline.json",
"reports": {
"baseline": "eval/coding-bench/reports/baseline.json",
"remem_smoke": "eval/coding-bench/reports/fix-remem-smoke.json"
},
"resolution": {
"no_memory": {
"resolved": 2,
"total": 15,
"rate": 0.13333333333333333,
"tokens_total_mean": 115372.6,
"wall_time_ms_mean": 75733.13333333333
},
"remem": {
"resolved": 15,
"total": 15,
"rate": 1.0,
"tokens_total_mean": 104749.33333333333,
"wall_time_ms_mean": 58725.666666666664
},
"curated_file": {
"resolved": 15,
"total": 15,
"rate": 1.0,
"tokens_total_mean": 94016.53333333334,
"wall_time_ms_mean": 62813.0
}
},
"interpretation": "remem matches curated_file resolution and strongly beats no_memory on the first small memory-dependent fixture; curated_file remains cheaper, so this does not prove remem beats a carefully maintained MEMORY.md.",
"isolation": {
"source_dirty": false,
"remem_rev": "c6a46aec3fe44c8a256138d839ebeea396b6cdb7",
"artifact_marker_scan": "no host home, host .codex, auth, virtualenv, sandbox error, or benchmark-private Codex home markers found in raw artifacts",
"runner_policy": "temporary HOME/CODEX_HOME, ignored host Codex config/rules/hooks/session persistence, stripped host env, macOS host-read sandbox"
},
"caveats": [
"This fixture is small and synthetic; it is directional evidence only.",
"Codex non-interactive MCP calls can be cancelled by the host, so remem preloads full seeded memory details into REMEM_CONTEXT.md after using the production SessionStart render path.",
"The next benchmark version should add pinned real-repo tasks inspired by SWE-bench style issue patching and LiveCodeBench style fresh/self-repair tasks."
]
}