{
"dataset": "princeton-nlp/SWE-bench_Verified",
"metric": "resolve_rate",
"default_n": 10,
"default_sample_seed": 0,
"full_n": 500,
"harnesses": [
"codex",
"pi",
"tk",
"omp",
"fx",
"opencode"
],
"notes": {
"source": "Detected ~/.codex/config.toml model=gpt-5.6-sol effort=low; second slot gpt-5.6-terra from ~/.codex/models_cache.json (priority 2). Same model+effort on every harness.",
"tk_effort": "tk exec --effort is passed through when present; current telekinesis-rx4-consume tk exec only documents --model (no --effort yet).",
"fx": "fx (vercel-labs/fx) non-interactive: fx ask --yolo --no-save; stdin /dev/null. Model via --model if advertised else FX_MODEL. Effort via --effort/--thinking if advertised. Unauthenticated (no fx login / setup / AI_GATEWAY_API_KEY) fail closed.",
"omp": "omp v18 non-interactive: -p --print --cwd TASK --no-session --provider openai-codex --model openai-codex/<id> --thinking <effort>; stdin /dev/null. Bare gpt-5.6-sol fuzzy-matches cursor first (no CURSOR_ACCESS_TOKEN).",
"opencode": "opencode non-interactive: opencode run --auto --dir TASK -m <id>; stdin /dev/null. Auth via ~/.local/share/opencode/auth.json (adapter never prints keys). deepseek-v4-flash uses cline-pass/cline-pass/deepseek-v4-flash."
},
"models": [
{
"id": "gpt-5.6-sol",
"effort": "low",
"codex": {
"model": "gpt-5.6-sol",
"config": "model_reasoning_effort=low"
},
"pi": {
"provider": "openai-codex",
"model": "gpt-5.6-sol",
"thinking": "low"
},
"tk": {
"model": "gpt-5.6-sol",
"effort": "low"
},
"omp": {
"model": "gpt-5.6-sol",
"thinking": "low",
"provider": "openai-codex"
},
"fx": {
"model": "gpt-5.6-sol",
"effort": "low"
}
},
{
"id": "gpt-5.6-terra",
"effort": "low",
"codex": {
"model": "gpt-5.6-terra",
"config": "model_reasoning_effort=low"
},
"pi": {
"provider": "openai-codex",
"model": "gpt-5.6-terra",
"thinking": "low"
},
"tk": {
"model": "gpt-5.6-terra",
"effort": "low"
},
"omp": {
"model": "gpt-5.6-terra",
"thinking": "low",
"provider": "openai-codex"
},
"fx": {
"model": "gpt-5.6-terra",
"effort": "low"
}
},
{
"id": "deepseek-v4-flash",
"effort": "low",
"opencode": {
"model": "cline-pass/cline-pass/deepseek-v4-flash"
}
}
]
}