speccoder_runtime/
bench.rs

1//! Public benchmarks as evaluation cases.
2//!
3//! `speccoder bench import` turns instances in the SWE-bench format
4//! (`instance_id`, `repo`, `base_commit`, `problem_statement`, `test_patch`,
5//! `FAIL_TO_PASS`, …: SWE-bench, SWE-bench Verified/Multilingual/Pro, and
6//! benchmarks that reuse the format) into cases of the evaluation suite:
7//!
8//! - `project/`: the repository at `base_commit` as a Spec Kit project — the
9//!   issue becomes the user story of `spec.md`, with one task to resolve it;
10//! - `oracle/`: the files the benchmark's `test_patch` adds or changes, which
11//!   the agent never sees and the oracle runs after the attempt;
12//! - `case.toml`: the task, `expect = "complete"`, the test command.
13//!
14//! Then `speccoder eval <out> [--executor claude]` runs them like any case.
15//! Downloading the datasets and cloning the repositories is the person's
16//! (`--repos` holds one clone per repository, named `owner__name`), and so
17//! is the environment the tests need: SWE-bench runs each instance in a
18//! container image; here the tests run on this machine, in the sandbox.
19
20use std::fs;
21use std::path::{Path, PathBuf};
22
23use serde::Deserialize;
24use serde_json::Value;
25use speccoder_core::{BenchReport, CommandData, CommandResult, CommandStatus};
26
27use crate::git::Git;
28
29/// One benchmark instance.
30#[derive(Debug, Clone, Deserialize)]
31pub struct Instance {
32    /// e.g. `owner__repo-1234`.
33    pub instance_id: String,
34    /// `owner/name`.
35    pub repo: String,
36    /// The commit the issue was filed against.
37    pub base_commit: String,
38    /// The issue text.
39    pub problem_statement: String,
40    /// The tests that decide the instance, as a patch.
41    pub test_patch: String,
42    /// Tests that must go from failing to passing (a list, or a JSON string of one).
43    #[serde(default, rename = "FAIL_TO_PASS")]
44    pub fail_to_pass: Value,
45    /// The command that runs the tests, when the dataset gives one.
46    #[serde(default)]
47    pub test_cmd: Option<Value>,
48}
49
50/// The issue a benchmark case is built around.
51pub struct Issue<'a> {
52    /// The instance id.
53    pub id: &'a str,
54    /// `owner/name`.
55    pub repo: &'a str,
56    /// The commit the issue was reported against.
57    pub base_commit: &'a str,
58    /// The issue text.
59    pub statement: &'a str,
60}
61
62/// How to import.
63#[derive(Debug, Clone)]
64pub struct ImportOptions<'a> {
65    /// One clone per repository: `<repos>/<owner>__<name>`.
66    pub repos: &'a Path,
67    /// The suite directory to write cases into.
68    pub out: &'a Path,
69    /// The test command, if the dataset has none (default: detected).
70    pub test_cmd: Option<Vec<String>>,
71    /// At most this many instances.
72    pub limit: Option<usize>,
73}
74
75fn names(v: &Value) -> Vec<String> {
76    match v {
77        Value::Array(a) => a
78            .iter()
79            .filter_map(|t| t.as_str().map(str::to_owned))
80            .collect(),
81        Value::String(s) => serde_json::from_str::<Vec<String>>(s).unwrap_or_default(),
82        _ => Vec::new(),
83    }
84}
85
86/// The test command for a project, by what it contains.
87fn detect_test_cmd(project: &Path) -> Option<Vec<String>> {
88    let cmd = |a: &[&str]| Some(a.iter().map(|s| (*s).to_owned()).collect());
89    if project.join("Cargo.toml").exists() {
90        cmd(&["cargo", "test", "--quiet"])
91    } else if project.join("package.json").exists() {
92        cmd(&["npm", "test", "--silent"])
93    } else if project.join("pyproject.toml").exists() || project.join("setup.py").exists() {
94        cmd(&["python3", "-m", "pytest", "-q"])
95    } else if project.join("go.mod").exists() {
96        cmd(&["go", "test", "./..."])
97    } else {
98        None
99    }
100}
101
102fn slug(id: &str) -> String {
103    id.chars()
104        .map(|c| {
105            if c.is_ascii_alphanumeric() || c == '-' {
106                c.to_ascii_lowercase()
107            } else {
108                '-'
109            }
110        })
111        .collect()
112}
113
114fn toml_list(argv: &[String]) -> String {
115    let items: Vec<String> = argv.iter().map(|a| format!("{a:?}")).collect();
116    format!("[{}]", items.join(", "))
117}
118
119/// Paths a patch writes (`+++ b/<path>`), deletions excluded.
120fn patched_paths(patch: &str) -> Vec<String> {
121    patch
122        .lines()
123        .filter_map(|l| l.strip_prefix("+++ b/"))
124        .map(|p| p.trim().to_owned())
125        .collect()
126}
127
128/// A checkout of `src` at `commit` in `dir`, without history.
129fn checkout(src: &Path, commit: &str, dir: &Path) -> Result<(), String> {
130    let parent = dir.parent().ok_or("no parent directory")?;
131    fs::create_dir_all(parent).map_err(|e| e.to_string())?;
132    Git::at(parent)
133        .run(&[
134            "clone",
135            "--quiet",
136            "--no-checkout",
137            &src.to_string_lossy(),
138            &dir.to_string_lossy(),
139        ])
140        .map_err(|e| e.0)?;
141    Git::at(dir)
142        .run(&["checkout", "--quiet", commit])
143        .map_err(|e| e.0)?;
144    Ok(())
145}
146
147fn import_one(inst: &Instance, opts: &ImportOptions<'_>) -> Result<PathBuf, String> {
148    let src = opts.repos.join(inst.repo.replace('/', "__"));
149    if !src.join(".git").exists() && !src.join("HEAD").exists() {
150        return Err(format!(
151            "no clone of {} at {} (git clone https://github.com/{} {})",
152            inst.repo,
153            src.display(),
154            inst.repo,
155            src.display()
156        ));
157    }
158    let case = opts.out.join(slug(&inst.instance_id));
159    if case.exists() {
160        return Err(format!("{} exists", case.display()));
161    }
162    let project = case.join("project");
163    checkout(&src, &inst.base_commit, &project)?;
164    fs::remove_dir_all(project.join(".git")).map_err(|e| e.to_string())?;
165
166    // The oracle: the files the test patch writes, taken from a checkout
167    // with the patch applied — the agent's worktree never has them.
168    let scratch = case.join(".oracle-checkout");
169    checkout(&src, &inst.base_commit, &scratch)?;
170    let applied = Git::at(&scratch).run_with_stdin(
171        &["apply", "--whitespace=nowarn", "-"],
172        inst.test_patch.as_bytes(),
173    );
174    if let Err(e) = applied {
175        let _ = fs::remove_dir_all(&case);
176        return Err(format!("the test patch does not apply: {}", e.0));
177    }
178    for rel in patched_paths(&inst.test_patch) {
179        let from = scratch.join(&rel);
180        let to = case.join("oracle").join(&rel);
181        if let Some(parent) = to.parent() {
182            fs::create_dir_all(parent).map_err(|e| e.to_string())?;
183        }
184        fs::copy(&from, &to).map_err(|e| format!("{rel}: {e}"))?;
185    }
186    fs::remove_dir_all(&scratch).map_err(|e| e.to_string())?;
187
188    let test_cmd = inst
189        .test_cmd
190        .as_ref()
191        .and_then(|v| match v {
192            Value::String(s) => Some(s.split_whitespace().map(str::to_owned).collect::<Vec<_>>()),
193            Value::Array(_) => Some(names(v)),
194            _ => None,
195        })
196        .or_else(|| opts.test_cmd.clone())
197        .or_else(|| detect_test_cmd(&project))
198        .ok_or("no test command: pass --test-cmd")?;
199
200    // The spec project around the issue.
201    crate::format::speckit::bench::scaffold(
202        &project,
203        &slug(&inst.instance_id),
204        &Issue {
205            id: &inst.instance_id,
206            repo: &inst.repo,
207            base_commit: &inst.base_commit,
208            statement: &inst.problem_statement,
209        },
210        &test_cmd,
211    )?;
212    let write = |p: PathBuf, text: String| fs::write(p, text).map_err(|e| e.to_string());
213    write(
214        project.join("speccoder.toml"),
215        format!(
216            "[checks.test]\nargv = {}\ntimeout_secs = 900\n",
217            toml_list(&test_cmd)
218        ),
219    )?;
220    let fail_to_pass = names(&inst.fail_to_pass);
221    write(
222        case.join("case.toml"),
223        format!(
224            "# Imported from {} ({}).\n# FAIL_TO_PASS: {}\ntask = \"T001\"\nexpect = \"complete\"\n\n[oracle]\nargv = {}\n",
225            inst.instance_id,
226            inst.repo,
227            fail_to_pass.join(", "),
228            toml_list(&test_cmd)
229        ),
230    )?;
231    Ok(case)
232}
233
234/// `speccoder bench import`.
235#[must_use]
236pub fn import(instances: &Path, opts: &ImportOptions<'_>) -> CommandResult {
237    let text = match fs::read_to_string(instances) {
238        Ok(t) => t,
239        Err(e) => {
240            return CommandResult::new(
241                "bench",
242                CommandStatus::NotFound,
243                &format!("{}: {e}", instances.display()),
244            );
245        }
246    };
247    // JSON Lines, or one JSON array.
248    let parsed: Result<Vec<Instance>, String> = if text.trim_start().starts_with('[') {
249        serde_json::from_str(&text).map_err(|e| e.to_string())
250    } else {
251        text.lines()
252            .filter(|l| !l.trim().is_empty())
253            .map(|l| serde_json::from_str(l).map_err(|e| e.to_string()))
254            .collect()
255    };
256    let list = match parsed {
257        Ok(l) => l,
258        Err(e) => {
259            return CommandResult::new(
260                "bench",
261                CommandStatus::ConfigError,
262                &format!("{}: {e}", instances.display()),
263            );
264        }
265    };
266    let mut report = BenchReport {
267        out: opts.out.display().to_string(),
268        imported: Vec::new(),
269        skipped: Vec::new(),
270    };
271    for inst in list.iter().take(opts.limit.unwrap_or(usize::MAX)) {
272        match import_one(inst, opts) {
273            Ok(_) => report.imported.push(inst.instance_id.clone()),
274            Err(e) => report.skipped.push(format!("{}: {e}", inst.instance_id)),
275        }
276    }
277    let suite = opts.out.join(crate::eval::SUITE);
278    if !report.imported.is_empty() && !suite.exists() {
279        let _ = fs::write(
280            &suite,
281            "# The model every case runs with; the key comes from the environment.\n[model]\nprovider = \"openai\"\nmodel = \"gpt-5.4-mini\"\nmax_output_tokens = 8000\n\n[budget]\nmax_turns = 40\nmax_tool_calls = 80\n",
282        );
283    }
284    let status = if report.imported.is_empty() {
285        CommandStatus::NotFound
286    } else {
287        CommandStatus::Ok
288    };
289    let message = format!(
290        "{} imported, {} skipped; run them with `speccoder eval {}`",
291        report.imported.len(),
292        report.skipped.len(),
293        opts.out.display()
294    );
295    CommandResult::new("bench", status, &message).with_data(CommandData::Bench(Box::new(report)))
296}
speccoder_runtime/bench.rs

speccoder_runtime/
bench.rs
