eval_containers/
benchmark.rs

1//! Benchmark metadata derived from a benchmark's `Dockerfile`.
2
3/// A benchmark is *per-task* — one eval image baked per task (swe-bench-style) —
4/// iff its `Dockerfile` declares `LABEL eval.benchmark.env="per-task"`. That
5/// label is the single source of truth for per-task detection across the CLI
6/// (build, run, oracle) and the chart's `perTask` value (benchmarks/RULES.md 24f).
7/// Matched on a `LABEL` line so a comment or `RUN echo` mentioning the string
8/// cannot false-positive.
9pub fn is_per_task(dockerfile: &str) -> bool {
10    dockerfile.lines().any(|line| {
11        let t = line.trim_start();
12        t.starts_with("LABEL ") && t.contains(r#"eval.benchmark.env="per-task""#)
13    })
14}
15
16/// [`is_per_task`] for a benchmark by name — reads `benchmarks/<name>/Dockerfile`
17/// (a missing file reads as shared-env, `false`).
18pub fn is_per_task_by_name(name: &str) -> bool {
19    std::fs::read_to_string(format!("benchmarks/{name}/Dockerfile"))
20        .as_deref()
21        .map(is_per_task)
22        .unwrap_or(false)
23}
24
25#[cfg(test)]
26mod tests {
27    use super::*;
28
29    #[test]
30    fn is_per_task_keys_off_the_label() {
31        assert!(is_per_task(
32            "FROM scratch\nLABEL eval.benchmark.env=\"per-task\"\n"
33        ));
34        assert!(!is_per_task(
35            "FROM scratch\nLABEL eval.benchmark.env=\"shared\"\n"
36        ));
37        // A FROM/ARG mentioning EVAL_TASK_ID is NOT per-task without the label.
38        assert!(!is_per_task("ARG EVAL_TASK_ID\nFROM x-${EVAL_TASK_ID}\n"));
39        // A comment / RUN echo mentioning the label string must not false-positive.
40        assert!(!is_per_task("# eval.benchmark.env=\"per-task\"\nFROM x\n"));
41    }
42}
eval_containers/benchmark.rs

eval_containers/
benchmark.rs
