use std::path::{Path, PathBuf};
use std::process::{Command, Output};
fn repo_root() -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.parent()
.unwrap()
.parent()
.unwrap()
.to_path_buf()
}
fn benchmarks_dir() -> PathBuf {
repo_root().join("benchmarks")
}
fn scc() -> &'static str {
env!("CARGO_BIN_EXE_scc")
}
fn run_in(dir: &Path, args: &[&str]) -> Output {
Command::new(scc())
.args(args)
.current_dir(dir)
.output()
.expect("scc binary runs")
}
fn run_in_env(dir: &Path, args: &[&str], envs: &[(&str, &str)]) -> Output {
let mut cmd = Command::new(scc());
cmd.args(args).current_dir(dir);
for (k, v) in envs {
cmd.env(k, v);
}
cmd.output().expect("scc binary runs")
}
fn copy_tree(src: &Path, dst: &Path) {
std::fs::create_dir_all(dst).unwrap();
for entry in std::fs::read_dir(src).unwrap() {
let entry = entry.unwrap();
if entry.file_name() == ".scc" {
continue;
}
let from = entry.path();
let to = dst.join(entry.file_name());
if from.is_dir() {
copy_tree(&from, &to);
} else {
std::fs::copy(&from, &to).unwrap();
}
}
}
fn python3() -> String {
let out = Command::new("python3")
.arg("-c")
.arg("import sys; print(sys.executable)")
.output()
.expect("python3 resolves");
assert!(out.status.success(), "python3 must be installed");
String::from_utf8_lossy(&out.stdout).trim().to_string()
}
const FAKE_AGENT: &str = r#"printf '%s\n' '{"type":"item.completed","item":{"type":"agent_message","text":"Plan: cli.rs"}}' '{"type":"item.completed","item":{"type":"command_execution","command":"/bin/zsh -lc \"rg -n paging .\"","exit_code":0}}' '{"type":"item.completed","item":{"type":"mcp_tool_call","tool":"read_file","arguments":{"file_path":"cli.rs"}}}' '{"type":"item.completed","item":{"type":"mcp_tool_call","tool":"read_file","arguments":{"file_path":"main.go"}}}'"#;
#[test]
fn external_lock_pins_aider_and_repomix() {
let text = std::fs::read_to_string(benchmarks_dir().join("external-lock.json"))
.expect("external-lock.json exists");
let lock: serde_json::Value = serde_json::from_str(&text).expect("external-lock.json parses");
let aider = &lock["aider"];
assert_eq!(aider["repository"], "Aider-AI/aider");
assert_eq!(
aider["commit"],
"5dc9490bb35f9729ef2c95d00a19ccd30c26339c"
);
assert_eq!(aider["commit"].as_str().unwrap().len(), 40, "full git sha");
let repomix = &lock["repomix"];
assert_eq!(repomix["repository"], "yamadashy/repomix");
assert_eq!(
repomix["commit"],
"e3b15a406ed78d8a463620a032a059ce911bfc0e"
);
assert_eq!(repomix["commit"].as_str().unwrap().len(), 40, "full git sha");
}
#[test]
fn external_scripts_exist_and_are_executable() {
for name in [
"aider_adapter.py",
"repomix_adapter.py",
"run_context_bench.py",
"ground-truth.yaml",
] {
let path = benchmarks_dir().join("external").join(name);
assert!(path.is_file(), "{name} exists at {}", path.display());
}
for name in ["aider_adapter.py", "repomix_adapter.py", "run_context_bench.py"] {
let path = benchmarks_dir().join("external").join(name);
let meta = std::fs::metadata(&path).expect("metadata");
use std::os::unix::fs::PermissionsExt;
assert!(
meta.permissions().mode() & 0o111 != 0,
"{name} is executable"
);
}
}
#[test]
fn harness_help_exits_zero() {
let py = python3();
let out = Command::new(&py)
.arg(benchmarks_dir().join("external").join("run_context_bench.py"))
.arg("--help")
.output()
.expect("harness runs");
assert!(
out.status.success(),
"--help exited {}: {}",
out.status,
String::from_utf8_lossy(&out.stderr)
);
let help = String::from_utf8_lossy(&out.stdout);
for variant in ["raw", "aider-repomap", "repomix-compress", "scc-atlas-surface", "scc-full"] {
assert!(help.contains(variant), "--help lists variant {variant}");
}
}
fn paid_or_skip() -> bool {
if std::env::var("SCC_ALLOW_PAID_BENCHMARKS").as_deref() != Ok("1") {
eprintln!("skip: needs SCC_ALLOW_PAID_BENCHMARKS=1 (paid agent path)");
return false;
}
true
}
#[test]
fn external_variant_emits_metric_row() {
if !paid_or_skip() {
return;
}
let workdir = tempfile::TempDir::new().unwrap();
let out = run_in(
workdir.path(),
&[
"bench",
"external",
"--variant",
"scc-atlas-surface",
"--repo",
"cli-service",
"--budget",
"8000",
"--workdir",
workdir.path().to_str().unwrap(),
"--cmd",
FAKE_AGENT,
"--json",
],
);
assert!(
out.status.success(),
"scc bench external failed: {}",
String::from_utf8_lossy(&out.stderr)
);
let row: serde_json::Value =
serde_json::from_slice(&out.stdout).expect("stdout is the metric row JSON");
assert_eq!(row["variant"], "scc-atlas-surface");
assert_eq!(row["repo"], "cli-service");
assert_eq!(row["budget"], 8000);
assert_eq!(row["tasks"], 3, "cli-service has 3 ground-truth tasks");
assert_eq!(row["success_rate"], 1.0);
assert_eq!(row["mean_files_opened"], 2.0, "cli.rs + main.go read");
assert_eq!(row["mean_search_tool_calls"], 1.0, "one rg search per task");
assert!(
row["context_tokens"].as_u64().unwrap() > 0,
"the startup artifact carries tokens"
);
assert!(
row["mean_exploration"].as_f64().unwrap() >= 3.0,
"exploration = files + searches + graph queries"
);
assert!(
row["first_plan_accuracy"].as_f64().unwrap() >= 1.0,
"the fake plan names a ground-truth key for every task"
);
assert!(
row.get("mean_files_opened_before_first_correct").is_some(),
"wrong-first metric present"
);
let artifact = workdir.path().join("artifacts").join("cli-service.txt");
assert!(artifact.is_file(), "artifact written: {}", artifact.display());
assert!(
!std::fs::read_to_string(&artifact).unwrap().is_empty(),
"artifact non-empty"
);
}
#[test]
fn scc_surface_variant_generates_artifact() {
if !paid_or_skip() {
return;
}
let workdir = tempfile::TempDir::new().unwrap();
let out = run_in(
workdir.path(),
&[
"bench",
"external",
"--variant",
"scc-surface",
"--repo",
"cli-service",
"--budget",
"8000",
"--workdir",
workdir.path().to_str().unwrap(),
"--cmd",
FAKE_AGENT,
"--json",
],
);
assert!(
out.status.success(),
"scc bench external (scc-surface) failed: {}",
String::from_utf8_lossy(&out.stderr)
);
let artifact = workdir.path().join("artifacts").join("cli-service.txt");
let text = std::fs::read_to_string(&artifact)
.expect("surface artifact exists")
.trim()
.to_string();
assert!(!text.is_empty(), "surface artifact is non-empty");
assert!(
text.contains("cli.rs") || text.contains("cli.py") || text.contains("main.go"),
"surface artifact names cli-service surfaces: {text}"
);
}
fn restricted_env() -> Vec<(&'static str, &'static str)> {
vec![
("PATH", "/usr/bin:/bin"),
("SCC_BENCH_VENV", "/nonexistent-scc-bench-venv"),
]
}
#[test]
fn aider_adapter_skips_cleanly_when_tool_missing() {
let py = python3();
let out_dir = tempfile::TempDir::new().unwrap();
let out = Command::new(&py)
.arg(benchmarks_dir().join("external").join("aider_adapter.py"))
.arg(repo_root().join("fixtures").join("cli-service"))
.arg("8000")
.arg(out_dir.path())
.env("PATH", "/usr/bin:/bin")
.output()
.expect("adapter runs");
assert_eq!(out.status.code(), Some(2), "exit 2 = SKIPPED-UNINSTALLED");
let payload: serde_json::Value =
serde_json::from_slice(&out.stdout).expect("error JSON on stdout");
assert_eq!(payload["ok"], false);
assert!(
payload["error"]
.as_str()
.unwrap()
.starts_with("SKIPPED-UNINSTALLED"),
"error names the skip reason: {}",
payload["error"]
);
}
#[test]
fn repomix_adapter_skips_cleanly_when_tool_missing() {
let py = python3();
let out_dir = tempfile::TempDir::new().unwrap();
let out = Command::new(&py)
.arg(benchmarks_dir().join("external").join("repomix_adapter.py"))
.arg(repo_root().join("fixtures").join("cli-service"))
.arg("8000")
.arg(out_dir.path())
.arg("--compress")
.env("PATH", "/usr/bin:/bin")
.output()
.expect("adapter runs");
assert_eq!(out.status.code(), Some(2), "exit 2 = SKIPPED-UNINSTALLED");
let payload: serde_json::Value =
serde_json::from_slice(&out.stdout).expect("error JSON on stdout");
assert_eq!(payload["ok"], false);
assert!(
payload["error"]
.as_str()
.unwrap()
.starts_with("SKIPPED-UNINSTALLED")
);
}
#[test]
fn external_variant_delegation_reports_skipped_status() {
if !paid_or_skip() {
return;
}
let workdir = tempfile::TempDir::new().unwrap();
let out = run_in_env(
workdir.path(),
&[
"bench",
"external",
"--variant",
"aider-repomap",
"--repo",
"cli-service",
"--budget",
"8000",
],
&restricted_env(),
);
assert!(
out.status.success(),
"delegated run reports the skip without failing: {}",
String::from_utf8_lossy(&out.stderr)
);
let stdout = String::from_utf8_lossy(&out.stdout);
assert!(
stdout.contains("SKIPPED-UNINSTALLED"),
"row names the skip: {stdout}"
);
}
#[test]
fn ablation_variants_emit_metric_rows() {
if !paid_or_skip() {
return;
}
for variant in ["lexical", "task-ppr", "ppr-quotas"] {
let workdir = tempfile::TempDir::new().unwrap();
let out = run_in(
workdir.path(),
&[
"bench",
"external",
"--variant",
variant,
"--repo",
"cli-service",
"--budget",
"8000",
"--workdir",
workdir.path().to_str().unwrap(),
"--cmd",
FAKE_AGENT,
"--json",
],
);
assert!(
out.status.success(),
"scc bench external ({variant}) failed: {}",
String::from_utf8_lossy(&out.stderr)
);
let row: serde_json::Value =
serde_json::from_slice(&out.stdout).expect("stdout is the metric row JSON");
assert_eq!(row["variant"], variant);
assert_eq!(row["repo"], "cli-service");
assert_eq!(row["tasks"], 3, "cli-service has 3 ground-truth tasks");
assert!(
row["context_tokens"].as_u64().unwrap() > 0,
"{variant} artifact carries tokens"
);
assert!(
row["context_tokens"].as_u64().unwrap() <= 8000,
"{variant} artifact respects the budget"
);
let artifact = workdir.path().join("artifacts").join("cli-service.txt");
let text = std::fs::read_to_string(&artifact).expect("artifact exists");
assert!(
text.contains(&format!("ablation {variant}")),
"{variant} artifact is mode-labeled: {text}"
);
}
}
#[test]
fn scc_full_structural_section_is_goal_selected_not_ground_truth() {
if !paid_or_skip() {
return;
}
let workdir = tempfile::TempDir::new().unwrap();
let out = run_in(
workdir.path(),
&[
"bench",
"external",
"--variant",
"scc-full",
"--repo",
"cli-service",
"--budget",
"8000",
"--workdir",
workdir.path().to_str().unwrap(),
"--cmd",
FAKE_AGENT,
"--json",
],
);
assert!(
out.status.success(),
"scc bench external (scc-full) failed: {}",
String::from_utf8_lossy(&out.stderr)
);
let artifact = workdir.path().join("artifacts").join("cli-service.txt");
let text = std::fs::read_to_string(&artifact).expect("artifact exists");
let estimate = scc_core::estimate_tokens;
assert!(
estimate(&text) <= 8000,
"concatenated artifact fits the budget ({} tokens)",
estimate(&text)
);
let mut units: Vec<&str> = Vec::new();
let lines: Vec<&str> = text.lines().collect();
let mut i = 0;
while i + 2 < lines.len() {
if lines[i + 1].is_empty()
&& lines[i + 2].starts_with("source: ")
&& lines[i + 2].contains(&format!("source: {}:", lines[i]))
{
units.push(lines[i].trim());
i += 3;
} else {
i += 1;
}
}
assert!(!units.is_empty(), "structural section present: {text}");
let tmp = tempfile::TempDir::new().unwrap();
let oracle_root = tmp.path().join("repo");
copy_tree(&repo_root().join("fixtures").join("cli-service"), &oracle_root);
let idx = run_in(&oracle_root, &["index", "--quiet"]);
assert!(idx.status.success(), "oracle index succeeds");
let surface = run_in(
&oracle_root,
&["surface", "--task", "add a --paging flag to the serve subcommand", "--budget", "8000"],
);
assert!(surface.status.success());
let surface_text = String::from_utf8_lossy(&surface.stdout);
let mut oracle_paths: Vec<String> = Vec::new();
let s_lines: Vec<&str> = surface_text.lines().collect();
for i in 0..s_lines.len() {
let path = s_lines[i].trim();
if path.is_empty() || s_lines[i].starts_with(' ') {
continue; }
let prev_blank = i
.checked_sub(1)
.map(|j| s_lines[j].trim().is_empty())
.unwrap_or(false);
let next_blank = s_lines
.get(i + 1)
.map(|l| l.trim().is_empty())
.unwrap_or(false);
let above_component = i
.checked_sub(2)
.map(|j| !s_lines[j].trim().is_empty())
.unwrap_or(false);
if !(prev_blank && next_blank && above_component) {
continue;
}
if !(path.contains('/') || path.contains('.')) {
continue;
}
if path.chars().all(|c| !c.is_lowercase()) {
continue;
}
let path = path.split('[').next().unwrap_or(path).trim().to_string();
oracle_paths.push(path);
}
assert!(
!oracle_paths.is_empty(),
"surface oracle renders paths: {surface_text}"
);
for unit in &units {
assert!(
oracle_paths.iter().any(|p| p == unit),
"structural unit {unit} comes from the task surface selection (never task.files)"
);
}
}
#[test]
fn aider_adapter_pin_mismatch_exits_3() {
let py = python3();
let out_dir = tempfile::TempDir::new().unwrap();
let fake_site = tempfile::TempDir::new().unwrap();
let dist = fake_site.path().join("aider_chat-9.9.9.dist-info");
std::fs::create_dir_all(&dist).unwrap();
std::fs::write(
dist.join("direct_url.json"),
r#"{"url": "https://github.com/Aider-AI/aider.git", "vcs_info": {"commit_id": "deadbeef00000000000000000000000000000000"}}"#,
)
.unwrap();
let out = Command::new(&py)
.arg(benchmarks_dir().join("external").join("aider_adapter.py"))
.arg(repo_root().join("fixtures").join("cli-service"))
.arg("8000")
.arg(out_dir.path())
.arg("--goal")
.arg("add a --paging flag")
.env("SCC_AIDER_SITE_PACKAGES", fake_site.path())
.output()
.expect("adapter runs");
assert_eq!(out.status.code(), Some(3), "exit 3 = PIN-MISMATCH");
let payload: serde_json::Value =
serde_json::from_slice(&out.stdout).expect("error JSON on stdout");
assert_eq!(payload["ok"], false);
assert!(
payload["error"]
.as_str()
.unwrap()
.starts_with("PIN-MISMATCH"),
"error names the mismatch: {}",
payload["error"]
);
}
#[test]
fn repomix_adapter_pin_mismatch_exits_3() {
let py = python3();
let out_dir = tempfile::TempDir::new().unwrap();
let fake_pkg = tempfile::TempDir::new().unwrap();
std::fs::write(
fake_pkg.path().join("package.json"),
r#"{"name": "repomix", "version": "9.9.9"}"#,
)
.unwrap();
let out = Command::new(&py)
.arg(benchmarks_dir().join("external").join("repomix_adapter.py"))
.arg(repo_root().join("fixtures").join("cli-service"))
.arg("8000")
.arg(out_dir.path())
.arg("--compress")
.env("SCC_REPOMIX_PKG_DIR", fake_pkg.path())
.env("SCC_REPOMIX_SRC_DIR", "/nonexistent-scc-bench-repomix")
.output()
.expect("adapter runs");
assert_eq!(out.status.code(), Some(3), "exit 3 = PIN-MISMATCH");
let payload: serde_json::Value =
serde_json::from_slice(&out.stdout).expect("error JSON on stdout");
assert_eq!(payload["ok"], false);
assert!(
payload["error"]
.as_str()
.unwrap()
.starts_with("PIN-MISMATCH"),
"error names the mismatch: {}",
payload["error"]
);
}
#[test]
fn repomix_adapter_pin_unverified_exits_4() {
let py = python3();
let out_dir = tempfile::TempDir::new().unwrap();
let fake_pkg = tempfile::TempDir::new().unwrap();
std::fs::write(
fake_pkg.path().join("package.json"),
r#"{"name": "repomix", "version": "1.18.0"}"#,
)
.unwrap();
let out = Command::new(&py)
.arg(benchmarks_dir().join("external").join("repomix_adapter.py"))
.arg(repo_root().join("fixtures").join("cli-service"))
.arg("8000")
.arg(out_dir.path())
.arg("--compress")
.env("SCC_REPOMIX_PKG_DIR", fake_pkg.path())
.env("SCC_REPOMIX_SRC_DIR", "/nonexistent-scc-bench-repomix")
.output()
.expect("adapter runs");
assert_eq!(out.status.code(), Some(4), "exit 4 = PIN-UNVERIFIED");
let payload: serde_json::Value =
serde_json::from_slice(&out.stdout).expect("error JSON on stdout");
assert_eq!(payload["ok"], false);
assert!(
payload["error"]
.as_str()
.unwrap()
.starts_with("PIN-UNVERIFIED"),
"error names the unverified pin: {}",
payload["error"]
);
}
#[test]
fn repomix_adapter_pin_passes_on_githead_match() {
let py = python3();
let out_dir = tempfile::TempDir::new().unwrap();
let fake_pkg = tempfile::TempDir::new().unwrap();
std::fs::write(
fake_pkg.path().join("package.json"),
r#"{"name": "repomix", "version": "1.18.0", "gitHead": "e3b15a406ed78d8a463620a032a059ce911bfc0e"}"#,
)
.unwrap();
let bin_dir = tempfile::TempDir::new().unwrap();
let fake_repomix = bin_dir.path().join("repomix");
std::fs::write(
&fake_repomix,
"#!/bin/sh\nout=''\nwhile [ \"$#\" -gt 0 ]; do\n if [ \"$1\" = \"-o\" ]; then out=\"$2\"; shift 2; else shift; fi\ndone\nprintf '<repository><file path=\"main.py\">print(1)</file></repository>' > \"$out\"\n",
)
.unwrap();
use std::os::unix::fs::PermissionsExt;
std::fs::set_permissions(&fake_repomix, std::fs::Permissions::from_mode(0o755)).unwrap();
let out = Command::new(&py)
.arg(benchmarks_dir().join("external").join("repomix_adapter.py"))
.arg(repo_root().join("fixtures").join("cli-service"))
.arg("8000")
.arg(out_dir.path())
.arg("--compress")
.env("SCC_REPOMIX_PKG_DIR", fake_pkg.path())
.env("SCC_REPOMIX_SRC_DIR", "/nonexistent-scc-bench-repomix")
.env("PATH", format!("{}:/usr/bin:/bin", bin_dir.path().display()))
.output()
.expect("adapter runs");
assert_eq!(out.status.code(), Some(0), "gitHead match packs successfully");
let payload: serde_json::Value =
serde_json::from_slice(&out.stdout).expect("JSON on stdout");
assert_eq!(payload["ok"], true);
assert_eq!(payload["pinned"], "e3b15a406ed78d8a463620a032a059ce911bfc0e");
let artifact = PathBuf::from(payload["artifact"].as_str().unwrap());
assert!(artifact.is_file(), "artifact written: {}", artifact.display());
}