use clap::{CommandFactory, Parser, Subcommand};
use scc_cli::commands;
use scc_core::estimate_tokens;
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
#[derive(Parser)]
#[command(
name = "scc",
version,
about = "System Context Compiler — compile repositories into evidence-backed system context for coding agents",
long_about = "Continuously compiles code, configuration, infrastructure, runtime evidence, and architectural intent \
into an evidence-backed machine model of a software system, then emits small task-specific context packs \
for coding agents. Give agents more repository understanding per token."
)]
struct Cli {
#[command(subcommand)]
command: Commands,
#[arg(long, global = true)]
root: Option<PathBuf>,
}
#[derive(Subcommand)]
enum PluginSub {
List,
Describe {
id: String,
},
Inspect {
id: String,
},
Doctor,
Lock,
Check,
Graph,
Invoke {
operation: String,
#[arg(default_value = "{}")]
input: String,
},
Enable {
id: String,
},
Disable {
id: String,
},
}
#[derive(Subcommand)]
enum Commands {
Init,
Index {
#[arg(long, num_args = 1.., value_delimiter = ' ')]
paths: Vec<String>,
#[arg(long)]
quiet: bool,
#[arg(long)]
resolve: bool,
},
Status,
Scan {
path: Option<String>,
#[arg(long)]
json: bool,
},
Languages,
Watch,
Overview {
#[arg(long)]
json: bool,
},
Context {
#[command(subcommand)]
sub: ContextSub,
},
Surface {
#[arg(long)]
task: Option<String>,
#[arg(long)]
budget: Option<usize>,
#[arg(long)]
explain: bool,
},
Impact {
#[arg(long)]
diff: Option<String>,
#[arg(value_delimiter = ' ')]
files: Vec<String>,
#[arg(long)]
symbols: Vec<String>,
#[arg(long)]
json: bool, #[arg(long)]
unbounded: bool,
},
Atlas {
#[arg(long)]
budget: Option<usize>,
#[arg(long)]
json: bool,
#[arg(long)]
full: bool, #[arg(long)]
unbounded: bool,
#[arg(long)]
resolve: bool,
},
Verify {
#[arg(long)]
warnings: bool,
#[arg(long)]
json: bool, #[arg(long)]
unbounded: bool,
},
Drift {
#[arg(long)]
json: bool,
},
System {
#[arg(long, required = true)]
member: Vec<PathBuf>,
#[arg(long)]
json: bool,
},
Export {
#[arg(default_value = "system-ir.json")]
format: String,
},
Query {
query: String,
#[arg(long, default_value_t = 20)]
limit: usize,
},
Traverse {
#[arg(long)]
kind: Option<String>,
#[arg(long)]
name: Option<String>,
#[arg(long)]
from: Vec<String>,
#[arg(long = "step")]
steps: Vec<String>,
#[arg(long, default_value_t = 100)]
limit: usize,
},
Components,
Diagram {
#[arg(long, default_value = "mermaid")]
format: String,
#[arg(long)]
out: Option<String>,
},
View {
#[arg(long)]
port: Option<u16>,
#[arg(long)]
no_open: bool,
},
Snap {
#[arg(long)]
out: Option<String>,
#[arg(long, default_value_t = 40000)]
max_chars: usize,
#[arg(long)]
png: Option<String>,
#[arg(long)]
color: bool,
},
Flows,
Cochange {
#[arg(long, default_value_t = 2)]
min_commits: u32,
},
Checkpoint {
#[command(subcommand)]
sub: CheckpointSub,
},
History {
#[arg(long)]
json: bool,
},
Diff {
#[arg(long)]
from: i64,
#[arg(long)]
to: i64,
#[arg(long)]
json: bool,
},
Snapshot {
#[command(subcommand)]
sub: SnapshotSub,
},
CheckInvariants,
Setup {
#[command(subcommand)]
sub: Option<SetupSub>,
},
Update {
#[arg(long)]
version: Option<String>,
#[arg(long)]
dir: Option<PathBuf>,
#[arg(long)]
dry_run: bool,
},
Serve,
Mcp,
Rpc {
#[arg(long)]
stdio: bool,
},
#[command(alias = "operation")]
Operations {
#[arg(long)]
describe: Option<String>,
},
Plugin {
#[command(subcommand)]
sub: PluginSub,
},
Ingest {
body: String,
},
Resolve {
#[arg(long)]
lsp: bool,
},
Embed,
Important {
#[arg(long, default_value_t = 15)]
limit: usize,
#[arg(long)]
component: Option<String>,
#[arg(long)]
task: Option<String>,
#[arg(long)]
json: bool,
},
Adapters {
#[arg(long)]
json: bool,
},
Doctor {
#[arg(long)]
json: bool,
#[arg(long)]
deep: bool,
#[arg(long)]
network: bool,
#[arg(long)]
strict: bool,
},
Timing {
#[arg(long)]
all: bool,
#[arg(long, default_value_t = 0)]
last: usize,
#[arg(long)]
json: bool,
},
Clean {
#[arg(long)]
force: bool,
},
Completions {
shell: clap_complete::Shell,
},
Lessons {
#[command(subcommand)]
sub: Option<LessonsSub>,
},
Beads,
Import {
format: String,
file: String,
},
Runtime {
#[command(subcommand)]
sub: RuntimeSub,
},
Ci {
#[command(subcommand)]
sub: CiSub,
},
Bench {
#[command(subcommand)]
sub: BenchSub,
},
StatePath,
}
#[derive(Subcommand)]
enum RuntimeSub {
Status {
#[arg(long)]
json: bool,
},
Reconcile {
#[arg(long)]
json: bool,
},
}
#[derive(Subcommand)]
enum CiSub {
Check {
#[arg(long, default_value = "medium")]
max_severity: String,
},
}
#[derive(Subcommand)]
enum BenchSub {
Index {
#[arg(long, default_value_t = 200)]
files: usize,
#[arg(long, default_value_t = 250)]
lines: usize,
},
Context {
#[arg(long, default_value_t = 0.6)]
min_recall: f64,
#[arg(long, default_value_t = 0.0)]
min_precision: f64,
},
Retrieval {
#[arg(long, default_value_t = 10)]
k: usize,
#[arg(long, default_value = "lexical-then-graph,query-routed,production-blended")]
arms: String,
#[arg(long)]
repo: Option<String>,
#[arg(long, default_value_t = 0.0)]
min_recall: f64,
#[arg(long)]
json: bool,
},
Loop {
#[arg(long, default_value = "baseline,scc,ripwire")]
arms: String,
#[arg(long, default_value_t = 10)]
k: usize,
#[arg(long)]
repo: Option<String>,
#[arg(long)]
ripwire_bin: Option<PathBuf>,
#[arg(long, default_value_t = 0.0)]
min_delta: f64,
#[arg(long)]
json: bool,
#[arg(long)]
explore: bool,
},
Agent {
#[arg(long)]
cmd: String,
#[arg(long, default_value_t = 0.0)]
min_files: f64,
#[arg(long)]
gate: bool,
#[arg(long)]
baseline_cmd: Option<String>,
},
Resolution {
#[arg(long, default_value_t = scc_cli::benchres::DEFAULT_MIN_AGREEMENT)]
min_agreement: f64,
},
Atlas {
#[arg(long)]
corpus: Option<PathBuf>,
#[arg(long)]
ground_truth: Option<PathBuf>,
#[arg(long)]
json: bool,
#[arg(long)]
diagnose: bool,
#[arg(long, conflicts_with = "blind")]
holdout: bool,
#[arg(long, conflicts_with = "holdout")]
blind: bool,
#[arg(long)]
no_resolve: bool,
#[arg(long, num_args = 2, value_names = ["OLD", "NEW"], conflicts_with_all = ["blind", "holdout"])]
compare: Option<Vec<PathBuf>>,
#[arg(long, default_value_t = 0.0)]
gate_ge: f64,
#[arg(long, default_value_t = 0.05)]
guard_section_delta: f64,
},
External {
#[arg(long)]
variant: String,
#[arg(long)]
repo: Option<String>,
#[arg(long, default_value_t = 8000)]
budget: usize,
#[arg(long)]
cmd: Option<String>,
#[arg(long)]
workdir: Option<PathBuf>,
#[arg(long)]
json: bool,
#[arg(long)]
artifact_only: Option<String>,
},
}
#[derive(Subcommand)]
enum ContextSub {
Startup {
#[arg(long)]
budget: Option<usize>,
#[arg(long)]
full_size: bool,
},
Structural {
#[arg(long, value_delimiter = ' ', required_unless_present = "task")]
files: Vec<String>,
#[arg(long)]
task: Option<String>,
#[arg(long)]
budget: Option<usize>,
},
Task {
goal: String,
#[arg(long, value_delimiter = ' ')]
files: Vec<String>,
#[arg(long)]
symbols: Vec<String>,
#[arg(long)]
budget: Option<usize>,
#[arg(long)]
json: bool,
#[arg(long, hide = true)]
hook: bool,
#[arg(long)]
resolve: bool,
},
Component {
id: String,
#[arg(long)]
json: bool, #[arg(long)]
unbounded: bool,
},
Flow {
id: String,
#[arg(long)]
json: bool, #[arg(long)]
unbounded: bool,
},
Compress {
goal: String,
#[arg(long)]
cmd: Option<String>,
#[arg(long)]
claims: bool,
#[arg(long)]
budget: Option<usize>,
#[arg(long)]
json: bool,
},
Docs {
dependency: String,
},
Subagent {
goal: String,
#[arg(long, value_delimiter = ' ')]
files: Vec<String>,
#[arg(long)]
symbols: Vec<String>,
#[arg(long)]
budget: Option<usize>,
#[arg(long)]
json: bool,
},
}
#[derive(Subcommand)]
enum SnapshotSub {
Save {
#[arg(long)]
task: String,
#[arg(long)]
budget: Option<usize>,
#[arg(long)]
json: bool,
},
Show {
id: String,
},
Diff {
id: String,
#[arg(long)]
json: bool,
},
}
#[derive(Subcommand)]
enum CheckpointSub {
Save {
#[arg(long)]
json: bool,
},
Load {
#[arg(long)]
inject: bool,
},
}
#[derive(Subcommand)]
enum LessonsSub {
Add {
text: String,
},
List {
#[arg(long, default_value_t = 20)]
limit: usize,
},
}
#[derive(Subcommand)]
enum SetupSub {
Claude,
Codex,
Opencode,
Hermes,
Omp,
Pi,
All,
}
fn main() -> Result<(), Box<dyn std::error::Error>> {
unsafe {
libc::signal(libc::SIGPIPE, libc::SIG_DFL);
}
let cli = Cli::parse();
let root = match cli.root.clone() {
Some(r) if r.is_absolute() => r,
Some(r) => std::env::current_dir().unwrap().join(r),
None => scc_cli::find_root(&std::env::current_dir().unwrap()),
};
let result = match cli.command {
Commands::Init => commands::cmd_init(&root),
Commands::Index { paths, quiet, resolve } => {
if resolve {
commands::cmd_index(&root, quiet)?;
let rep = scc_cli::resolve_and_recompile(&root)?;
if !quiet {
println!(
"resolved: {} upgraded, {} unresolved, {} errors",
rep.upgraded, rep.unresolved, rep.errors
);
}
Ok(())
} else if paths.is_empty() {
commands::cmd_index(&root, quiet)
} else {
commands::cmd_index_paths(&root, &paths, quiet)
}
}
Commands::Status => commands::cmd_status(&root),
Commands::Scan { path, json } => commands::cmd_scan(&root, path.as_deref(), json),
Commands::Languages => commands::cmd_languages(),
Commands::Watch => commands::cmd_watch(&root),
Commands::Overview { json } => commands::cmd_overview(&root, json),
Commands::Context { sub } => match sub {
ContextSub::Startup { budget, full_size } => commands::cmd_context_startup(&root, budget, full_size),
ContextSub::Structural { files, task, budget } => {
let out =
commands::cmd_context_structural(&root, &files, task.as_deref(), budget)?;
print!("{out}");
Ok(())
}
ContextSub::Task { goal, files, symbols, budget, json, hook, resolve } => {
if resolve {
let rep = scc_cli::resolve_and_recompile(&root)?;
eprintln!(
"resolved: {} upgraded, {} unresolved, {} errors",
rep.upgraded, rep.unresolved, rep.errors
);
}
commands::cmd_context_task(&root, &goal, &files, &symbols, budget, json, hook)
}
ContextSub::Component { id, json, unbounded } => commands::cmd_context_component(&root, &id, json, unbounded),
ContextSub::Flow { id, json, unbounded } => commands::cmd_context_flow(&root, &id, json, unbounded),
ContextSub::Docs { dependency } => commands::cmd_context_docs(&root, &dependency),
ContextSub::Subagent { goal, files, symbols, budget, json } => {
commands::cmd_context_subagent(&root, &goal, &files, &symbols, budget, json)
}
ContextSub::Compress { goal, cmd, claims, budget, json } => {
if claims {
let out = scc_cli::compress::cmd_context_compress_json_claims(
&root, &goal, cmd, budget, true,
)?;
if json {
println!("{out}");
} else {
let pack: scc_context::ContextPack = serde_json::from_str(&out)?;
print!("{}", pack.content);
}
Ok(())
} else {
scc_cli::compress::cmd_context_compress(&root, &goal, cmd, budget, json)
}
}
},
Commands::Impact { diff, files, symbols, json, unbounded } => {
commands::cmd_impact(&root, &files, &symbols, diff.as_deref(), json, unbounded)
}
Commands::Surface { task, budget, explain } => {
commands::cmd_surface(&root, task.as_deref(), budget, explain)
}
Commands::Atlas { budget, json, full, unbounded, resolve } => {
if resolve {
let rep = scc_cli::resolve_and_recompile(&root)?;
eprintln!(
"resolved: {} upgraded, {} unresolved, {} errors",
rep.upgraded, rep.unresolved, rep.errors
);
}
commands::cmd_atlas(&root, budget, json, full, unbounded)
}
Commands::Verify { warnings, json, unbounded } => commands::cmd_verify(&root, warnings, json, unbounded),
Commands::Drift { json } => commands::cmd_drift(&root, json),
Commands::System { member, json } => commands::cmd_system(&root, &member, json),
Commands::Export { format } => commands::cmd_export(&root, &format),
Commands::Query { query, limit } => commands::cmd_query(&root, &query, limit),
Commands::Traverse { kind, name, from, steps, limit } => {
commands::cmd_traverse(&root, kind.as_deref(), name.as_deref(), &from, &steps, limit)
}
Commands::Components => commands::cmd_list_components(&root),
Commands::Diagram { format, out } => {
commands::cmd_diagram(&root, &format, out.as_deref())
}
Commands::View { port, no_open } => commands::cmd_view(&root, port, no_open),
Commands::Snap { out, max_chars, png, color } => {
commands::cmd_snap(&root, out.as_deref(), max_chars, png.as_deref(), color)
}
Commands::Flows => commands::cmd_list_flows(&root),
Commands::Cochange { min_commits } => commands::cmd_cochange(&root, min_commits),
Commands::Checkpoint { sub } => match sub {
CheckpointSub::Save { json } => commands::cmd_checkpoint_save(&root, json),
CheckpointSub::Load { inject } => commands::cmd_checkpoint_load(&root, inject),
},
Commands::History { json } => commands::cmd_history(&root, json),
Commands::Diff { from, to, json } => commands::cmd_diff(&root, from, to, json),
Commands::Snapshot { sub } => match sub {
SnapshotSub::Save { task, budget, json } => {
commands::cmd_snapshot_save(&root, &task, budget, json)
}
SnapshotSub::Show { id } => commands::cmd_snapshot_show(&root, &id),
SnapshotSub::Diff { id, json } => commands::cmd_snapshot_diff(&root, &id, json),
},
Commands::CheckInvariants => match commands::cmd_check_invariants(&root) {
Ok(true) => Ok(()),
Ok(false) => Err(scc_cli::CliError::Other(
"graph invariants violated".into(),
)),
Err(e) => Err(e),
},
Commands::Update { version, dir, dry_run } => {
commands::cmd_update(version.as_deref(), dir.as_deref(), dry_run)
}
Commands::Setup { sub } => match sub {
None => commands::cmd_setup_detected(&root, false),
Some(SetupSub::Claude) => commands::cmd_setup_claude(&root),
Some(SetupSub::Codex) => scc_cli::compress::cmd_setup_codex(&root),
Some(SetupSub::Opencode) => scc_cli::compress::cmd_setup_opencode(&root),
Some(SetupSub::Hermes) => scc_cli::plugin_hermes::cmd_setup_hermes(&root),
Some(SetupSub::Omp) => scc_cli::plugin_omp::cmd_setup_omp(&root),
Some(SetupSub::Pi) => scc_cli::plugin_omp::cmd_setup_pi(&root),
Some(SetupSub::All) => commands::cmd_setup_detected(&root, true),
},
Commands::Serve => commands::cmd_serve(&root),
Commands::Mcp => commands::cmd_mcp(&root),
Commands::Rpc { stdio } => commands::cmd_rpc(&root, stdio),
Commands::Operations { describe } => commands::cmd_operations(describe.as_deref()),
Commands::Plugin { sub } => match sub {
PluginSub::List => commands::cmd_plugin_list(&root),
PluginSub::Describe { id } => commands::cmd_plugin_describe(&root, id.as_str()),
PluginSub::Inspect { id } => commands::cmd_plugin_describe(&root, id.as_str()),
PluginSub::Doctor => commands::cmd_plugin_doctor(&root),
PluginSub::Lock => commands::cmd_plugin_lock(&root),
PluginSub::Check => commands::cmd_plugin_check(&root),
PluginSub::Graph => commands::cmd_plugin_graph(&root),
PluginSub::Invoke { operation, input } => commands::cmd_plugin_invoke(&root, operation.as_str(), input.as_str()),
PluginSub::Enable { id } => commands::cmd_plugin_enable(&root, id.as_str()),
PluginSub::Disable { id } => commands::cmd_plugin_disable(&root, id.as_str()),
},
Commands::Ingest { body } => commands::cmd_ingest_runtime(&root, &body),
Commands::Embed => scc_cli::embed_cli::cmd_embed(&root),
Commands::Important { limit, component, task, json } => {
commands::cmd_important(&root, limit, component.as_deref(), task.as_deref(), json)
}
Commands::Adapters { json } => commands::cmd_adapters(&root, json),
Commands::Completions { shell } => {
clap_complete::generate(shell, &mut Cli::command(), "scc", &mut std::io::stdout());
Ok(())
}
Commands::Timing { all, last, json } => commands::cmd_timing(&root, all, last, json),
Commands::Clean { force } => commands::cmd_clean(&root, force),
Commands::Doctor { json, deep, network, strict } => match commands::cmd_doctor(&root, json, deep, network, strict) {
Ok(true) => Ok(()),
Ok(false) => Err(scc_cli::CliError::Other("scc doctor: issues found".into())),
Err(e) => Err(e),
},
Commands::Lessons { sub } => match sub.unwrap_or(LessonsSub::List { limit: 20 }) {
LessonsSub::Add { text } => commands::cmd_lessons_add(&root, &text),
LessonsSub::List { limit } => commands::cmd_lessons_list(&root, limit),
},
Commands::Beads => commands::cmd_beads(&root),
Commands::Resolve { lsp: true } => {
let pre = {
let store = scc_cli::open_store(&root)?;
scc_cli::benchres::collect_external_edges(&store)?
};
scc_cli::resolve::cmd_resolve_lsp(&root)?;
let store = scc_cli::open_store(&root)?;
scc_cli::recompile(&store)?;
let mut by_file: std::collections::BTreeMap<
String,
Vec<scc_indexer::conflicts::UpgradeRecord>,
> = std::collections::BTreeMap::new();
for (file, rec) in scc_cli::benchres::diff_upgrades(&store, &pre)? {
by_file.entry(file).or_default().push(rec);
}
for (file, recs) in &by_file {
let report =
scc_indexer::conflicts::record_resolution_conflicts(&store, file, recs)?;
if report.conflicts > 0 {
println!(
"scc resolve: {} resolution conflict(s) in {file}",
report.conflicts
);
}
}
Ok(())
}
Commands::Resolve { lsp: false } => Err(scc_cli::CliError::Other(
"no resolver selected (use --lsp)".into(),
)),
Commands::Import { format, file } => commands::cmd_import(&root, &format, &file),
Commands::Runtime { sub } => match sub {
RuntimeSub::Status { json } => commands::cmd_runtime_status(&root, json),
RuntimeSub::Reconcile { json } => commands::cmd_runtime_reconcile(&root, json),
},
Commands::Ci { sub } => match sub {
CiSub::Check { max_severity } => match commands::cmd_ci_check(&root, &max_severity) {
Ok(true) => Ok(()),
Ok(false) => Err(scc_cli::CliError::Other(
"CI check failed: invariants or drift policy violated".into(),
)),
Err(e) => Err(e),
},
},
Commands::Bench { sub } => match sub {
BenchSub::Agent { cmd, min_files, gate, baseline_cmd } => {
if gate {
let bc = baseline_cmd.ok_or(scc_cli::CliError::Other(
"--gate requires --baseline-cmd (the A-variant command)".into(),
))?;
match scc_cli::benchagent::run_agent_gate(&bc, &cmd, min_files) {
Ok(g) => {
scc_cli::benchagent::print_agent_gate(&g);
if g.passed {
Ok(())
} else {
Err(scc_cli::CliError::Other(
"agent-behavior gate FAILED: the atlas variant does not reduce exploration (search calls / files opened / first-correct time)".into(),
))
}
}
Err(e) => Err(scc_cli::CliError::Other(e)),
}
} else {
match scc_cli::benchagent::run_agent_benchmark(&cmd, min_files) {
Ok(summary) => {
scc_cli::benchagent::print_agent_summary(&summary);
Ok(())
}
Err(e) => Err(scc_cli::CliError::Other(e)),
}
}
},
BenchSub::Loop {
arms,
k,
repo,
ripwire_bin,
min_delta,
json,
explore,
} => {
let parsed: Vec<scc_cli::benchloop::LoopArm> = arms
.split(',')
.map(|s| s.trim())
.filter(|s| !s.is_empty())
.map(|s| {
scc_cli::benchloop::LoopArm::parse(s).ok_or_else(|| {
scc_cli::CliError::Other(format!("unknown loop arm {s}"))
})
})
.collect::<std::result::Result<Vec<_>, _>>()?;
let opts = scc_cli::benchloop::LoopOptions {
k,
repo_filter: repo,
ripwire_bin,
explore,
agent_cmd: std::env::var("SCC_EXPLORE_AGENT_CMD").ok().filter(|s| !s.is_empty()),
};
match scc_cli::benchloop::run_agent_loop(&parsed, &opts) {
Ok(summary) => {
if json {
println!("{}", serde_json::to_string_pretty(&summary)?);
} else {
scc_cli::benchloop::print_loop_summary(&summary);
}
let gate_fail = {
let base = summary.arms.iter().find(|a| a.arm == "baseline");
let scc = summary.arms.iter().find(|a| a.arm == "scc");
match (base, scc) {
(Some(b), Some(s))
if s.clustered_localization + min_delta
< b.clustered_localization =>
{
Some(format!(
"agent-loop gate failed: SCC clustered {:.3} < baseline {:.3} + {min_delta}",
s.clustered_localization, b.clustered_localization
))
}
_ => None,
}
};
match gate_fail {
Some(msg) => Err(scc_cli::CliError::Other(msg)),
None => Ok(()),
}
}
Err(e) => Err(scc_cli::CliError::Other(e)),
}
},
BenchSub::Context { min_recall, min_precision } => match scc_cli::benchctx::run_context_benchmark(min_recall, min_precision)
{
Ok(summary) => {
scc_cli::benchctx::print_summary(&summary);
Ok(())
}
Err(e) => Err(scc_cli::CliError::Other(e)),
},
BenchSub::Retrieval {
k,
arms,
repo,
min_recall,
json,
} => {
let parsed: Vec<scc_core::RankingArm> = arms
.split(',')
.map(|s| s.trim())
.filter(|s| !s.is_empty())
.map(|s| {
scc_core::RankingArm::parse(s).ok_or_else(|| {
scc_cli::CliError::Other(format!("unknown ranking arm {s}"))
})
})
.collect::<std::result::Result<Vec<_>, _>>()?;
match scc_cli::benchret::run_retrieval_benchmark(
k,
&parsed,
min_recall,
repo.as_deref(),
) {
Ok(summary) => {
if json {
println!("{}", serde_json::to_string_pretty(&summary)?);
} else {
scc_cli::benchret::print_summary(&summary);
}
Ok(())
}
Err(e) => Err(scc_cli::CliError::Other(e)),
}
},
BenchSub::Index { files, lines } => {
let dir = std::env::temp_dir().join(format!("scc-bench-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir)?;
match scc_cli::bench::bench_index(&dir, files, lines) {
Ok(r) => {
scc_cli::bench::print_report(&r);
let _ = std::fs::remove_dir_all(&dir);
Ok(())
}
Err(e) => {
let _ = std::fs::remove_dir_all(&dir);
Err(e)
}
}
}
BenchSub::Resolution { min_agreement } => {
match scc_cli::benchres::run_resolution_benchmark(min_agreement) {
Ok(summary) => {
scc_cli::benchres::print_summary(&summary);
Ok(())
}
Err(e) => Err(scc_cli::CliError::Other(e)),
}
}
BenchSub::External { variant, repo, budget, cmd, workdir, json, artifact_only } => {
let v = variant.as_str();
if let Some(goal) = artifact_only {
if !NATIVE_VARIANTS.contains(&v) {
return Err(scc_cli::CliError::Other(format!(
"--artifact-only supports native variants only, got {v:?}"
))
.into());
}
let repo = repo.ok_or_else(|| {
scc_cli::CliError::Other("--artifact-only requires --repo".into())
})?;
let workdir = workdir
.clone()
.unwrap_or_else(|| std::env::temp_dir().join("scc-artifact-only"));
std::fs::create_dir_all(&workdir)
.map_err(|e| scc_cli::CliError::Other(e.to_string()))?;
let task = scc_cli::benchagent::VariantTask {
id: format!("{repo}--artifact-only"),
repo: repo.clone(),
goal: goal.clone(),
files: vec![],
plan_keys: vec![],
};
let fixtures = scc_cli::benchctx::locate_fixtures_dir()
.ok_or_else(|| scc_cli::CliError::Other(
"cannot locate fixtures/ directory".into(),
))?;
let root = fixtures.join(&repo);
let (path, tokens) = generate_variant_artifact(
v, &task, &root, budget, &workdir,
)
.map_err(scc_cli::CliError::Other)?;
let _ = json; println!(
"{}",
serde_json::json!({
"artifact": path,
"tokens": tokens,
"budget": budget,
"variant": v,
"goal": goal,
"repo": repo,
})
);
return Ok(());
}
scc_cli::benchagent::require_paid_opt_in().map_err(scc_cli::CliError::Other)?;
if EXTERNAL_VARIANTS.contains(&v) {
run_external_python_delegation(v, repo.as_deref(), budget, cmd.as_deref(), json)
} else if NATIVE_VARIANTS.contains(&v) {
run_external_variant(v, repo.as_deref(), budget, cmd.as_deref(), workdir.as_deref(), json)
} else {
Err(scc_cli::CliError::Other(format!(
"unknown external variant {v:?} (expected one of: {})",
EXTERNAL_VARIANTS
.iter()
.chain(NATIVE_VARIANTS.iter())
.copied()
.collect::<Vec<_>>()
.join(", ")
)))
}
}
BenchSub::Atlas {
corpus,
ground_truth,
json,
diagnose,
holdout,
blind,
no_resolve,
compare,
gate_ge,
guard_section_delta,
} => {
let resolve = !no_resolve;
if let Some(pair) = compare {
if pair.len() != 2 {
return Err(scc_cli::CliError::Other(
"--compare requires exactly two result files: OLD NEW".into(),
)
.into());
}
let old = scc_cli::benchatlas::load_holdout_result(&pair[0])?;
let new = scc_cli::benchatlas::load_holdout_result(&pair[1])?;
let mut report = scc_cli::benchatlas::compare_runs(
&old,
&new,
gate_ge,
guard_section_delta,
);
report.old_file = pair[0].display().to_string();
report.new_file = pair[1].display().to_string();
if json {
println!(
"{}",
serde_json::to_string_pretty(&report)
.map_err(|e| scc_cli::CliError::Other(e.to_string()))?
);
} else {
scc_cli::benchatlas::print_compare_report(&report);
}
if report.passed() {
Ok(())
} else {
Err(scc_cli::CliError::Other(
"Wave-11 generalization gates FAILED: the semantic wave does not generalize (see --compare report)".into(),
))
}
} else if blind {
match scc_cli::benchatlas::run_atlas_blind(diagnose, resolve) {
Ok(comparison) => {
if json {
println!(
"{}",
serde_json::to_string_pretty(&comparison)
.map_err(|e| scc_cli::CliError::Other(e.to_string()))?
);
} else {
scc_cli::benchatlas::print_blind_report(&comparison);
}
Ok(())
}
Err(e) => Err(scc_cli::CliError::Other(e)),
}
} else if holdout {
match scc_cli::benchatlas::run_atlas_holdout(
corpus.as_deref(),
ground_truth.as_deref(),
diagnose,
resolve,
) {
Ok(comparison) => {
if json {
println!(
"{}",
serde_json::to_string_pretty(&comparison)
.map_err(|e| scc_cli::CliError::Other(e.to_string()))?
);
} else {
scc_cli::benchatlas::print_holdout_report(&comparison, diagnose);
}
Ok(())
}
Err(e) => Err(scc_cli::CliError::Other(e)),
}
} else {
match scc_cli::benchatlas::run_atlas_bench(
corpus.as_deref(),
ground_truth.as_deref(),
diagnose,
resolve,
) {
Ok(report) => {
if json {
println!(
"{}",
serde_json::to_string_pretty(&report)
.map_err(|e| scc_cli::CliError::Other(e.to_string()))?
);
} else {
scc_cli::benchatlas::print_report(&report, diagnose);
}
Ok(())
}
Err(e) => Err(scc_cli::CliError::Other(e)),
}
}
},
},
Commands::StatePath => {
println!("{}", scc_cli::state_dir(&root).display());
Ok(())
}
};
match result {
Ok(()) => Ok(()),
Err(e) => {
eprintln!("error: {e}");
std::process::exit(1);
}
}
}
const EXTERNAL_VARIANTS: [&str; 2] = ["aider-repomap", "repomix-compress"];
const NATIVE_VARIANTS: [&str; 11] = [
"raw",
"scc-atlas",
"scc-surface",
"scc-atlas-surface",
"scc-full",
"lexical",
"global-ppr",
"task-ppr",
"ppr-mmr",
"ppr-quotas",
"ppr-optimizer",
];
const DEFAULT_AGENT_CMD: &str = "codex exec --json --sandbox read-only --skip-git-repo-check --ephemeral --color never -C . -";
fn benchmarks_dir() -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.parent()
.and_then(|p| p.parent())
.map(|p| p.join("benchmarks"))
.unwrap_or_else(|| PathBuf::from("benchmarks"))
}
#[derive(Debug, Clone, serde::Deserialize)]
struct ExternalGroundTruth {
#[serde(default)]
repos: BTreeMap<String, ExternalRepoTasks>,
}
#[derive(Debug, Clone, Default, serde::Deserialize)]
struct ExternalRepoTasks {
#[serde(default)]
tasks: Vec<ExternalTaskDef>,
}
#[derive(Debug, Clone, serde::Deserialize)]
struct ExternalTaskDef {
id: String,
goal: String,
#[serde(default)]
public_surfaces: Vec<String>,
#[serde(default)]
important_types: Vec<String>,
#[serde(default)]
implementation_landmarks: Vec<String>,
#[serde(default)]
symbols: Vec<String>,
}
fn load_external_tasks(repo_filter: Option<&str>) -> Result<Vec<scc_cli::benchagent::VariantTask>, String> {
let mut by_id: BTreeMap<String, scc_cli::benchagent::VariantTask> = BTreeMap::new();
let tasks_json = benchmarks_dir().join("tasks.json");
if tasks_json.is_file() {
let text = std::fs::read_to_string(&tasks_json).map_err(|e| e.to_string())?;
let corpus: scc_cli::benchctx::BenchmarkCorpus =
serde_json::from_str(&text).map_err(|e| e.to_string())?;
for t in &corpus.tasks {
let mut plan_keys = t.ground_truth.files.clone();
plan_keys.extend(t.ground_truth.symbols.iter().cloned());
by_id.insert(
t.id.clone(),
scc_cli::benchagent::VariantTask {
id: t.id.clone(),
repo: t.repo.clone(),
goal: t.goal.clone(),
files: t.ground_truth.files.clone(),
plan_keys,
},
);
}
}
let yaml_path = benchmarks_dir().join("external").join("ground-truth.yaml");
if yaml_path.is_file() {
let text = std::fs::read_to_string(&yaml_path).map_err(|e| e.to_string())?;
let gt: ExternalGroundTruth = serde_yaml::from_str(&text).map_err(|e| e.to_string())?;
for (repo, spec) in gt.repos {
for t in spec.tasks {
let mut plan_keys = t.implementation_landmarks.clone();
plan_keys.extend(t.public_surfaces.iter().cloned());
plan_keys.extend(t.important_types.iter().cloned());
plan_keys.extend(t.symbols.iter().cloned());
by_id.insert(
t.id.clone(),
scc_cli::benchagent::VariantTask {
id: t.id.clone(),
repo: repo.clone(),
goal: t.goal.clone(),
files: t.implementation_landmarks.clone(),
plan_keys,
},
);
}
}
}
let mut tasks: Vec<scc_cli::benchagent::VariantTask> = by_id.into_values().collect();
if let Some(r) = repo_filter {
tasks.retain(|t| t.repo == r);
}
tasks.sort_by(|a, b| a.repo.cmp(&b.repo).then(a.id.cmp(&b.id)));
Ok(tasks)
}
fn run_scc_capture(root: &Path, args: &[&str]) -> Result<String, String> {
let exe = std::env::current_exe().map_err(|e| format!("current exe: {e}"))?;
let out = std::process::Command::new(&exe)
.args(args)
.current_dir(root)
.output()
.map_err(|e| format!("spawn {}: {e}", exe.display()))?;
if !out.status.success() {
return Err(format!(
"`scc {}` exited {}: {}",
args.join(" "),
out.status,
String::from_utf8_lossy(&out.stderr).trim()
));
}
Ok(String::from_utf8_lossy(&out.stdout).to_string())
}
fn generate_variant_artifact(
variant: &str,
task: &scc_cli::benchagent::VariantTask,
root: &Path,
budget: usize,
artifacts_dir: &Path,
) -> Result<(PathBuf, usize), String> {
std::fs::create_dir_all(artifacts_dir).map_err(|e| e.to_string())?;
let artifact_path = artifacts_dir.join(format!("{}.txt", task.repo));
let budget_s = budget.to_string();
let text: String = match variant {
"raw" => String::new(),
"scc-atlas" => {
run_scc_capture(root, &["atlas", "--budget", &budget_s])?
}
"scc-surface" => run_scc_capture(root, &["surface", "--budget", &budget_s])?,
"scc-atlas-surface" => run_scc_capture(root, &["context", "startup", "--budget", &budget_s])?,
"scc-full" => {
let startup_budget = (budget / 2).max(1);
let startup = run_scc_capture(
root,
&["context", "startup", "--budget", &startup_budget.to_string()],
)?;
let est_s = estimate_tokens(&startup);
let remaining = budget.saturating_sub(est_s);
let task_budget = (budget / 4).min(remaining / 2).max(1);
let task_pack = run_scc_capture(
root,
&["context", "task", &task.goal, "--budget", &task_budget.to_string()],
)?;
let est_t = estimate_tokens(&task_pack);
let mut structural_budget = budget.saturating_sub(est_s + est_t);
let mut structural = scc_cli::benchagent::structural_source_for_goal(
root,
&task.goal,
structural_budget,
)?;
let mut text = format!("{startup}\n\n{task_pack}\n\n# STRUCTURAL SOURCE\n\n{structural}");
text = text.trim_end().to_string();
while estimate_tokens(&text) > budget && structural_budget > 0 {
structural_budget =
structural_budget.saturating_sub(estimate_tokens(&text) - budget + 1);
structural = scc_cli::benchagent::structural_source_for_goal(
root,
&task.goal,
structural_budget,
)?;
text = format!("{startup}\n\n{task_pack}\n\n# STRUCTURAL SOURCE\n\n{structural}")
.trim_end()
.to_string();
}
text
}
other => {
let mode = scc_cli::benchagent::SurfaceAblation::parse(other).ok_or_else(|| {
format!("unsupported native variant: {other}")
})?;
scc_cli::benchagent::render_ablation_surface(root, mode, &task.goal, budget)?
}
};
std::fs::write(&artifact_path, &text).map_err(|e| e.to_string())?;
Ok((artifact_path, estimate_tokens(&text)))
}
fn sh_single_quote(s: &str) -> String {
format!("'{}'", s.replace('\'', "'\\''"))
}
fn build_variant_command(agent_cmd: &str, artifact_path: &Path) -> String {
let art = sh_single_quote(&artifact_path.display().to_string());
let agent = sh_single_quote(agent_cmd);
format!(
"CTX=$(cat {art} 2>/dev/null || true); printf 'SCC CONTEXT:\\n%s\\n\\nTASK: %s\\n' \"$CTX\" \"$SCC_GOAL\" | sh -c {agent}"
)
}
fn run_external_variant(
variant: &str,
repo: Option<&str>,
budget: usize,
agent_cmd: Option<&str>,
workdir: Option<&Path>,
json: bool,
) -> Result<(), scc_cli::CliError> {
let agent = agent_cmd.unwrap_or(DEFAULT_AGENT_CMD);
let tasks = load_external_tasks(repo).map_err(scc_cli::CliError::Other)?;
if tasks.is_empty() {
return Err(scc_cli::CliError::Other(format!(
"no ground-truth tasks for repo {}",
repo.unwrap_or("(all)")
)));
}
let workdir = workdir.map(Path::to_path_buf).unwrap_or_else(|| {
std::env::temp_dir().join(format!("scc-external-{}", std::process::id()))
});
let artifacts_dir = workdir.join("artifacts");
std::fs::create_dir_all(&artifacts_dir).map_err(|e| scc_cli::CliError::Other(e.to_string()))?;
let mut tokens_seen: Vec<usize> = Vec::new();
let summary = scc_cli::benchagent::run_variant_tasks(variant, &tasks, |task, root| {
let (path, tokens) = generate_variant_artifact(variant, task, root, budget, &artifacts_dir)?;
tokens_seen.push(tokens);
Ok(build_variant_command(agent, &path))
}, 0.0)
.map_err(scc_cli::CliError::Other)?;
let mean_tokens = if tokens_seen.is_empty() {
0
} else {
tokens_seen.iter().sum::<usize>() / tokens_seen.len()
};
let n = summary.tasks.max(1) as f64;
let row = serde_json::json!({
"variant": variant,
"budget": budget,
"repo": repo.unwrap_or("all"),
"tasks": summary.tasks,
"success_rate": summary.passed as f64 / n,
"mean_exploration": summary.mean_files_opened
+ summary.mean_search_tool_calls
+ summary.mean_graph_tool_calls,
"first_plan_accuracy": summary.mean_first_plan_correct,
"context_tokens": mean_tokens,
"mean_files_opened": summary.mean_files_opened,
"mean_search_tool_calls": summary.mean_search_tool_calls,
"mean_graph_tool_calls": summary.mean_graph_tool_calls,
"mean_files_opened_before_first_correct": summary.mean_wrong_first_locations,
});
let results_dir = workdir.join("results");
std::fs::create_dir_all(&results_dir).map_err(|e| scc_cli::CliError::Other(e.to_string()))?;
let result_path = results_dir.join(format!(
"{variant}_{}_{budget}.json",
repo.unwrap_or("all")
));
let result_json = serde_json::json!({ "row": row, "summary": summary });
let _ = std::fs::write(
&result_path,
serde_json::to_string_pretty(&result_json)
.map_err(|e| scc_cli::CliError::Other(e.to_string()))?,
);
if json {
println!(
"{}",
serde_json::to_string_pretty(&row).map_err(|e| scc_cli::CliError::Other(e.to_string()))?
);
} else {
println!(
"{variant:<18} success {:>6.3} mean exploration {:>7.2} first-plan accuracy {:>6.3} context tokens {:>6}",
summary.passed as f64 / n,
summary.mean_files_opened + summary.mean_search_tool_calls + summary.mean_graph_tool_calls,
summary.mean_first_plan_correct,
mean_tokens
);
scc_cli::benchagent::print_agent_summary(&summary);
}
Ok(())
}
fn run_external_python_delegation(
variant: &str,
repo: Option<&str>,
budget: usize,
agent_cmd: Option<&str>,
json: bool,
) -> Result<(), scc_cli::CliError> {
let script = benchmarks_dir().join("external").join("run_context_bench.py");
if !script.is_file() {
return Err(scc_cli::CliError::Other(format!(
"missing harness: {}",
script.display()
)));
}
let mut cmd = std::process::Command::new("python3");
cmd.arg(&script)
.arg("--variant")
.arg(variant)
.arg("--budget")
.arg(budget.to_string())
.arg("--single");
if let Some(r) = repo {
cmd.arg("--repo").arg(r);
}
if let Some(a) = agent_cmd {
cmd.arg("--agent-cmd").arg(a);
}
if json {
cmd.arg("--json");
}
let out = cmd
.output()
.map_err(|e| scc_cli::CliError::Other(format!("spawn python harness: {e}")))?;
let stdout = String::from_utf8_lossy(&out.stdout);
if out.status.code() == Some(2) {
if json {
print!("{stdout}");
} else {
println!("{variant} SKIPPED-UNINSTALLED");
eprintln!("{stdout}");
}
return Ok(());
}
if out.status.code() == Some(3) {
if json {
print!("{stdout}");
} else {
println!("{variant} PIN-MISMATCH");
eprintln!("{stdout}");
}
return Ok(());
}
if out.status.code() == Some(4) {
if json {
print!("{stdout}");
} else {
println!("{variant} PIN-UNVERIFIED");
eprintln!("{stdout}");
}
return Ok(());
}
if !out.status.success() {
return Err(scc_cli::CliError::Other(format!(
"run_context_bench.py exited {}: {}",
out.status,
String::from_utf8_lossy(&out.stderr).trim()
)));
}
print!("{stdout}");
Ok(())
}