#![allow(
clippy::all,
unused_imports,
unused_variables,
dead_code,
unused_mut,
unused_assignments,
non_camel_case_types,
clippy::approx_constant,
unexpected_cfgs,
unused_must_use,
unused_parens
)]
use ruvllm::backends::ModelConfig;
use ruvllm::evaluation::{
swe_bench::{SweBenchConfig, SweBenchLoader},
AblationMode, EvalConfig, EvalTask, RealEvaluationHarness, RealInferenceConfig,
};
use std::env;
use std::path::PathBuf;
use std::process;
fn main() {
if env::var("RUST_LOG").is_err() {
env::set_var("RUST_LOG", "info");
}
tracing_subscriber::fmt::init();
let args: Vec<String> = env::args().collect();
if args.len() < 2 || args.contains(&"--help".to_string()) || args.contains(&"-h".to_string()) {
print_help();
return;
}
let config = match parse_args(&args[1..]) {
Ok(c) => c,
Err(e) => {
eprintln!("Error: {}", e);
eprintln!("\nRun with --help for usage information.");
process::exit(1);
}
};
if let Err(e) = run_evaluation(config) {
eprintln!("Evaluation failed: {}", e);
process::exit(1);
}
}
fn print_help() {
println!(
r#"RuvLLM Evaluation CLI
Run real LLM evaluations on SWE-Bench tasks with SONA learning and HNSW routing.
USAGE:
run_eval [OPTIONS] --model <PATH>
OPTIONS:
--model <PATH> Path to GGUF model file (required)
--tasks <SOURCE> Task source: sample, swe-bench-lite, swe-bench, or file path
(default: sample)
--max-tasks <N> Maximum number of tasks to evaluate (default: all)
--modes <MODES> Comma-separated ablation modes (default: all)
Options: baseline, retrieval, adapters, retrieval+adapters, full
--seeds <SEEDS> Comma-separated random seeds (default: 42,123,456)
--output <PATH> Output file for results JSON (default: stdout summary)
--quality-threshold <F> Minimum quality score for acceptance (default: 0.7)
--cost-target <F> Target cost per patch in dollars (default: 0.10)
--no-sona Disable SONA learning
--no-hnsw Disable HNSW routing
--repo <NAME> Filter tasks by repository name
--verbose Enable verbose output
-h, --help Show this help message
EXAMPLES:
# Quick test with sample tasks
run_eval --model ./model.gguf --tasks sample
# Run SWE-bench-lite evaluation
run_eval --model ./model.gguf --tasks swe-bench-lite --max-tasks 100
# Compare baseline vs full mode
run_eval --model ./model.gguf --modes baseline,full --output results.json
# Run on custom task file
run_eval --model ./model.gguf --tasks ./my-tasks.json --verbose
"#
);
}
#[derive(Debug)]
struct CliConfig {
model_path: PathBuf,
task_source: TaskSource,
max_tasks: Option<usize>,
ablation_modes: Vec<AblationMode>,
seeds: Vec<u64>,
output_path: Option<PathBuf>,
quality_threshold: f64,
cost_target: f64,
enable_sona: bool,
enable_hnsw: bool,
repo_filter: Option<String>,
verbose: bool,
}
#[derive(Debug)]
enum TaskSource {
Sample,
SweBenchLite,
SweBenchFull,
File(PathBuf),
}
fn parse_args(args: &[String]) -> Result<CliConfig, String> {
let mut model_path: Option<PathBuf> = None;
let mut task_source = TaskSource::Sample;
let mut max_tasks = None;
let mut ablation_modes = Vec::new();
let mut seeds = vec![42, 123, 456];
let mut output_path = None;
let mut quality_threshold = 0.7;
let mut cost_target = 0.10;
let mut enable_sona = true;
let mut enable_hnsw = true;
let mut repo_filter = None;
let mut verbose = false;
let mut i = 0;
while i < args.len() {
match args[i].as_str() {
"--model" => {
i += 1;
model_path = Some(PathBuf::from(args.get(i).ok_or("--model requires a path")?));
}
"--tasks" => {
i += 1;
let source = args.get(i).ok_or("--tasks requires a value")?;
task_source = match source.as_str() {
"sample" => TaskSource::Sample,
"swe-bench-lite" => TaskSource::SweBenchLite,
"swe-bench" => TaskSource::SweBenchFull,
path => TaskSource::File(PathBuf::from(path)),
};
}
"--max-tasks" => {
i += 1;
let n: usize = args
.get(i)
.ok_or("--max-tasks requires a number")?
.parse()
.map_err(|_| "Invalid number for --max-tasks")?;
max_tasks = Some(n);
}
"--modes" => {
i += 1;
let modes_str = args.get(i).ok_or("--modes requires a value")?;
ablation_modes = parse_modes(modes_str)?;
}
"--seeds" => {
i += 1;
let seeds_str = args.get(i).ok_or("--seeds requires a value")?;
seeds = seeds_str
.split(',')
.map(|s| s.trim().parse().map_err(|_| "Invalid seed"))
.collect::<Result<Vec<_>, _>>()?;
}
"--output" => {
i += 1;
output_path = Some(PathBuf::from(
args.get(i).ok_or("--output requires a path")?,
));
}
"--quality-threshold" => {
i += 1;
quality_threshold = args
.get(i)
.ok_or("--quality-threshold requires a value")?
.parse()
.map_err(|_| "Invalid quality threshold")?;
}
"--cost-target" => {
i += 1;
cost_target = args
.get(i)
.ok_or("--cost-target requires a value")?
.parse()
.map_err(|_| "Invalid cost target")?;
}
"--repo" => {
i += 1;
repo_filter = Some(args.get(i).ok_or("--repo requires a value")?.clone());
}
"--no-sona" => enable_sona = false,
"--no-hnsw" => enable_hnsw = false,
"--verbose" => verbose = true,
arg => {
if arg.starts_with('-') {
return Err(format!("Unknown option: {}", arg));
}
}
}
i += 1;
}
let model_path = model_path.ok_or("--model is required")?;
if ablation_modes.is_empty() {
ablation_modes = vec![
AblationMode::Baseline,
AblationMode::RetrievalOnly,
AblationMode::AdaptersOnly,
AblationMode::RetrievalPlusAdapters,
AblationMode::Full,
];
}
Ok(CliConfig {
model_path,
task_source,
max_tasks,
ablation_modes,
seeds,
output_path,
quality_threshold,
cost_target,
enable_sona,
enable_hnsw,
repo_filter,
verbose,
})
}
fn parse_modes(modes_str: &str) -> Result<Vec<AblationMode>, String> {
modes_str
.split(',')
.map(|s| match s.trim().to_lowercase().as_str() {
"baseline" => Ok(AblationMode::Baseline),
"retrieval" | "retrieval-only" | "retrieval_only" => Ok(AblationMode::RetrievalOnly),
"adapters" | "adapters-only" | "adapters_only" => Ok(AblationMode::AdaptersOnly),
"retrieval+adapters" | "retrieval_plus_adapters" => {
Ok(AblationMode::RetrievalPlusAdapters)
}
"full" => Ok(AblationMode::Full),
other => Err(format!("Unknown ablation mode: {}", other)),
})
.collect()
}
fn run_evaluation(config: CliConfig) -> Result<(), Box<dyn std::error::Error>> {
println!("RuvLLM Evaluation");
println!("=================\n");
if !config.model_path.exists() {
return Err(format!("Model not found: {}", config.model_path.display()).into());
}
println!("Model: {}", config.model_path.display());
println!("\nLoading tasks...");
let tasks = load_tasks(&config)?;
println!("Loaded {} tasks", tasks.len());
if config.verbose {
for task in tasks.iter().take(5) {
println!(" - {} ({})", task.id, task.repo);
}
if tasks.len() > 5 {
println!(" ... and {} more", tasks.len() - 5);
}
}
let eval_config = EvalConfig {
task_count: config.max_tasks.unwrap_or(tasks.len()),
seeds: config.seeds.clone(),
ablation_modes: config.ablation_modes.clone(),
quality_threshold: config.quality_threshold,
cost_target: config.cost_target,
..Default::default()
};
println!("\nConfiguration:");
println!(" Tasks: {}", eval_config.task_count);
println!(" Seeds: {:?}", eval_config.seeds);
println!(
" Modes: {:?}",
eval_config
.ablation_modes
.iter()
.map(|m| m.name())
.collect::<Vec<_>>()
);
println!(
" Quality threshold: {:.0}%",
eval_config.quality_threshold * 100.0
);
println!(
" SONA: {}",
if config.enable_sona {
"enabled"
} else {
"disabled"
}
);
println!(
" HNSW: {}",
if config.enable_hnsw {
"enabled"
} else {
"disabled"
}
);
let inference_config = RealInferenceConfig {
model_path: config.model_path.to_string_lossy().to_string(),
model_config: ModelConfig::default(),
enable_sona: config.enable_sona,
enable_hnsw: config.enable_hnsw,
..Default::default()
};
println!("\nInitializing evaluation harness...");
let mut harness = RealEvaluationHarness::with_config(eval_config, inference_config)?;
if !harness.is_model_loaded() {
return Err("Failed to load model".into());
}
println!("Model loaded successfully!");
println!("\nRunning evaluation...");
println!("This may take a while depending on model size and task count.\n");
let runtime = tokio::runtime::Runtime::new()?;
let report = runtime.block_on(harness.run_evaluation(&tasks))?;
println!("\n{}", "=".repeat(60));
println!("EVALUATION COMPLETE");
println!("{}\n", "=".repeat(60));
println!("{}", report.summary());
println!();
println!("Leaderboard:");
println!("{:-<60}", "");
println!(
"{:<5} {:<20} {:>10} {:>10} {:>10}",
"Rank", "Mode", "Success%", "Quality", "$/patch"
);
println!("{:-<60}", "");
for entry in report.to_leaderboard_entries() {
println!(
"{:<5} {:<20} {:>9.1}% {:>10.2} {:>10.4}",
entry.rank,
entry.mode.name(),
entry.success_rate * 100.0,
entry.quality_score,
entry.cost_per_patch
);
}
println!();
println!("Ablation Analysis vs Baseline:");
for comparison in report.compare_all_to_baseline() {
let direction = if comparison.success_delta > 0.0 {
"+"
} else {
""
};
let sig = if comparison.is_significant { "*" } else { "" };
println!(
" {}: {}{:.1}%{} success rate",
comparison.target.name(),
direction,
comparison.success_delta * 100.0,
sig
);
}
if let Some(output_path) = config.output_path {
println!("\nSaving results to {}...", output_path.display());
let json = report.to_json()?;
std::fs::write(&output_path, json)?;
println!("Results saved!");
let md_path = output_path.with_extension("md");
std::fs::write(&md_path, report.to_markdown())?;
println!("Markdown report saved to {}", md_path.display());
}
Ok(())
}
fn load_tasks(config: &CliConfig) -> Result<Vec<EvalTask>, Box<dyn std::error::Error>> {
let swe_config = SweBenchConfig {
max_tasks: config.max_tasks,
repo_filter: config.repo_filter.clone(),
..Default::default()
};
let loader = SweBenchLoader::new(swe_config);
let tasks: Vec<EvalTask> = match &config.task_source {
TaskSource::Sample => {
println!("Using sample tasks (3 tasks)");
SweBenchLoader::sample_tasks()
.into_iter()
.map(|t| t.into())
.collect()
}
TaskSource::SweBenchLite => {
println!("Loading SWE-bench-lite dataset...");
println!("Note: Using sample tasks. Run with async for full dataset download.");
SweBenchLoader::sample_tasks()
.into_iter()
.map(|t| t.into())
.collect()
}
TaskSource::SweBenchFull => {
println!("Loading full SWE-bench dataset...");
println!("Note: Using sample tasks. Run with async for full dataset download.");
SweBenchLoader::sample_tasks()
.into_iter()
.map(|t| t.into())
.collect()
}
TaskSource::File(path) => {
println!("Loading tasks from {}...", path.display());
let swe_tasks = if path.extension().map_or(false, |e| e == "jsonl") {
loader.load_from_jsonl(path)?
} else {
loader.load_from_file(path)?
};
let stats = SweBenchLoader::stats(&swe_tasks);
if config.verbose {
println!("{}", stats);
}
swe_tasks.into_iter().map(|t| t.into()).collect()
}
};
let tasks = if let Some(max) = config.max_tasks {
tasks.into_iter().take(max).collect()
} else {
tasks
};
Ok(tasks)
}