use std::time::Instant;
use anyhow::Result;
use kodegraf_core::build;
use kodegraf_core::config::Config;
use kodegraf_core::graph::GraphStore;
pub fn run(json_output: bool) -> Result<()> {
let repo_root = kodegraf_core::config::find_repo_root()?;
let config_path = Config::config_path(&repo_root);
let db_path = Config::graph_db_path(&repo_root);
if !db_path.exists() {
anyhow::bail!("Graph not built. Run `kodegraf build` first.");
}
let config = Config::load(&config_path)?;
let store = GraphStore::open(&db_path)?;
let stats = store.get_stats()?;
if stats.total_nodes == 0 {
anyhow::bail!("Graph is empty. Run `kodegraf build` first.");
}
eprintln!("Running benchmarks on {} ({} nodes, {} edges)...\n",
repo_root.display(), stats.total_nodes, stats.total_edges);
let build_perf = benchmark_build_performance(&repo_root, &config, &store)?;
let search_quality = benchmark_search_quality(&store)?;
let token_efficiency = benchmark_token_efficiency(&repo_root, &store)?;
if json_output {
let report = serde_json::json!({
"build_performance": build_perf,
"search_quality": search_quality,
"token_efficiency": token_efficiency,
"graph_stats": {
"total_nodes": stats.total_nodes,
"total_edges": stats.total_edges,
"files": stats.files,
"functions": stats.functions,
"languages": stats.languages,
}
});
println!("{}", serde_json::to_string_pretty(&report)?);
} else {
print_markdown_report(&stats, &build_perf, &search_quality, &token_efficiency);
}
Ok(())
}
#[derive(serde::Serialize)]
struct BuildPerformance {
file_count: i64,
node_count: i64,
edge_count: i64,
db_size_mb: u64,
search_avg_ms: f64,
search_p99_ms: f64,
incremental_update_ms: f64,
}
fn benchmark_build_performance(
repo_root: &std::path::Path,
config: &Config,
store: &GraphStore,
) -> Result<BuildPerformance> {
let stats = store.get_stats()?;
let db_path = Config::graph_db_path(repo_root);
let db_size = std::fs::metadata(&db_path).map(|m| m.len()).unwrap_or(0);
let queries = [
"useState", "render", "export", "import", "function",
"class", "interface", "return", "async", "test",
];
let mut search_times = Vec::new();
for q in &queries {
let start = Instant::now();
let _ = store.search_nodes(q, 10)?;
search_times.push(start.elapsed().as_secs_f64() * 1000.0);
}
search_times.sort_by(|a, b| a.partial_cmp(b).unwrap());
let search_avg = search_times.iter().sum::<f64>() / search_times.len() as f64;
let search_p99 = search_times.last().copied().unwrap_or(0.0);
let start = Instant::now();
let _ = build::incremental_update(repo_root, config, store, "HEAD~1");
let incremental_ms = start.elapsed().as_secs_f64() * 1000.0;
Ok(BuildPerformance {
file_count: stats.files,
node_count: stats.total_nodes,
edge_count: stats.total_edges,
db_size_mb: db_size / 1_048_576,
search_avg_ms: (search_avg * 10.0).round() / 10.0,
search_p99_ms: (search_p99 * 10.0).round() / 10.0,
incremental_update_ms: (incremental_ms * 10.0).round() / 10.0,
})
}
#[derive(serde::Serialize)]
struct SearchQuality {
queries_tested: usize,
queries_found: usize,
mean_reciprocal_rank: f64,
results: Vec<SearchQueryResult>,
}
#[derive(serde::Serialize)]
struct SearchQueryResult {
query: String,
expected: String,
rank: Option<usize>,
reciprocal_rank: f64,
result_count: usize,
latency_ms: f64,
}
fn benchmark_search_quality(store: &GraphStore) -> Result<SearchQuality> {
let test_cases: Vec<(&str, &str)> = vec![
("useState", "useState"),
("render", "render"),
("handleClick", "handleClick"),
("test", "test"),
];
let sample_nodes = store.search_nodes("function", 5)?;
let mut all_cases: Vec<(String, String)> = test_cases
.iter()
.map(|(q, e)| (q.to_string(), e.to_string()))
.collect();
for node in sample_nodes.iter().take(4) {
all_cases.push((node.name.clone(), node.name.clone()));
}
let mut results = Vec::new();
let mut total_found = 0;
for (query, expected) in &all_cases {
let start = Instant::now();
let search_results = store.search_nodes(query, 20)?;
let latency = start.elapsed().as_secs_f64() * 1000.0;
let rank = search_results.iter().position(|r| {
r.name.contains(expected.as_str()) || r.qualified_name.contains(expected.as_str())
});
let rr = rank.map(|r| 1.0 / (r as f64 + 1.0)).unwrap_or(0.0);
if rank.is_some() { total_found += 1; }
results.push(SearchQueryResult {
query: query.clone(),
expected: expected.clone(),
rank: rank.map(|r| r + 1),
reciprocal_rank: (rr * 1000.0).round() / 1000.0,
result_count: search_results.len(),
latency_ms: (latency * 10.0).round() / 10.0,
});
}
let mrr = if results.is_empty() { 0.0 } else {
let sum_rr: f64 = results.iter().map(|r| r.reciprocal_rank).sum();
(sum_rr / results.len() as f64 * 1000.0).round() / 1000.0
};
Ok(SearchQuality {
queries_tested: results.len(),
queries_found: total_found,
mean_reciprocal_rank: mrr,
results,
})
}
#[derive(serde::Serialize)]
struct TokenEfficiency {
changed_files: usize,
naive_tokens: usize,
graph_tokens: usize,
ratio: f64,
reduction_percent: f64,
}
fn estimate_tokens(text: &str) -> usize {
text.len() / 4
}
fn benchmark_token_efficiency(
repo_root: &std::path::Path,
store: &GraphStore,
) -> Result<TokenEfficiency> {
let changed = build::get_changed_files(repo_root, "HEAD~1")?;
if changed.is_empty() {
return Ok(TokenEfficiency {
changed_files: 0, naive_tokens: 0, graph_tokens: 0, ratio: 1.0, reduction_percent: 0.0,
});
}
let mut total_naive = 0;
let mut total_graph = 0;
for file_path in changed.iter().take(20) {
let full_path = repo_root.join(file_path);
let raw_content = match std::fs::read_to_string(&full_path) {
Ok(c) => c,
Err(_) => continue,
};
let raw_tokens = estimate_tokens(&raw_content);
let nodes = store.search_nodes(
full_path.file_stem().and_then(|s| s.to_str()).unwrap_or(file_path), 10,
)?;
let edges = store.get_edges_for(file_path).unwrap_or_default();
let mut graph_context = String::new();
for n in &nodes {
graph_context.push_str(&format!("{} {} [{}:{}-{}]\n",
n.kind.as_str(), n.qualified_name, n.file_path,
n.line_start.unwrap_or(0), n.line_end.unwrap_or(0)));
if let Some(sig) = &n.signature { graph_context.push_str(&format!(" {}\n", sig)); }
}
for e in &edges {
graph_context.push_str(&format!("{} {} -> {}\n",
e.kind.as_str(), e.source_qualified, e.target_qualified));
}
total_naive += raw_tokens;
total_graph += estimate_tokens(&graph_context);
}
let ratio = if total_graph > 0 { total_naive as f64 / total_graph as f64 } else { 1.0 };
Ok(TokenEfficiency {
changed_files: changed.len(),
naive_tokens: total_naive,
graph_tokens: total_graph,
ratio: (ratio * 10.0).round() / 10.0,
reduction_percent: ((1.0 - 1.0 / ratio) * 1000.0).round() / 10.0,
})
}
fn print_markdown_report(
stats: &kodegraf_core::models::GraphStats,
build: &BuildPerformance,
search: &SearchQuality,
tokens: &TokenEfficiency,
) {
println!("# Kodegraf Benchmark Report\n");
println!("## Graph Statistics\n");
println!("| Metric | Value |");
println!("|--------|-------|");
println!("| Files | {} |", stats.files);
println!("| Functions | {} |", stats.functions);
println!("| Classes | {} |", stats.classes);
println!("| Tests | {} |", stats.tests);
println!("| Total Nodes | {} |", stats.total_nodes);
println!("| Total Edges | {} |", stats.total_edges);
println!("| DB Size | {}MB |", build.db_size_mb);
println!("| Languages | {} |", stats.languages.join(", "));
println!();
println!("## Search Quality\n");
println!("| Metric | Value |");
println!("|--------|-------|");
println!("| Queries tested | {} |", search.queries_tested);
println!("| Hit rate | {:.0}% |", search.queries_found as f64 / search.queries_tested as f64 * 100.0);
println!("| Mean Reciprocal Rank | {:.3} |", search.mean_reciprocal_rank);
println!();
println!("| Query | Rank | MRR | Results | Latency |");
println!("|-------|------|-----|---------|---------|");
for r in &search.results {
let rank_str = r.rank.map(|r| format!("#{}", r)).unwrap_or_else(|| "miss".to_string());
println!("| {} | {} | {:.2} | {} | {:.1}ms |",
r.query, rank_str, r.reciprocal_rank, r.result_count, r.latency_ms);
}
println!();
println!("## Token Efficiency\n");
if tokens.changed_files == 0 {
println!("No changed files detected. Commit changes and re-run.\n");
} else {
println!("| Metric | Value |");
println!("|--------|-------|");
println!("| Changed files | {} |", tokens.changed_files);
println!("| Naive tokens (Read full files) | {} |", tokens.naive_tokens);
println!("| Graph tokens (Kodegraf context) | {} |", tokens.graph_tokens);
println!("| **Reduction ratio** | **{:.1}x** |", tokens.ratio);
println!("| **Tokens saved** | **{:.1}%** |", tokens.reduction_percent);
println!();
}
println!("## Performance\n");
println!("| Metric | Value |");
println!("|--------|-------|");
println!("| Search avg latency | {:.1}ms |", build.search_avg_ms);
println!("| Search P99 latency | {:.1}ms |", build.search_p99_ms);
println!("| Incremental update | {:.0}ms |", build.incremental_update_ms);
println!();
}