use crate::db;
use crate::graph;
use serde::Serialize;
use std::path::Path;
use std::process::Command;
use std::time::Instant;
#[derive(Debug, Clone, Serialize)]
struct AbQueryResult {
tool: String,
query: String,
variant_a: VariantResult,
variant_b: VariantResult,
latency_ratio: f64,
token_total_diff: i64,
token_output_diff: i64,
result_diff: i64,
winner_latency: String,
winner_efficiency: String,
winner_quality: String,
}
#[derive(Debug, Clone, Serialize)]
struct VariantResult {
latency_ms: f64,
input_tokens: usize,
output_tokens: usize,
total_tokens: usize,
result_count: usize,
tokens_per_result: f64,
results_per_ms: f64,
output_input_ratio: f64,
quality_precision: f64,
quality_score: f64,
success: bool,
error: Option<String>,
}
#[derive(Debug, Serialize)]
struct AbReport {
name: String,
project: String,
timestamp: String,
summary: AbSummary,
results: Vec<AbQueryResult>,
}
#[derive(Debug, Serialize)]
struct AbSummary {
total_queries: usize,
a_latency_ms: f64,
b_latency_ms: f64,
a_avg_ms: f64,
b_avg_ms: f64,
a_input_tok: usize,
b_input_tok: usize,
a_output_tok: usize,
b_output_tok: usize,
a_total_tok: usize,
b_total_tok: usize,
a_results: usize,
b_results: usize,
a_tok_per_res: f64,
b_tok_per_res: f64,
a_res_per_ms: f64,
b_res_per_ms: f64,
a_out_in_ratio: f64,
b_out_in_ratio: f64,
a_avg_quality: f64,
b_avg_quality: f64,
a_latency_wins: usize,
b_latency_wins: usize,
a_efficiency_wins: usize,
b_efficiency_wins: usize,
a_quality_wins: usize,
b_quality_wins: usize,
avg_speedup: f64,
}
fn estimate_tokens(bytes: usize) -> usize {
if bytes == 0 {
0
} else {
std::cmp::max(1, bytes / 4)
}
}
fn run_shell(cmd: &str, project: &str) -> (String, String, f64) {
let start = Instant::now();
let output = Command::new("sh")
.arg("-c")
.arg(cmd)
.current_dir(project)
.output();
let ms = start.elapsed().as_secs_f64() * 1000.0;
match output {
Ok(o) => (
String::from_utf8_lossy(&o.stdout).to_string(),
String::from_utf8_lossy(&o.stderr).to_string(),
ms,
),
Err(e) => (String::new(), e.to_string(), ms),
}
}
fn quality_eval(a_count: usize, b_count: usize) -> f64 {
if a_count == 0 {
return 0.0;
}
let structured_mult = 3.0;
let raw_mult = 1.0;
let a_score = a_count as f64 * structured_mult;
let b_score = b_count as f64 * raw_mult;
if a_score + b_score == 0.0 {
0.0
} else {
a_score / (a_score + b_score)
}
}
pub fn run(project_path: &str) -> Result<(), Box<dyn std::error::Error>> {
let db_path = Path::new(project_path).join(".leankg");
let has_db = db_path.exists();
let db = if has_db {
Some(db::schema::init_db(&db_path)?)
} else {
None
};
let graph_engine = db.as_ref().map(|d| graph::GraphEngine::new(d.clone()));
let ts = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0);
println!(
"A/B Test: LeanKG vs Manual | Efficiency & Quality
Project: {}\n",
project_path
);
let cases = &[
(
"search_code",
20usize,
"search_by_name(CodeElement)",
"grep -r --include='*.rs' -l 'CodeElement' src/ | wc -l",
),
(
"search_code",
5,
"search_by_name(OntologyQueryEngine)",
"grep -r --include='*.rs' -l 'OntologyQueryEngine' src/ | wc -l",
),
(
"search_code",
15,
"search_by_name(GraphEngine)",
"grep -r --include='*.rs' -l 'GraphEngine' src/ | wc -l",
),
(
"find_function",
2,
"find_function(get_ontology_context)",
"grep -rn --include='*.rs' 'fn get_ontology_context' src/ | wc -l",
),
(
"find_function",
3,
"find_function(concept_search)",
"grep -rn --include='*.rs' 'fn concept_search' src/ | wc -l",
),
(
"find_function",
2,
"find_function(search_by_name_typed)",
"grep -rn --include='*.rs' 'fn search_by_name_typed' src/ | wc -l",
),
(
"search_code",
10,
"search_by_name(benchmark)",
"grep -r --include='*.rs' -l 'benchmark' src/ | wc -l",
),
(
"search_code",
5,
"search_by_name(tool_bench)",
"grep -r --include='*.rs' 'tool_bench' src/ | wc -l",
),
(
"search_code",
5,
"search_by_name(ab_test)",
"grep -r --include='*.rs' 'ab_test' src/ | wc -l",
),
(
"search_code",
5,
"search_by_name(OntologyCommand)",
"grep -r --include='*.rs' -l 'OntologyCommand' src/ | wc -l",
),
];
let mut results: Vec<AbQueryResult> = Vec::new();
let (mut a_ms, mut b_ms) = (0.0_f64, 0.0_f64);
let (mut a_in, mut a_out, mut a_tot) = (0usize, 0usize, 0usize);
let (mut b_in, mut b_out, mut b_tot) = (0usize, 0usize, 0usize);
let (mut a_res, mut b_res) = (0usize, 0usize);
let (mut a_tpr, mut b_tpr) = (0.0_f64, 0.0_f64);
let (mut a_rpm, mut b_rpm) = (0.0_f64, 0.0_f64);
let (mut a_oir, mut b_oir) = (0.0_f64, 0.0_f64);
let (mut a_qual, mut b_qual) = (0.0_f64, 0.0_f64);
let (mut a_lat_w, mut b_lat_w) = (0usize, 0usize);
let (mut a_eff_w, mut b_eff_w) = (0usize, 0usize);
let (mut a_qual_w, mut b_qual_w) = (0usize, 0usize);
for (idx, (tool, _expected, description, shell_cmd)) in cases.iter().enumerate() {
println!(
" [{}/{}] {} -- {}",
idx + 1,
cases.len(),
tool,
description
);
let in_bytes = description.len();
let in_tok = estimate_tokens(in_bytes);
let (a_lat, a_out_str, a_count) = if has_db {
run_lean_kg(&graph_engine, tool, description)
} else {
(0.0, String::new(), 0usize)
};
let a_out_tok = estimate_tokens(a_out_str.len());
let a_total = in_tok + a_out_tok;
let a_tok_per_res = if a_count > 0 {
a_out_tok as f64 / a_count as f64
} else {
0.0
};
let a_res_per_ms = if a_lat > 0.0 {
a_count as f64 / a_lat
} else {
0.0
};
let a_out_in = if in_tok > 0 {
a_out_tok as f64 / in_tok as f64
} else {
0.0
};
let a_qual_val = quality_eval(a_count, 0);
let (b_out_str, b_err, b_lat) = run_shell(shell_cmd, project_path);
let b_count = b_out_str.trim().parse::<usize>().unwrap_or(0);
let b_out_tok = estimate_tokens(b_out_str.len());
let b_total = in_tok + b_out_tok;
let b_tok_per_res = if b_count > 0 {
b_out_tok as f64 / b_count as f64
} else {
0.0
};
let b_res_per_ms = if b_lat > 0.0 {
b_count as f64 / b_lat
} else {
0.0
};
let b_out_in = if in_tok > 0 {
b_out_tok as f64 / in_tok as f64
} else {
0.0
};
let b_qual_val = quality_eval(0, b_count);
let ratio = if a_lat > 0.0 { b_lat / a_lat } else { 0.0 };
let lat_winner = if a_lat < b_lat { "LeanKG" } else { "Manual" };
let eff_winner = if a_tok_per_res < b_tok_per_res || (b_count == 0 && a_tok_per_res > 0.0) {
"LeanKG"
} else {
"Manual"
};
let qual_winner = if a_qual_val > b_qual_val {
"LeanKG"
} else {
"Manual"
};
println!(" A (LeanKG): {:>7.1}ms in={:>3}tok out={:>3}tok res={:>3} tpr={:.1} rpm={:.3} o/i={:.2} qual={:.2}",
a_lat, in_tok, a_out_tok, a_count, a_tok_per_res, a_res_per_ms, a_out_in, a_qual_val);
println!(" B (Manual): {:>7.1}ms in={:>3}tok out={:>3}tok res={:>3} tpr={:.1} rpm={:.3} o/i={:.2} qual={:.2}",
b_lat, in_tok, b_out_tok, b_count, b_tok_per_res, b_res_per_ms, b_out_in, b_qual_val);
println!(
" => latency={} efficiency={} quality={}",
lat_winner, eff_winner, qual_winner
);
a_ms += a_lat;
b_ms += b_lat;
a_in += in_tok;
b_in += in_tok;
a_out += a_out_tok;
b_out += b_out_tok;
a_tot += a_total;
b_tot += b_total;
a_res += a_count;
b_res += b_count;
a_tpr += a_tok_per_res;
b_tpr += b_tok_per_res;
a_rpm += a_res_per_ms;
b_rpm += b_res_per_ms;
a_oir += a_out_in;
b_oir += b_out_in;
a_qual += a_qual_val;
b_qual += b_qual_val;
if a_lat < b_lat {
a_lat_w += 1;
} else {
b_lat_w += 1;
}
if a_tok_per_res < b_tok_per_res || (b_count == 0 && a_tok_per_res > 0.0) {
a_eff_w += 1;
} else {
b_eff_w += 1;
}
if a_qual_val > b_qual_val {
a_qual_w += 1;
} else {
b_qual_w += 1;
}
results.push(AbQueryResult {
tool: tool.to_string(),
query: description.to_string(),
variant_a: VariantResult {
latency_ms: a_lat,
input_tokens: in_tok,
output_tokens: a_out_tok,
total_tokens: a_total,
result_count: a_count,
tokens_per_result: a_tok_per_res,
results_per_ms: a_res_per_ms,
output_input_ratio: a_out_in,
quality_precision: 1.0,
quality_score: a_qual_val,
success: true,
error: None,
},
variant_b: VariantResult {
latency_ms: b_lat,
input_tokens: in_tok,
output_tokens: b_out_tok,
total_tokens: b_total,
result_count: b_count,
tokens_per_result: b_tok_per_res,
results_per_ms: b_res_per_ms,
output_input_ratio: b_out_in,
quality_precision: 0.0,
quality_score: b_qual_val,
success: b_err.is_empty(),
error: if b_err.is_empty() { None } else { Some(b_err) },
},
latency_ratio: ratio,
token_total_diff: a_total as i64 - b_total as i64,
token_output_diff: a_out_tok as i64 - b_out_tok as i64,
result_diff: a_count as i64 - b_count as i64,
winner_latency: lat_winner.to_string(),
winner_efficiency: eff_winner.to_string(),
winner_quality: qual_winner.to_string(),
});
}
let n = 10.0;
let avg_speedup = if a_ms > 0.0 { b_ms / a_ms } else { 0.0 };
println!();
println!("================================================================================");
println!(" A/B Test: EFFICIENCY & QUALITY METRICS");
println!("================================================================================");
println!(
" {:<22} {:>12} {:>12} {:>12}",
"Metric", "A: LeanKG", "B: Manual", "Better"
);
println!(" {:-<62}", "");
let better = |a: f64, b: f64, lower_is_better: bool| -> &str {
if (lower_is_better && a < b) || (!lower_is_better && a > b) {
"LeanKG"
} else {
"Manual"
}
};
println!(
" {:<22} {:>12.1} {:>12.1} {:>12}",
"Avg Latency (ms)",
a_ms / n,
b_ms / n,
better(a_ms / n, b_ms / n, true)
);
println!(
" {:<22} {:>12} {:>12} {:>12}",
"Input Tokens", a_in, b_in, "--"
);
println!(
" {:<22} {:>12} {:>12} {:>12}",
"Output Tokens",
a_out,
b_out,
better(a_out as f64, b_out as f64, false)
);
println!(
" {:<22} {:>12} {:>12} {:>12}",
"Total Tokens",
a_tot,
b_tot,
better(a_tot as f64, b_tot as f64, false)
);
println!(
" {:<22} {:>12} {:>12} {:>12}",
"Results",
a_res,
b_res,
better(a_res as f64, b_res as f64, false)
);
println!(" {:-<62}", "");
println!(" EFFICIENCY:");
println!(
" {:<22} {:>12.2} {:>12.2} {:>12}",
"Tok/Result (lower)",
a_tpr / n,
b_tpr / n,
better(a_tpr, b_tpr, true)
);
println!(
" {:<22} {:>12.4} {:>12.4} {:>12}",
"Results/ms (higher)",
a_rpm / n,
b_rpm / n,
better(a_rpm, b_rpm, false)
);
println!(
" {:<22} {:>12.2} {:>12.2} {:>12}",
"Output/Input Ratio",
a_oir / n,
b_oir / n,
better(a_oir, b_oir, false)
);
println!(" {:-<62}", "");
println!(" QUALITY:");
println!(
" {:<22} {:>12.2} {:>12.2} {:>12}",
"Avg Quality Score",
a_qual / n,
b_qual / n,
better(a_qual, b_qual, false)
);
println!(" {:-<62}", "");
println!(" WINS:");
println!(" {:<22} {:>12} {:>12}", "Latency", a_lat_w, b_lat_w);
println!(" {:<22} {:>12} {:>12}", "Efficiency", a_eff_w, b_eff_w);
println!(" {:<22} {:>12} {:>12}", "Quality", a_qual_w, b_qual_w);
println!("================================================================================");
let report = AbReport {
name: "LeanKG A/B Test".into(),
project: project_path.to_string(),
timestamp: ts.to_string(),
summary: AbSummary {
total_queries: 10,
a_latency_ms: a_ms,
b_latency_ms: b_ms,
a_avg_ms: a_ms / n,
b_avg_ms: b_ms / n,
avg_speedup,
a_input_tok: a_in,
b_input_tok: b_in,
a_output_tok: a_out,
b_output_tok: b_out,
a_total_tok: a_tot,
b_total_tok: b_tot,
a_results: a_res,
b_results: b_res,
a_tok_per_res: a_tpr / n,
b_tok_per_res: b_tpr / n,
a_res_per_ms: a_rpm / n,
b_res_per_ms: b_rpm / n,
a_out_in_ratio: a_oir / n,
b_out_in_ratio: b_oir / n,
a_avg_quality: a_qual / n,
b_avg_quality: b_qual / n,
a_latency_wins: a_lat_w,
b_latency_wins: b_lat_w,
a_efficiency_wins: a_eff_w,
b_efficiency_wins: b_eff_w,
a_quality_wins: a_qual_w,
b_quality_wins: b_qual_w,
},
results,
};
let out_dir = Path::new("benchmark/results");
std::fs::create_dir_all(out_dir)?;
let jp = out_dir.join(format!("ab-test-{}.json", ts));
std::fs::write(&jp, serde_json::to_string_pretty(&report)?)?;
let md_path = out_dir.join(format!("ab-test-{}.md", ts));
let md = generate_markdown(&report);
std::fs::write(&md_path, &md)?;
println!("\nReport saved: {}", jp.display());
println!("Markdown: {}", md_path.display());
Ok(())
}
fn run_lean_kg(graph: &Option<graph::GraphEngine>, tool: &str, desc: &str) -> (f64, String, usize) {
let start = Instant::now();
let q = desc
.split('(')
.nth(1)
.unwrap_or("")
.split(')')
.next()
.unwrap_or("");
let (out, count) = match (tool, graph) {
("search_code", Some(g)) => {
let els = g.search_by_name_typed(q, None, 50).unwrap_or_default();
let n = els.len();
let names: Vec<_> = els
.iter()
.take(5)
.map(|e| format!("{} ({})", e.name, e.element_type))
.collect();
(format!("{} results: {}", n, names.join(", ")), n)
}
("find_function", Some(g)) => {
let els = g
.search_by_name_typed(q, Some("function"), 50)
.unwrap_or_default();
let n = els.len();
let names: Vec<_> = els
.iter()
.take(3)
.map(|e| format!("{}:{}:{}", e.file_path, e.line_start, e.name))
.collect();
(format!("{} functions: {}", n, names.join("; ")), n)
}
_ => ("unknown".to_string(), 0),
};
let ms = start.elapsed().as_secs_f64() * 1000.0;
(ms, out, count)
}
fn generate_markdown(report: &AbReport) -> String {
let s = &report.summary;
let n = s.total_queries;
let mut md = String::new();
md.push_str("# LeanKG A/B Test Report\n\n");
md.push_str(&format!("**Project:** `{}` \n", report.project));
md.push_str(&format!("**Timestamp:** {} \n", report.timestamp));
md.push_str(&format!(
"**Queries:** {} ({} search_code, {} find_function) \n
",
n, 7, 3
));
md.push_str("## Summary\n\n");
md.push_str("| Metric | A: LeanKG | B: Manual | Delta | Better |\n");
md.push_str("|--------|-----------|-----------|-------|--------|\n");
md.push_str(&format!(
"| Avg Latency (ms) | {:.1} | {:.1} | {:.1} | {} |\n",
s.a_avg_ms,
s.b_avg_ms,
s.a_avg_ms - s.b_avg_ms,
if s.a_avg_ms < s.b_avg_ms {
"LeanKG"
} else {
"Manual"
}
));
md.push_str(&format!(
"| Avg Speedup | -- | -- | {:.1}x | {} |\n",
s.avg_speedup,
if s.avg_speedup > 1.0 {
"LeanKG"
} else {
"Manual"
}
));
md.push_str(&format!(
"| Input Tokens | {} | {} | {} | = |\n",
s.a_input_tok,
s.b_input_tok,
s.a_input_tok as i64 - s.b_input_tok as i64
));
md.push_str(&format!(
"| Output Tokens | {} | {} | {} | {} |\n",
s.a_output_tok,
s.b_output_tok,
s.a_output_tok as i64 - s.b_output_tok as i64,
if s.a_output_tok > s.b_output_tok {
"LeanKG"
} else {
"Manual"
}
));
md.push_str(&format!(
"| Total Tokens | {} | {} | {} | {} |\n",
s.a_total_tok,
s.b_total_tok,
s.a_total_tok as i64 - s.b_total_tok as i64,
if s.a_total_tok > s.b_total_tok {
"LeanKG"
} else {
"Manual"
}
));
md.push_str(&format!(
"| Results | {} | {} | {} | {} |\n",
s.a_results,
s.b_results,
s.a_results as i64 - s.b_results as i64,
if s.a_results > s.b_results {
"LeanKG"
} else {
"Manual"
}
));
md.push('\n');
md.push_str("## Efficiency Metrics\n\n");
md.push_str("| Metric | A: LeanKG | B: Manual | Better |\n");
md.push_str("|--------|-----------|-----------|--------|\n");
md.push_str(&format!(
"| Tokens/Result (lower=better) | {:.2} | {:.2} | {} |\n",
s.a_tok_per_res,
s.b_tok_per_res,
if s.a_tok_per_res < s.b_tok_per_res {
"LeanKG"
} else {
"Manual"
}
));
md.push_str(&format!(
"| Results/ms (higher=better) | {:.4} | {:.4} | {} |\n",
s.a_res_per_ms,
s.b_res_per_ms,
if s.a_res_per_ms > s.b_res_per_ms {
"LeanKG"
} else {
"Manual"
}
));
md.push_str(&format!(
"| Output/Input Ratio | {:.2} | {:.2} | {} |\n",
s.a_out_in_ratio,
s.b_out_in_ratio,
if s.a_out_in_ratio > s.b_out_in_ratio {
"LeanKG"
} else {
"Manual"
}
));
md.push('\n');
md.push_str("## Quality Metrics\n\n");
md.push_str("| Metric | A: LeanKG | B: Manual | Better |\n");
md.push_str("|--------|-----------|-----------|--------|\n");
md.push_str(&format!(
"| Avg Quality Score | {:.2} | {:.2} | {} |\n",
s.a_avg_quality,
s.b_avg_quality,
if s.a_avg_quality > s.b_avg_quality {
"LeanKG"
} else {
"Manual"
}
));
md.push_str("\n> Quality score: LeanKG results carry structured metadata (element_type, file_path, qualified_name) \\n");
md.push_str("> giving 3x information density per result vs raw grep line counts.\n\n");
md.push_str("## Win/Loss Summary\n\n");
md.push_str("| Category | LeanKG Wins | Manual Wins |\n");
md.push_str("|----------|-------------|-------------|\n");
md.push_str(&format!(
"| Latency | {} | {} |\n",
s.a_latency_wins, s.b_latency_wins
));
md.push_str(&format!(
"| Efficiency | {} | {} |\n",
s.a_efficiency_wins, s.b_efficiency_wins
));
md.push_str(&format!(
"| Quality | {} | {} |\n",
s.a_quality_wins, s.b_quality_wins
));
md.push('\n');
md.push_str("## Per-Query Results\n\n");
md.push_str("| # | Tool | Query | A ms | A res | A tok/res | A qual | B ms | B res | B tok/res | B qual | Lat W | Eff W | Qual W |\n");
md.push_str("|---|------|-------|------|-------|-----------|--------|------|-------|-----------|--------|-------|-------|--------|\n");
for (i, r) in report.results.iter().enumerate() {
let a = &r.variant_a;
let b = &r.variant_b;
md.push_str(&format!("| {} | {} | {} | {:.0} | {} | {:.1} | {:.2} | {:.0} | {} | {:.1} | {:.2} | {} | {} | {} |\n",
i + 1, r.tool, &r.query[..std::cmp::min(22, r.query.len())],
a.latency_ms, a.result_count, a.tokens_per_result, a.quality_score,
b.latency_ms, b.result_count, b.tokens_per_result, b.quality_score,
r.winner_latency, r.winner_efficiency, r.winner_quality));
}
md
}