use super::competitors::CompetitorProfile;
use super::metrics::{self, ComparativeMetrics};
use super::system_info::SystemInfo;
#[derive(Debug)]
pub struct CompareReport {
pub metrics: ComparativeMetrics,
pub system: SystemInfo,
pub competitors: Vec<CompetitorProfile>,
}
pub fn generate_markdown(report: &CompareReport) -> String {
let mut out = Vec::new();
write_header(&mut out, report);
write_methodology(&mut out);
write_compression_table(&mut out, report);
write_mode_detail(&mut out, report);
write_search_latency(&mut out, report);
write_cold_start(&mut out, report);
write_disk_footprint(&mut out, report);
write_feature_matrix(&mut out, report);
write_session_simulation(&mut out, report);
write_chart(&mut out, report);
write_system_info(&mut out, &report.system);
write_reproducibility(&mut out);
write_footer(&mut out);
out.join("\n")
}
pub fn generate_terminal(report: &CompareReport) -> String {
let mut out = Vec::new();
let sep = "\u{2550}".repeat(72);
out.push(sep.clone());
out.push(" lean-ctx Head-to-Head Benchmark".to_string());
out.push(sep.clone());
out.push(String::new());
out.push(" Compression Comparison:".to_string());
out.push(format!(
" {:<28} {:>12} {:>12} {:>10}",
"Tool", "Compression", "Search", "Features"
));
out.push(format!(" {}", "\u{2500}".repeat(66)));
for c in &report.competitors {
let comp = c
.compression_pct
.map_or("N/A".to_string(), |p| format!("{p:.0}%"));
let search = if c.supports_search { "Yes" } else { "No" };
out.push(format!(
" {:<28} {:>12} {:>12} {:>10}",
c.name, comp, search, c.feature_count
));
}
for mc in &report.metrics.mode_comparisons {
if mc.mode == "full" {
continue;
}
out.push(format!(
" {:<28} {:>11.1}% {:>12} {:>10}",
format!("lean-ctx ({})", mc.mode),
mc.avg_compression_pct,
"Yes",
report.metrics.feature_count
));
}
out.push(String::new());
let avg_search = metrics::avg_search_latency_us(&report.metrics.search_latencies);
out.push(format!(
" Avg search latency: {}",
metrics::format_duration_us(avg_search)
));
out.push(format!(
" Cold start: {}",
metrics::format_duration_us(report.metrics.cold_start.total_us)
));
out.push(format!(
" Disk footprint: {}",
metrics::format_bytes(report.metrics.disk_footprint.total_index_bytes)
));
out.push(sep);
out.join("\n")
}
fn write_header(out: &mut Vec<String>, report: &CompareReport) {
out.push("# lean-ctx Benchmark: Head-to-Head Comparison".to_string());
out.push(String::new());
out.push(format!(
"> Generated by lean-ctx v{} on {}",
report.system.lean_ctx_version,
chrono::Local::now().format("%Y-%m-%d %H:%M:%S")
));
out.push(String::new());
out.push(format!(
"**Project:** `{}` ",
report.metrics.project_benchmark.root
));
out.push(format!(
"**Files measured:** {} ",
report.metrics.project_benchmark.files_measured
));
out.push(format!(
"**Total raw tokens:** {} ",
format_num(report.metrics.project_benchmark.total_raw_tokens)
));
out.push(String::new());
}
fn write_methodology(out: &mut Vec<String>) {
out.push("## Methodology".to_string());
out.push(String::new());
out.push(
"All lean-ctx measurements are **real values** measured on the test repository. \
Competitor numbers use **published figures** from their official documentation, \
papers, or README files. Sources are cited in the comparison table."
.to_string(),
);
out.push(String::new());
out.push("- **Token counting**: tiktoken `o200k_base` (GPT-4o tokenizer)".to_string());
out.push("- **Compression**: Each lean-ctx read mode is applied to the same files".to_string());
out.push("- **Latency**: Wall-clock time, median of all measured files".to_string());
out.push("- **Quality**: Preservation score (structural + semantic fidelity)".to_string());
out.push(String::new());
}
fn write_compression_table(out: &mut Vec<String>, report: &CompareReport) {
out.push("## Compression Comparison".to_string());
out.push(String::new());
out.push("| Tool | Compression | Tokens | Source |".to_string());
out.push("|------|------------:|-------:|--------|".to_string());
let raw_tokens = report.metrics.project_benchmark.total_raw_tokens;
for c in &report.competitors {
let comp = c
.compression_pct
.map_or("N/A".to_string(), |p| format!("{p:.0}%"));
let tokens = c.compression_pct.map_or("—".to_string(), |p| {
format_num((raw_tokens as f64 * (1.0 - p / 100.0)) as usize)
});
out.push(format!(
"| {} | {} | {} | {} |",
c.name, comp, tokens, c.source
));
}
for mc in &report.metrics.mode_comparisons {
if mc.mode == "full" {
continue;
}
out.push(format!(
"| **lean-ctx ({})** | **{:.1}%** | **{}** | Measured |",
mc.mode,
mc.avg_compression_pct,
format_num(mc.total_compressed_tokens),
));
}
out.push(String::new());
}
fn write_mode_detail(out: &mut Vec<String>, report: &CompareReport) {
out.push("## lean-ctx Mode Performance".to_string());
out.push(String::new());
out.push("| Mode | Compression | Latency | Quality | Use Case |".to_string());
out.push("|------|------------:|--------:|--------:|----------|".to_string());
for mc in &report.metrics.mode_comparisons {
let use_case = match mc.mode.as_str() {
"full" => "Editing files (cached, ~13 tok on re-read)",
"map" => "Understanding structure, deps, exports",
"signatures" => "API surface only",
"aggressive" => "Maximum compression for large files",
"entropy" => "Information-theoretic filtering",
_ => "",
};
let quality = if mc.avg_quality > 0.0 {
format!("{:.0}%", mc.avg_quality * 100.0)
} else {
"100%".to_string()
};
out.push(format!(
"| {} | {:.1}% | {} | {} | {} |",
mc.mode,
mc.avg_compression_pct,
metrics::format_duration_us(mc.avg_latency_us),
quality,
use_case,
));
}
out.push(String::new());
}
fn write_search_latency(out: &mut Vec<String>, report: &CompareReport) {
out.push("## Search Latency".to_string());
out.push(String::new());
out.push("| Query | BM25 Latency | Results |".to_string());
out.push("|-------|-------------:|--------:|".to_string());
for sl in &report.metrics.search_latencies {
out.push(format!(
"| `{}` | {} | {} |",
sl.query,
metrics::format_duration_us(sl.bm25_us),
sl.result_count,
));
}
let avg = metrics::avg_search_latency_us(&report.metrics.search_latencies);
out.push(format!(
"| **Average** | **{}** | — |",
metrics::format_duration_us(avg)
));
out.push(String::new());
}
fn write_cold_start(out: &mut Vec<String>, report: &CompareReport) {
let cs = &report.metrics.cold_start;
out.push("## Cold Start Performance".to_string());
out.push(String::new());
out.push("| Phase | Duration |".to_string());
out.push("|-------|--------:|".to_string());
out.push(format!(
"| File scan | {} |",
metrics::format_duration_us(cs.scan_us)
));
out.push(format!(
"| BM25 index build | {} |",
metrics::format_duration_us(cs.bm25_build_us)
));
out.push(format!(
"| First file read + tokenize | {} |",
metrics::format_duration_us(cs.first_read_us)
));
out.push(format!(
"| **Total cold start** | **{}** |",
metrics::format_duration_us(cs.total_us)
));
out.push(String::new());
}
fn write_disk_footprint(out: &mut Vec<String>, report: &CompareReport) {
let df = &report.metrics.disk_footprint;
out.push("## Disk Footprint".to_string());
out.push(String::new());
out.push("| Component | Size |".to_string());
out.push("|-----------|-----:|".to_string());
out.push(format!(
"| BM25 index | {} |",
metrics::format_bytes(df.bm25_index_bytes)
));
out.push(format!(
"| Total `.lean-ctx/` | {} |",
metrics::format_bytes(df.total_index_bytes)
));
out.push(String::new());
}
fn write_feature_matrix(out: &mut Vec<String>, report: &CompareReport) {
out.push("## Feature Comparison".to_string());
out.push(String::new());
out.push("| Feature | Raw | Repomix | aider | codebase-memory | **lean-ctx** |".to_string());
out.push("|---------|:---:|:-------:|:-----:|:---------------:|:------------:|".to_string());
let features = [
("Multi-mode compression", [false, false, false, false, true]),
("BM25 code search", [false, false, false, true, true]),
("Session caching", [false, false, true, true, true]),
(
"Cross-session memory (CCP)",
[false, false, false, true, true],
),
(
"Shell output compression",
[false, false, false, false, true],
),
("Call graph analysis", [false, false, false, false, true]),
("Repo map generation", [false, true, true, false, true]),
("Knowledge base", [false, false, false, true, true]),
(
"Tree-sitter AST (18 langs)",
[false, true, true, false, true],
),
("MCP server", [false, false, false, true, true]),
];
for (feature, support) in &features {
let cells: Vec<&str> = support
.iter()
.map(|s| if *s { "✅" } else { "—" })
.collect();
out.push(format!(
"| {} | {} | {} | {} | {} | {} |",
feature, cells[0], cells[1], cells[2], cells[3], cells[4]
));
}
out.push(String::new());
out.push(format!(
"**lean-ctx feature count:** {} operations across {} MCP tools",
report.metrics.feature_count,
crate::server::registry::tool_count()
));
out.push(String::new());
}
fn write_session_simulation(out: &mut Vec<String>, report: &CompareReport) {
let s = &report.metrics.project_benchmark.session_sim;
out.push("## Session Simulation (30-min coding)".to_string());
out.push(String::new());
out.push("| Approach | Tokens | Cost | Savings |".to_string());
out.push("|----------|-------:|-----:|--------:|".to_string());
out.push(format!(
"| Raw (no compression) | {} | ${:.3} | — |",
format_num(s.raw_tokens),
s.raw_cost
));
let lean_pct = if s.raw_tokens > 0 {
(1.0 - s.lean_tokens as f64 / s.raw_tokens as f64) * 100.0
} else {
0.0
};
out.push(format!(
"| lean-ctx (no CCP) | {} | ${:.3} | {:.1}% |",
format_num(s.lean_tokens),
s.lean_cost,
lean_pct
));
let ccp_pct = if s.raw_tokens > 0 {
(1.0 - s.lean_ccp_tokens as f64 / s.raw_tokens as f64) * 100.0
} else {
0.0
};
out.push(format!(
"| **lean-ctx + CCP** | **{}** | **${:.3}** | **{:.1}%** |",
format_num(s.lean_ccp_tokens),
s.ccp_cost,
ccp_pct
));
out.push(String::new());
}
fn write_chart(out: &mut Vec<String>, report: &CompareReport) {
out.push("## Compression Visualization".to_string());
out.push(String::new());
out.push("```".to_string());
out.push("Compression % (higher = better)".to_string());
out.push(String::new());
let mut entries: Vec<(String, f64)> = Vec::new();
for c in &report.competitors {
if let Some(pct) = c.compression_pct {
entries.push((c.name.to_string(), pct));
}
}
for mc in &report.metrics.mode_comparisons {
if mc.mode != "full" {
entries.push((format!("lean-ctx ({})", mc.mode), mc.avg_compression_pct));
}
}
entries.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal));
let max_bar = 50;
for (name, pct) in &entries {
let bar_len = (pct / 100.0 * max_bar as f64) as usize;
let bar = "\u{2588}".repeat(bar_len);
out.push(format!("{name:<28} {bar} {pct:.1}%"));
}
out.push("```".to_string());
out.push(String::new());
}
fn write_system_info(out: &mut Vec<String>, sys: &SystemInfo) {
out.push("## System Information".to_string());
out.push(String::new());
out.push(format!("- **OS:** {} {}", sys.os, sys.arch));
out.push(format!(
"- **CPU:** {} ({} cores)",
sys.cpu_brand, sys.cpu_cores
));
out.push(format!("- **RAM:** {:.1} GB", sys.memory_gb));
out.push(format!("- **lean-ctx:** v{}", sys.lean_ctx_version));
out.push(format!("- **Rust:** {}", sys.rust_version));
out.push(String::new());
}
fn write_reproducibility(out: &mut Vec<String>) {
out.push("## Reproducibility".to_string());
out.push(String::new());
out.push("```bash".to_string());
out.push("# Install lean-ctx".to_string());
out.push("cargo install lean-ctx".to_string());
out.push(String::new());
out.push("# Run the comparative benchmark on this repo".to_string());
out.push("lean-ctx benchmark compare".to_string());
out.push(String::new());
out.push("# Run on a specific repository".to_string());
out.push("lean-ctx benchmark compare --repo /path/to/repo".to_string());
out.push(String::new());
out.push("# Output to file".to_string());
out.push("lean-ctx benchmark compare --output BENCHMARKS.md".to_string());
out.push("```".to_string());
out.push(String::new());
}
fn write_footer(out: &mut Vec<String>) {
out.push("---".to_string());
out.push(String::new());
out.push(format!(
"*Generated by [lean-ctx](https://leanctx.com) v{} — Context Runtime for AI Agents*",
env!("CARGO_PKG_VERSION")
));
out.push(String::new());
out.push("**Disclaimer:** Competitor numbers are from published sources (docs, papers, READMEs). \
lean-ctx numbers are measured live. Different test repos will produce different results. \
Run `lean-ctx benchmark compare` on your own codebase for project-specific numbers.".to_string());
}
fn format_num(n: usize) -> String {
if n >= 1_000_000 {
format!("{:.1}M", n as f64 / 1_000_000.0)
} else if n >= 1_000 {
format!("{:.1}K", n as f64 / 1_000.0)
} else {
format!("{n}")
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::core::benchmark_compare::{competitors, system_info};
use std::path::Path;
fn make_test_report() -> CompareReport {
let metrics = metrics::measure_all(Path::new("src"));
CompareReport {
metrics,
system: system_info::collect(),
competitors: competitors::all_competitors(),
}
}
#[test]
fn markdown_contains_all_sections() {
let report = make_test_report();
let md = generate_markdown(&report);
assert!(md.contains("# lean-ctx Benchmark: Head-to-Head Comparison"));
assert!(md.contains("## Methodology"));
assert!(md.contains("## Compression Comparison"));
assert!(md.contains("## lean-ctx Mode Performance"));
assert!(md.contains("## Search Latency"));
assert!(md.contains("## Cold Start Performance"));
assert!(md.contains("## Disk Footprint"));
assert!(md.contains("## Feature Comparison"));
assert!(md.contains("## Session Simulation"));
assert!(md.contains("## Compression Visualization"));
assert!(md.contains("## System Information"));
assert!(md.contains("## Reproducibility"));
assert!(md.contains("Disclaimer"));
}
#[test]
fn markdown_contains_competitors() {
let report = make_test_report();
let md = generate_markdown(&report);
assert!(md.contains("Repomix"));
assert!(md.contains("codebase-memory"));
assert!(md.contains("Raw file read"));
}
#[test]
fn terminal_output_is_readable() {
let report = make_test_report();
let term = generate_terminal(&report);
assert!(term.contains("Head-to-Head Benchmark"));
assert!(term.contains("Compression Comparison"));
assert!(term.contains("Avg search latency"));
}
#[test]
fn format_num_ranges() {
assert_eq!(format_num(500), "500");
assert_eq!(format_num(1500), "1.5K");
assert_eq!(format_num(2_500_000), "2.5M");
}
}