lean_ctx/core/benchmark_compare/

report.rs

use super::competitors::CompetitorProfile;
use super::metrics::{self, ComparativeMetrics};
use super::system_info::SystemInfo;

#[derive(Debug)]
pub struct CompareReport {
    pub metrics: ComparativeMetrics,
    pub system: SystemInfo,
    pub competitors: Vec<CompetitorProfile>,
}

pub fn generate_markdown(report: &CompareReport) -> String {
    let mut out = Vec::new();

    write_header(&mut out, report);
    write_methodology(&mut out);
    write_compression_table(&mut out, report);
    write_mode_detail(&mut out, report);
    write_search_latency(&mut out, report);
    write_cold_start(&mut out, report);
    write_disk_footprint(&mut out, report);
    write_feature_matrix(&mut out, report);
    write_session_simulation(&mut out, report);
    write_chart(&mut out, report);
    write_system_info(&mut out, &report.system);
    write_reproducibility(&mut out);
    write_footer(&mut out);

    out.join("\n")
}

pub fn generate_terminal(report: &CompareReport) -> String {
    let mut out = Vec::new();
    let sep = "\u{2550}".repeat(72);

    out.push(sep.clone());
    out.push("  lean-ctx Head-to-Head Benchmark".to_string());
    out.push(sep.clone());
    out.push(String::new());

    out.push("  Compression Comparison:".to_string());
    out.push(format!(
        "  {:<28} {:>12} {:>12} {:>10}",
        "Tool", "Compression", "Search", "Features"
    ));
    out.push(format!("  {}", "\u{2500}".repeat(66)));

    for c in &report.competitors {
        let comp = c
            .compression_pct
            .map_or("N/A".to_string(), |p| format!("{p:.0}%"));
        let search = if c.supports_search { "Yes" } else { "No" };
        out.push(format!(
            "  {:<28} {:>12} {:>12} {:>10}",
            c.name, comp, search, c.feature_count
        ));
    }

    for mc in &report.metrics.mode_comparisons {
        if mc.mode == "full" {
            continue;
        }
        out.push(format!(
            "  {:<28} {:>11.1}% {:>12} {:>10}",
            format!("lean-ctx ({})", mc.mode),
            mc.avg_compression_pct,
            "Yes",
            report.metrics.feature_count
        ));
    }

    out.push(String::new());

    let avg_search = metrics::avg_search_latency_us(&report.metrics.search_latencies);
    out.push(format!(
        "  Avg search latency: {}",
        metrics::format_duration_us(avg_search)
    ));
    out.push(format!(
        "  Cold start: {}",
        metrics::format_duration_us(report.metrics.cold_start.total_us)
    ));
    out.push(format!(
        "  Disk footprint: {}",
        metrics::format_bytes(report.metrics.disk_footprint.total_index_bytes)
    ));

    out.push(sep);
    out.join("\n")
}

fn write_header(out: &mut Vec<String>, report: &CompareReport) {
    out.push("# lean-ctx Benchmark: Head-to-Head Comparison".to_string());
    out.push(String::new());
    out.push(format!(
        "> Generated by lean-ctx v{} on {}",
        report.system.lean_ctx_version,
        chrono::Local::now().format("%Y-%m-%d %H:%M:%S")
    ));
    out.push(String::new());
    out.push(format!(
        "**Project:** `{}`  ",
        report.metrics.project_benchmark.root
    ));
    out.push(format!(
        "**Files measured:** {}  ",
        report.metrics.project_benchmark.files_measured
    ));
    out.push(format!(
        "**Total raw tokens:** {}  ",
        format_num(report.metrics.project_benchmark.total_raw_tokens)
    ));
    out.push(String::new());
}

fn write_methodology(out: &mut Vec<String>) {
    out.push("## Methodology".to_string());
    out.push(String::new());
    out.push(
        "All lean-ctx measurements are **real values** measured on the test repository. \
              Competitor numbers use **published figures** from their official documentation, \
              papers, or README files. Sources are cited in the comparison table."
            .to_string(),
    );
    out.push(String::new());
    out.push("- **Token counting**: tiktoken `o200k_base` (GPT-4o tokenizer)".to_string());
    out.push("- **Compression**: Each lean-ctx read mode is applied to the same files".to_string());
    out.push("- **Latency**: Wall-clock time, median of all measured files".to_string());
    out.push("- **Quality**: Preservation score (structural + semantic fidelity)".to_string());
    out.push(String::new());
}

fn write_compression_table(out: &mut Vec<String>, report: &CompareReport) {
    out.push("## Compression Comparison".to_string());
    out.push(String::new());
    out.push("| Tool | Compression | Tokens | Source |".to_string());
    out.push("|------|------------:|-------:|--------|".to_string());

    let raw_tokens = report.metrics.project_benchmark.total_raw_tokens;

    for c in &report.competitors {
        let comp = c
            .compression_pct
            .map_or("N/A".to_string(), |p| format!("{p:.0}%"));
        let tokens = c.compression_pct.map_or("—".to_string(), |p| {
            format_num((raw_tokens as f64 * (1.0 - p / 100.0)) as usize)
        });
        out.push(format!(
            "| {} | {} | {} | {} |",
            c.name, comp, tokens, c.source
        ));
    }

    for mc in &report.metrics.mode_comparisons {
        if mc.mode == "full" {
            continue;
        }
        out.push(format!(
            "| **lean-ctx ({})** | **{:.1}%** | **{}** | Measured |",
            mc.mode,
            mc.avg_compression_pct,
            format_num(mc.total_compressed_tokens),
        ));
    }

    out.push(String::new());
}

fn write_mode_detail(out: &mut Vec<String>, report: &CompareReport) {
    out.push("## lean-ctx Mode Performance".to_string());
    out.push(String::new());
    out.push("| Mode | Compression | Latency | Quality | Use Case |".to_string());
    out.push("|------|------------:|--------:|--------:|----------|".to_string());

    for mc in &report.metrics.mode_comparisons {
        let use_case = match mc.mode.as_str() {
            "full" => "Editing files (cached, ~13 tok on re-read)",
            "map" => "Understanding structure, deps, exports",
            "signatures" => "API surface only",
            "aggressive" => "Maximum compression for large files",
            "entropy" => "Information-theoretic filtering",
            _ => "",
        };
        let quality = if mc.avg_quality > 0.0 {
            format!("{:.0}%", mc.avg_quality * 100.0)
        } else {
            "100%".to_string()
        };
        out.push(format!(
            "| {} | {:.1}% | {} | {} | {} |",
            mc.mode,
            mc.avg_compression_pct,
            metrics::format_duration_us(mc.avg_latency_us),
            quality,
            use_case,
        ));
    }

    out.push(String::new());
}

fn write_search_latency(out: &mut Vec<String>, report: &CompareReport) {
    out.push("## Search Latency".to_string());
    out.push(String::new());
    out.push("| Query | BM25 Latency | Results |".to_string());
    out.push("|-------|-------------:|--------:|".to_string());

    for sl in &report.metrics.search_latencies {
        out.push(format!(
            "| `{}` | {} | {} |",
            sl.query,
            metrics::format_duration_us(sl.bm25_us),
            sl.result_count,
        ));
    }

    let avg = metrics::avg_search_latency_us(&report.metrics.search_latencies);
    out.push(format!(
        "| **Average** | **{}** | — |",
        metrics::format_duration_us(avg)
    ));
    out.push(String::new());
}

fn write_cold_start(out: &mut Vec<String>, report: &CompareReport) {
    let cs = &report.metrics.cold_start;
    out.push("## Cold Start Performance".to_string());
    out.push(String::new());
    out.push("| Phase | Duration |".to_string());
    out.push("|-------|--------:|".to_string());
    out.push(format!(
        "| File scan | {} |",
        metrics::format_duration_us(cs.scan_us)
    ));
    out.push(format!(
        "| BM25 index build | {} |",
        metrics::format_duration_us(cs.bm25_build_us)
    ));
    out.push(format!(
        "| First file read + tokenize | {} |",
        metrics::format_duration_us(cs.first_read_us)
    ));
    out.push(format!(
        "| **Total cold start** | **{}** |",
        metrics::format_duration_us(cs.total_us)
    ));
    out.push(String::new());
}

fn write_disk_footprint(out: &mut Vec<String>, report: &CompareReport) {
    let df = &report.metrics.disk_footprint;
    out.push("## Disk Footprint".to_string());
    out.push(String::new());
    out.push("| Component | Size |".to_string());
    out.push("|-----------|-----:|".to_string());
    out.push(format!(
        "| BM25 index | {} |",
        metrics::format_bytes(df.bm25_index_bytes)
    ));
    out.push(format!(
        "| Total `.lean-ctx/` | {} |",
        metrics::format_bytes(df.total_index_bytes)
    ));
    out.push(String::new());
}

fn write_feature_matrix(out: &mut Vec<String>, report: &CompareReport) {
    out.push("## Feature Comparison".to_string());
    out.push(String::new());
    out.push("| Feature | Raw | Repomix | aider | codebase-memory | **lean-ctx** |".to_string());
    out.push("|---------|:---:|:-------:|:-----:|:---------------:|:------------:|".to_string());

    let features = [
        ("Multi-mode compression", [false, false, false, false, true]),
        ("BM25 code search", [false, false, false, true, true]),
        ("Session caching", [false, false, true, true, true]),
        (
            "Cross-session memory (CCP)",
            [false, false, false, true, true],
        ),
        (
            "Shell output compression",
            [false, false, false, false, true],
        ),
        ("Call graph analysis", [false, false, false, false, true]),
        ("Repo map generation", [false, true, true, false, true]),
        ("Knowledge base", [false, false, false, true, true]),
        (
            "Tree-sitter AST (18 langs)",
            [false, true, true, false, true],
        ),
        ("MCP server", [false, false, false, true, true]),
    ];

    for (feature, support) in &features {
        let cells: Vec<&str> = support
            .iter()
            .map(|s| if *s { "✅" } else { "—" })
            .collect();
        out.push(format!(
            "| {} | {} | {} | {} | {} | {} |",
            feature, cells[0], cells[1], cells[2], cells[3], cells[4]
        ));
    }

    out.push(String::new());
    out.push(format!(
        "**lean-ctx feature count:** {} operations across {} MCP tools",
        report.metrics.feature_count,
        crate::server::registry::tool_count()
    ));
    out.push(String::new());
}

fn write_session_simulation(out: &mut Vec<String>, report: &CompareReport) {
    let s = &report.metrics.project_benchmark.session_sim;
    out.push("## Session Simulation (30-min coding)".to_string());
    out.push(String::new());
    out.push("| Approach | Tokens | Cost | Savings |".to_string());
    out.push("|----------|-------:|-----:|--------:|".to_string());

    out.push(format!(
        "| Raw (no compression) | {} | ${:.3} | — |",
        format_num(s.raw_tokens),
        s.raw_cost
    ));

    let lean_pct = if s.raw_tokens > 0 {
        (1.0 - s.lean_tokens as f64 / s.raw_tokens as f64) * 100.0
    } else {
        0.0
    };
    out.push(format!(
        "| lean-ctx (no CCP) | {} | ${:.3} | {:.1}% |",
        format_num(s.lean_tokens),
        s.lean_cost,
        lean_pct
    ));

    let ccp_pct = if s.raw_tokens > 0 {
        (1.0 - s.lean_ccp_tokens as f64 / s.raw_tokens as f64) * 100.0
    } else {
        0.0
    };
    out.push(format!(
        "| **lean-ctx + CCP** | **{}** | **${:.3}** | **{:.1}%** |",
        format_num(s.lean_ccp_tokens),
        s.ccp_cost,
        ccp_pct
    ));

    out.push(String::new());
}

fn write_chart(out: &mut Vec<String>, report: &CompareReport) {
    out.push("## Compression Visualization".to_string());
    out.push(String::new());
    out.push("```".to_string());
    out.push("Compression % (higher = better)".to_string());
    out.push(String::new());

    let mut entries: Vec<(String, f64)> = Vec::new();

    for c in &report.competitors {
        if let Some(pct) = c.compression_pct {
            entries.push((c.name.to_string(), pct));
        }
    }
    for mc in &report.metrics.mode_comparisons {
        if mc.mode != "full" {
            entries.push((format!("lean-ctx ({})", mc.mode), mc.avg_compression_pct));
        }
    }

    entries.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal));

    let max_bar = 50;
    for (name, pct) in &entries {
        let bar_len = (pct / 100.0 * max_bar as f64) as usize;
        let bar = "\u{2588}".repeat(bar_len);
        out.push(format!("{name:<28} {bar} {pct:.1}%"));
    }

    out.push("```".to_string());
    out.push(String::new());
}

fn write_system_info(out: &mut Vec<String>, sys: &SystemInfo) {
    out.push("## System Information".to_string());
    out.push(String::new());
    out.push(format!("- **OS:** {} {}", sys.os, sys.arch));
    out.push(format!(
        "- **CPU:** {} ({} cores)",
        sys.cpu_brand, sys.cpu_cores
    ));
    out.push(format!("- **RAM:** {:.1} GB", sys.memory_gb));
    out.push(format!("- **lean-ctx:** v{}", sys.lean_ctx_version));
    out.push(format!("- **Rust:** {}", sys.rust_version));
    out.push(String::new());
}

fn write_reproducibility(out: &mut Vec<String>) {
    out.push("## Reproducibility".to_string());
    out.push(String::new());
    out.push("```bash".to_string());
    out.push("# Install lean-ctx".to_string());
    out.push("cargo install lean-ctx".to_string());
    out.push(String::new());
    out.push("# Run the comparative benchmark on this repo".to_string());
    out.push("lean-ctx benchmark compare".to_string());
    out.push(String::new());
    out.push("# Run on a specific repository".to_string());
    out.push("lean-ctx benchmark compare --repo /path/to/repo".to_string());
    out.push(String::new());
    out.push("# Output to file".to_string());
    out.push("lean-ctx benchmark compare --output BENCHMARKS.md".to_string());
    out.push("```".to_string());
    out.push(String::new());
}

fn write_footer(out: &mut Vec<String>) {
    out.push("---".to_string());
    out.push(String::new());
    out.push(format!(
        "*Generated by [lean-ctx](https://leanctx.com) v{} — Context Runtime for AI Agents*",
        env!("CARGO_PKG_VERSION")
    ));
    out.push(String::new());
    out.push("**Disclaimer:** Competitor numbers are from published sources (docs, papers, READMEs). \
              lean-ctx numbers are measured live. Different test repos will produce different results. \
              Run `lean-ctx benchmark compare` on your own codebase for project-specific numbers.".to_string());
}

fn format_num(n: usize) -> String {
    if n >= 1_000_000 {
        format!("{:.1}M", n as f64 / 1_000_000.0)
    } else if n >= 1_000 {
        format!("{:.1}K", n as f64 / 1_000.0)
    } else {
        format!("{n}")
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::core::benchmark_compare::{competitors, system_info};
    use std::path::Path;

    fn make_test_report() -> CompareReport {
        let metrics = metrics::measure_all(Path::new("src"));
        CompareReport {
            metrics,
            system: system_info::collect(),
            competitors: competitors::all_competitors(),
        }
    }

    #[test]
    fn markdown_contains_all_sections() {
        let report = make_test_report();
        let md = generate_markdown(&report);

        assert!(md.contains("# lean-ctx Benchmark: Head-to-Head Comparison"));
        assert!(md.contains("## Methodology"));
        assert!(md.contains("## Compression Comparison"));
        assert!(md.contains("## lean-ctx Mode Performance"));
        assert!(md.contains("## Search Latency"));
        assert!(md.contains("## Cold Start Performance"));
        assert!(md.contains("## Disk Footprint"));
        assert!(md.contains("## Feature Comparison"));
        assert!(md.contains("## Session Simulation"));
        assert!(md.contains("## Compression Visualization"));
        assert!(md.contains("## System Information"));
        assert!(md.contains("## Reproducibility"));
        assert!(md.contains("Disclaimer"));
    }

    #[test]
    fn markdown_contains_competitors() {
        let report = make_test_report();
        let md = generate_markdown(&report);

        assert!(md.contains("Repomix"));
        assert!(md.contains("codebase-memory"));
        assert!(md.contains("Raw file read"));
    }

    #[test]
    fn terminal_output_is_readable() {
        let report = make_test_report();
        let term = generate_terminal(&report);

        assert!(term.contains("Head-to-Head Benchmark"));
        assert!(term.contains("Compression Comparison"));
        assert!(term.contains("Avg search latency"));
    }

    #[test]
    fn format_num_ranges() {
        assert_eq!(format_num(500), "500");
        assert_eq!(format_num(1500), "1.5K");
        assert_eq!(format_num(2_500_000), "2.5M");
    }
}