lean_ctx/core/scorecard/
mod.rs

1//! Reproducible scorecard (#211): one command → compression savings + retrieval
2//! recall/MRR + latency over a fixed, deterministic scenario matrix.
3//!
4//! The quality metrics (savings, recall, MRR) are reproducible run-to-run and
5//! machine-to-machine because the corpus is generated deterministically and the
6//! retrieval path is pure BM25. Latency is measured wall-clock and therefore
7//! reported but not part of the determinism contract (see `determinism_digest`).
8
9pub mod dual_arm;
10mod scenarios;
11
12use std::collections::BTreeMap;
13use std::time::Instant;
14
15use serde::Serialize;
16
17use crate::core::{benchmark, bm25_index::BM25Index};
18
19/// Per-scenario result.
20#[derive(Debug, Clone, Serialize, PartialEq)]
21pub struct ScenarioScore {
22    pub name: String,
23    pub files: usize,
24    pub raw_tokens: usize,
25    pub best_mode: String,
26    pub best_savings_pct: f64,
27    pub savings_by_mode: BTreeMap<String, f64>,
28    pub queries: usize,
29    pub recall_at_5: f64,
30    pub recall_at_10: f64,
31    pub mrr: f64,
32    /// Wall-clock; informational only (not part of the determinism contract).
33    pub search_latency_us_p50: u64,
34}
35
36/// Cross-scenario averages.
37#[derive(Debug, Clone, Serialize, PartialEq)]
38pub struct Aggregate {
39    pub avg_savings_pct: f64,
40    pub avg_recall_at_5: f64,
41    pub avg_recall_at_10: f64,
42    pub avg_mrr: f64,
43}
44
45/// The full scorecard.
46#[derive(Debug, Clone, Serialize)]
47pub struct Scorecard {
48    pub schema_version: u32,
49    pub tokenizer: String,
50    /// Stable fingerprint of the reproducible (latency-free) metrics. Serialized
51    /// so the JSON artifact is self-verifying: two runs on the same code (any
52    /// machine) yield the same digest.
53    pub determinism_digest: String,
54    pub scenarios: Vec<ScenarioScore>,
55    pub aggregate: Aggregate,
56}
57
58impl Scorecard {
59    /// Compute the stable fingerprint from scenario scores. Latency is excluded
60    /// by construction, so two runs on the same code must produce the same value.
61    fn compute_digest(scenarios: &[ScenarioScore]) -> String {
62        let mut parts: Vec<String> = Vec::new();
63        for s in scenarios {
64            let modes: Vec<String> = s
65                .savings_by_mode
66                .iter()
67                .map(|(m, v)| format!("{m}={v:.2}"))
68                .collect();
69            parts.push(format!(
70                "{}|raw={}|best={}:{:.2}|r5={:.4}|r10={:.4}|mrr={:.4}|{}",
71                s.name,
72                s.raw_tokens,
73                s.best_mode,
74                s.best_savings_pct,
75                s.recall_at_5,
76                s.recall_at_10,
77                s.mrr,
78                modes.join(",")
79            ));
80        }
81        parts.join(";")
82    }
83
84    pub fn to_json(&self) -> String {
85        serde_json::to_string_pretty(self).unwrap_or_else(|_| "{}".to_string())
86    }
87
88    /// Human-readable scorecard table.
89    pub fn to_human(&self) -> String {
90        let mut out = String::new();
91        out.push_str("lean-ctx scorecard\n");
92        out.push_str(&format!("tokenizer: {}\n", self.tokenizer));
93        out.push_str(&format!("digest:    {}\n\n", self.determinism_digest));
94        out.push_str(
95            "scenario   files  raw_tokens  best_mode      savings%  R@5    R@10   MRR    p50(us)\n",
96        );
97        out.push_str(
98            "--------------------------------------------------------------------------------\n",
99        );
100        for s in &self.scenarios {
101            out.push_str(&format!(
102                "{:<10} {:>5}  {:>10}  {:<13}  {:>7.2}  {:>4.2}  {:>4.2}  {:>4.2}  {:>7}\n",
103                s.name,
104                s.files,
105                s.raw_tokens,
106                s.best_mode,
107                s.best_savings_pct,
108                s.recall_at_5,
109                s.recall_at_10,
110                s.mrr,
111                s.search_latency_us_p50,
112            ));
113        }
114        out.push_str(
115            "--------------------------------------------------------------------------------\n",
116        );
117        out.push_str(&format!(
118            "aggregate                            {:<13}  {:>7.2}  {:>4.2}  {:>4.2}  {:>4.2}\n",
119            "",
120            self.aggregate.avg_savings_pct,
121            self.aggregate.avg_recall_at_5,
122            self.aggregate.avg_recall_at_10,
123            self.aggregate.avg_mrr,
124        ));
125        out
126    }
127}
128
129/// Run the full scenario matrix and assemble the scorecard.
130pub fn run_scorecard() -> std::io::Result<Scorecard> {
131    let mut scenarios = Vec::with_capacity(scenarios::SCENARIOS.len());
132    for sc in scenarios::SCENARIOS {
133        scenarios.push(run_one(sc)?);
134    }
135    let aggregate = aggregate(&scenarios);
136    let determinism_digest = Scorecard::compute_digest(&scenarios);
137    Ok(Scorecard {
138        schema_version: 1,
139        tokenizer: crate::core::tokens::counting_family_label(),
140        determinism_digest,
141        scenarios,
142        aggregate,
143    })
144}
145
146fn run_one(sc: &scenarios::Scenario) -> std::io::Result<ScenarioScore> {
147    let dir = tempfile::TempDir::new()?;
148    let root = dir.path();
149    let queries = scenarios::materialize(sc, root)?;
150
151    // --- Compression savings (existing, tested benchmark path) ---
152    let bench = benchmark::run_project_benchmark(&root.to_string_lossy());
153    let mut savings_by_mode = BTreeMap::new();
154    for m in &bench.mode_summaries {
155        savings_by_mode.insert(m.mode.clone(), round2(m.avg_savings_pct));
156    }
157    let (best_mode, best_savings_pct) = savings_by_mode
158        .iter()
159        .max_by(|a, b| a.1.partial_cmp(b.1).unwrap_or(std::cmp::Ordering::Equal))
160        .map_or_else(|| ("none".to_string(), 0.0), |(m, v)| (m.clone(), *v));
161
162    // --- Retrieval quality (pure BM25 → deterministic, feature-independent) ---
163    let index = BM25Index::build_from_directory(root);
164    let mut sum_r5 = 0.0;
165    let mut sum_r10 = 0.0;
166    let mut sum_mrr = 0.0;
167    let mut latencies: Vec<u64> = Vec::with_capacity(queries.len());
168    for q in &queries {
169        let start = Instant::now();
170        let results = index.search(&q.query, 10);
171        latencies.push(start.elapsed().as_micros() as u64);
172        let files = dedup_files(&results);
173        sum_r5 += recall_at_k(&files, &q.expected_file, 5);
174        sum_r10 += recall_at_k(&files, &q.expected_file, 10);
175        sum_mrr += reciprocal_rank(&files, &q.expected_file);
176    }
177    let n = queries.len().max(1) as f64;
178
179    Ok(ScenarioScore {
180        name: sc.name.to_string(),
181        files: sc.files,
182        raw_tokens: bench.total_raw_tokens,
183        best_mode,
184        best_savings_pct,
185        savings_by_mode,
186        queries: queries.len(),
187        recall_at_5: round2(sum_r5 / n),
188        recall_at_10: round2(sum_r10 / n),
189        mrr: round2(sum_mrr / n),
190        search_latency_us_p50: percentile_p50(&mut latencies),
191    })
192}
193
194fn aggregate(scenarios: &[ScenarioScore]) -> Aggregate {
195    if scenarios.is_empty() {
196        return Aggregate {
197            avg_savings_pct: 0.0,
198            avg_recall_at_5: 0.0,
199            avg_recall_at_10: 0.0,
200            avg_mrr: 0.0,
201        };
202    }
203    let n = scenarios.len() as f64;
204    Aggregate {
205        avg_savings_pct: round2(scenarios.iter().map(|s| s.best_savings_pct).sum::<f64>() / n),
206        avg_recall_at_5: round2(scenarios.iter().map(|s| s.recall_at_5).sum::<f64>() / n),
207        avg_recall_at_10: round2(scenarios.iter().map(|s| s.recall_at_10).sum::<f64>() / n),
208        avg_mrr: round2(scenarios.iter().map(|s| s.mrr).sum::<f64>() / n),
209    }
210}
211
212/// Unique file paths in rank order (the first chunk of each file defines rank).
213fn dedup_files(results: &[crate::core::bm25_index::SearchResult]) -> Vec<String> {
214    let mut seen = std::collections::HashSet::new();
215    let mut files = Vec::new();
216    for r in results {
217        if seen.insert(r.file_path.clone()) {
218            files.push(r.file_path.clone());
219        }
220    }
221    files
222}
223
224/// Platform-independent suffix match (mirrors `eval_harness` convention).
225fn path_matches(retrieved: &str, expected: &str) -> bool {
226    let r = retrieved.replace('\\', "/");
227    let e = expected.replace('\\', "/");
228    r.ends_with(&e) || e.ends_with(&r)
229}
230
231fn recall_at_k(files: &[String], expected: &str, k: usize) -> f64 {
232    if files.iter().take(k).any(|f| path_matches(f, expected)) {
233        1.0
234    } else {
235        0.0
236    }
237}
238
239fn reciprocal_rank(files: &[String], expected: &str) -> f64 {
240    for (i, f) in files.iter().enumerate() {
241        if path_matches(f, expected) {
242            return 1.0 / (i as f64 + 1.0);
243        }
244    }
245    0.0
246}
247
248fn percentile_p50(latencies: &mut [u64]) -> u64 {
249    if latencies.is_empty() {
250        return 0;
251    }
252    latencies.sort_unstable();
253    latencies[latencies.len() / 2]
254}
255
256fn round2(v: f64) -> f64 {
257    (v * 100.0).round() / 100.0
258}
259
260#[cfg(test)]
261mod tests {
262    use super::*;
263
264    #[test]
265    fn path_matching_is_suffix_based() {
266        assert!(path_matches(
267            "/tmp/x/src/auth/file_001.rs",
268            "src/auth/file_001.rs"
269        ));
270        assert!(!path_matches(
271            "src/auth/file_002.rs",
272            "src/auth/file_001.rs"
273        ));
274    }
275
276    #[test]
277    fn recall_and_rr_basics() {
278        let files = vec![
279            "src/db/file_000.rs".to_string(),
280            "src/auth/file_001.rs".to_string(),
281        ];
282        assert_eq!(recall_at_k(&files, "src/auth/file_001.rs", 5), 1.0);
283        assert_eq!(recall_at_k(&files, "src/auth/file_001.rs", 1), 0.0);
284        assert_eq!(reciprocal_rank(&files, "src/auth/file_001.rs"), 0.5);
285        assert_eq!(reciprocal_rank(&files, "src/missing.rs"), 0.0);
286    }
287
288    #[test]
289    fn round2_rounds() {
290        assert_eq!(round2(12.3456), 12.35);
291        assert_eq!(round2(0.0), 0.0);
292    }
293}
lean_ctx/core/scorecard/mod.rs

lean_ctx/core/scorecard/
mod.rs
