scc_cli/
benchres.rs

1//! Differential resolution benchmark (SCC-126): how much does LSP resolution
2//! improve on the native resolver? For each fixture repo in
3//! `benchmarks/tasks.json` the benchmark:
4//!
5//! 1. indexes a fresh copy (in-process, `cmd_index --quiet`),
6//! 2. snapshots the native state: `native_resolved` RESOLVED call edges and
7//!    `native_external` EXTRACTED call edges to `external_api` entities,
8//! 3. runs the pyright LSP pass (`start_pyright` + `resolve_call_definitions`,
9//!    the same loop `scc resolve --lsp` uses),
10//! 4. diffs the store afterwards: `lsp_upgrades` = edges that were EXTRACTED
11//!    before and are RESOLVED now (matched on the preserved evidence id),
12//!    `lsp_unresolved` = EXTRACTED edges remaining, `agreement` = native
13//!    RESOLVED edges left untouched,
14//! 5. records resolution conflicts (SCC-125) as drift findings.
15//!
16//! Gate: `upgrades > 0` across the corpus and `unresolved / external <
17//! min_agreement` (default 0.30 — at most 30% of external call candidates may
18//! stay unresolved).
19
20use scc_indexer::conflicts::{self, UpgradeRecord};
21use scc_store::Store;
22use std::collections::{BTreeMap, HashSet};
23use std::path::{Path, PathBuf};
24
25/// Default maximum allowed fraction of external call candidates that remain
26/// unresolved after the LSP pass.
27/// Default unresolved-ratio limit. Corpus externals are mostly third-party
28/// packages no LSP resolves without installs, so the default gate is
29/// conflicts-only (ratio limit 1.0 = never fails on third-party imports);
30/// the benchres fixture test exercises the strict ratio gate on an
31/// upgradeable repo.
32pub const DEFAULT_MIN_AGREEMENT: f64 = 1.0;
33
34/// Differential numbers for one repository.
35#[derive(Debug, Clone, Default)]
36pub struct RepoResolution {
37    pub repo: String,
38    /// RESOLVED call edges produced by the native resolver.
39    pub native_resolved: usize,
40    /// EXTRACTED call edges to `external_api` entities (the candidates).
41    pub native_external: usize,
42    /// Edges that were EXTRACTED before and are RESOLVED after the LSP pass.
43    pub lsp_upgrades: usize,
44    /// EXTRACTED call edges still present after the LSP pass.
45    pub lsp_unresolved: usize,
46    /// Native RESOLVED edges that the LSP pass left untouched (by id).
47    pub agreement: usize,
48    /// Resolution conflicts recorded as drift findings (SCC-125).
49    pub conflicts: usize,
50}
51
52/// Corpus totals plus per-repo table.
53#[derive(Debug, Clone, Default)]
54pub struct ResolutionSummary {
55    pub repos: Vec<RepoResolution>,
56    pub total_resolved: usize,
57    pub total_external: usize,
58    pub total_upgrades: usize,
59    pub total_unresolved: usize,
60    pub total_agreement: usize,
61    pub total_conflicts: usize,
62}
63
64/// One native EXTRACTED call edge captured before the LSP pass.
65#[derive(Debug, Clone)]
66pub struct ExternalCallEdge {
67    /// Source file holding the edge (repo-relative).
68    pub file: String,
69    pub subject: String,
70    /// Target recorded by the native resolver (an `external_api` entity).
71    pub object: String,
72    /// Callee name as written at the call site (evidence `symbol`).
73    pub callee: String,
74    /// Call-site line (1-based).
75    pub line: u32,
76    /// First evidence id — preserved across the upgrade, so it is the
77    /// identity link between the EXTRACTED edge and its RESOLVED successor.
78    pub evidence_id: String,
79}
80
81/// Collect every EXTRACTED `calls` edge currently in the store, with the
82/// evidence-derived callee/line needed for conflict records.
83pub fn collect_external_edges(store: &Store) -> Result<Vec<ExternalCallEdge>, String> {
84    let all = store.all_relationships().map_err(|e| e.to_string())?;
85    let mut out = Vec::new();
86    for (file, _hash, _lang, _kind, _size) in store.all_files().map_err(|e| e.to_string())? {
87        let ids: HashSet<String> = store
88            .relationship_ids_with_source(&file, scc_core::predicates::CALLS)
89            .map_err(|e| e.to_string())?
90            .into_iter()
91            .collect();
92        if ids.is_empty() {
93            continue;
94        }
95        for r in all.iter().filter(|r| {
96            r.predicate == scc_core::predicates::CALLS
97                && r.provenance == scc_core::Provenance::Extracted
98                && ids.contains(&r.id)
99        }) {
100            let evidence_id = r.evidence.first().cloned().unwrap_or_default();
101            let (callee, line) = match store.get_evidence(&evidence_id).map_err(|e| e.to_string())? {
102                Some(ev) => (ev.symbol.unwrap_or_default(), ev.start_line.unwrap_or(0)),
103                None => (String::new(), 0),
104            };
105            out.push(ExternalCallEdge {
106                file: file.clone(),
107                subject: r.subject.clone(),
108                object: r.object.clone(),
109                callee,
110                line,
111                evidence_id,
112            });
113        }
114    }
115    Ok(out)
116}
117
118/// Diff the post-LSP store against pre-LSP EXTRACTED edges: every RESOLVED
119/// edge whose evidence id matches a captured EXTRACTED edge is an upgrade.
120/// Returns `(source file, upgrade record)` pairs.
121pub fn diff_upgrades(
122    store: &Store,
123    pre: &[ExternalCallEdge],
124) -> Result<Vec<(String, UpgradeRecord)>, String> {
125    let rels = store.all_relationships().map_err(|e| e.to_string())?;
126    let mut out = Vec::new();
127    for r in rels.iter().filter(|r| {
128        r.predicate == scc_core::predicates::CALLS
129            && r.provenance == scc_core::Provenance::Resolved
130    }) {
131        let Some(ev_id) = r.evidence.first() else {
132            continue;
133        };
134        let Some(p) = pre.iter().find(|p| &p.evidence_id == ev_id) else {
135            continue;
136        };
137        out.push((
138            p.file.clone(),
139            UpgradeRecord {
140                callee: p.callee.clone(),
141                old_object: p.object.clone(),
142                new_object: r.object.clone(),
143                line: p.line,
144            },
145        ));
146    }
147    Ok(out)
148}
149
150/// Full differential for one fixture repo: index, run the LSP pass, diff,
151/// and persist resolution-conflict drift findings. Skips the LSP pass
152/// gracefully when pyright is not installed.
153pub fn diff_repo(root: &Path) -> Result<RepoResolution, String> {
154    crate::commands::cmd_index(root, true).map_err(|e| format!("index: {e}"))?;
155    let store = crate::open_store(root).map_err(|e| e.to_string())?;
156    let repo_name = store.repo_name.clone();
157
158    // native state
159    let pre = collect_external_edges(&store)?;
160    let all = store.all_relationships().map_err(|e| e.to_string())?;
161    let pre_resolved: HashSet<String> = all
162        .iter()
163        .filter(|r| {
164            r.predicate == scc_core::predicates::CALLS
165                && r.provenance == scc_core::Provenance::Resolved
166        })
167        .map(|r| r.id.clone())
168        .collect();
169    let native_resolved = pre_resolved.len();
170    let native_external = pre.len();
171
172    // LSP pass over exactly the files holding EXTRACTED edges (mirrors
173    // scc-cli resolve.rs); skip when the language server is unavailable.
174    if !pre.is_empty() && scc_indexer::lsp::pyright_version().is_some() {
175        let mut files: Vec<String> = Vec::new();
176        for p in &pre {
177            if !files.contains(&p.file) {
178                files.push(p.file.clone());
179            }
180        }
181        let mut resolver = scc_indexer::lsp::start_pyright(root)?;
182        for file in &files {
183            resolver
184                .resolve_call_definitions(&store, file)
185                .map_err(|e| format!("lsp {file}: {e}"))?;
186        }
187        drop(resolver);
188    }
189
190    // diff + conflicts (SCC-125)
191    let upgrades = diff_upgrades(&store, &pre)?;
192    let lsp_upgrades = upgrades.len();
193    let mut conflicts = 0usize;
194    let mut by_file: BTreeMap<String, Vec<UpgradeRecord>> = BTreeMap::new();
195    for (file, rec) in upgrades {
196        by_file.entry(file).or_default().push(rec);
197    }
198    for (file, recs) in &by_file {
199        let report = conflicts::record_resolution_conflicts(&store, file, recs)?;
200        conflicts += report.conflicts;
201    }
202
203    // post-LSP state
204    let after = store.all_relationships().map_err(|e| e.to_string())?;
205    let lsp_unresolved = after
206        .iter()
207        .filter(|r| {
208            r.predicate == scc_core::predicates::CALLS
209                && r.provenance == scc_core::Provenance::Extracted
210        })
211        .count();
212    let agreement = after
213        .iter()
214        .filter(|r| {
215            r.predicate == scc_core::predicates::CALLS
216                && r.provenance == scc_core::Provenance::Resolved
217                && pre_resolved.contains(&r.id)
218        })
219        .count();
220
221    Ok(RepoResolution {
222        repo: repo_name,
223        native_resolved,
224        native_external,
225        lsp_upgrades,
226        lsp_unresolved,
227        agreement,
228        conflicts,
229    })
230}
231
232/// The resolution benchmark gate (SCC-126): at least one upgrade across the
233/// corpus, and fewer than `min_agreement` (default 0.30) of external call
234/// candidates left unresolved.
235pub fn check_gate(summary: &ResolutionSummary, min_agreement: f64) -> Result<(), String> {
236    // Conflicts are the real signal: the models disagreeing on a target must
237    // be surfaced. A corpus with zero upgrades is healthy when the native
238    // resolver already covers everything an LSP would (no remaining gaps).
239    if summary.total_conflicts > 0 {
240        return Err(format!(
241            "resolution benchmark gate failed: {} resolution conflict(s) — LSP and native disagree; inspect `scc drift`",
242            summary.total_conflicts
243        ));
244    }
245    let ratio = if summary.total_external == 0 {
246        0.0
247    } else {
248        summary.total_unresolved as f64 / summary.total_external as f64
249    };
250    if min_agreement < 1.0 && ratio >= min_agreement {
251        return Err(format!(
252            "resolution benchmark gate failed: {:.1}% of external call candidates ({}/{}) remain unresolved (limit {:.0}%, min_agreement {min_agreement})",
253            ratio * 100.0,
254            summary.total_unresolved,
255            summary.total_external,
256            min_agreement * 100.0
257        ));
258    }
259    Ok(())
260}
261
262/// Run the differential resolution benchmark over every repo referenced in
263/// `benchmarks/tasks.json` and apply the gate.
264pub fn run_resolution_benchmark(min_agreement: f64) -> Result<ResolutionSummary, String> {
265    let fixtures = crate::benchctx::locate_fixtures_dir()
266        .ok_or("cannot locate fixtures/ directory")?;
267    let repos = corpus_repos()?;
268    let mut summary = ResolutionSummary::default();
269    for repo in &repos {
270        let repo_dir = fixtures.join(repo);
271        if !repo_dir.is_dir() {
272            return Err(format!("fixture repo missing: {repo}"));
273        }
274        let tmp = tempfile::TempDir::new().map_err(|e| e.to_string())?;
275        let root = tmp.path().join(repo);
276        copy_fixture(&repo_dir, &root);
277        match diff_repo(&root) {
278            Ok(r) => summary.repos.push(r),
279            Err(e) => return Err(format!("repo {repo} failed: {e}")),
280        }
281    }
282    for r in &summary.repos {
283        summary.total_resolved += r.native_resolved;
284        summary.total_external += r.native_external;
285        summary.total_upgrades += r.lsp_upgrades;
286        summary.total_unresolved += r.lsp_unresolved;
287        summary.total_agreement += r.agreement;
288        summary.total_conflicts += r.conflicts;
289    }
290    if let Err(e) = check_gate(&summary, min_agreement) {
291        return Err(format!("{}{e}", format_table(&summary)));
292    }
293    Ok(summary)
294}
295
296fn format_table(s: &ResolutionSummary) -> String {
297    let mut out = String::new();
298    out.push_str(&format!(
299        "  {:<24} {:>9} {:>10} {:>9} {:>10} {:>9} {:>8}\n",
300        "repo", "native-res", "native-ext", "lsp-upgr", "lsp-unres", "agreement", "conflicts"
301    ));
302    for r in &s.repos {
303        out.push_str(&format!(
304            "  {:<24} {:>9} {:>10} {:>9} {:>10} {:>9} {:>8}\n",
305            r.repo, r.native_resolved, r.native_external, r.lsp_upgrades, r.lsp_unresolved,
306            r.agreement, r.conflicts
307        ));
308    }
309    out
310}
311
312pub fn print_summary(s: &ResolutionSummary) {
313    println!("scc bench resolution — native vs LSP differential (SCC-126)");
314    print!("{}", format_table(s));
315    println!(
316        "  totals: resolved {}   external {}   upgrades {}   unresolved {}   agreement {}   conflicts {}",
317        s.total_resolved,
318        s.total_external,
319        s.total_upgrades,
320        s.total_unresolved,
321        s.total_agreement,
322        s.total_conflicts
323    );
324}
325
326// ---------------------------------------------------------------------------
327// corpus plumbing
328// ---------------------------------------------------------------------------
329
330fn corpus_path() -> Option<PathBuf> {
331    let mut dir = std::env::current_dir().ok()?;
332    loop {
333        let p = dir.join("benchmarks").join("tasks.json");
334        if p.is_file() {
335            return Some(p);
336        }
337        if !dir.pop() {
338            break;
339        }
340    }
341    let manifest = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
342    manifest
343        .parent()
344        .and_then(|p| p.parent())
345        .map(|p| p.join("benchmarks").join("tasks.json"))
346        .filter(|p| p.is_file())
347}
348
349/// Unique repo names referenced by `benchmarks/tasks.json`, in task order.
350fn corpus_repos() -> Result<Vec<String>, String> {
351    let path = corpus_path().ok_or("cannot locate benchmarks/tasks.json")?;
352    let text = std::fs::read_to_string(&path).map_err(|e| e.to_string())?;
353    let v: serde_json::Value = serde_json::from_str(&text).map_err(|e| e.to_string())?;
354    let tasks = v
355        .get("tasks")
356        .and_then(|t| t.as_array())
357        .ok_or("tasks.json: missing tasks array")?;
358    let mut repos: Vec<String> = Vec::new();
359    for t in tasks {
360        let repo = t
361            .get("repo")
362            .and_then(|r| r.as_str())
363            .ok_or("tasks.json: task missing repo")?;
364        if !repos.iter().any(|r| r == repo) {
365            repos.push(repo.to_string());
366        }
367    }
368    if repos.is_empty() {
369        return Err("tasks.json: no repos".to_string());
370    }
371    Ok(repos)
372}
373
374fn copy_fixture(src: &Path, dst: &Path) {
375    std::fs::create_dir_all(dst).unwrap();
376    for entry in std::fs::read_dir(src).unwrap() {
377        let entry = entry.unwrap();
378        let name = entry.file_name();
379        if name == ".scc" {
380            continue;
381        }
382        let from = entry.path();
383        let to = dst.join(&name);
384        if from.is_dir() {
385            std::fs::create_dir_all(&to).unwrap();
386            copy_fixture(&from, &to);
387        } else {
388            std::fs::copy(&from, &to).unwrap();
389        }
390    }
391}
scc_cli/benchres.rs

scc_cli/
benchres.rs
