comindly_core/
eval.rs

1//! Graph-quality evaluation: compare resolved edges against a gold set (`expected.toml`).
2//!
3//! A fixture dir holds one subdirectory per repo plus `expected.toml`. Edges are keyed by
4//! `repo:path` (file) or `repo:path::Qualified.name`, independent of the descriptor scheme.
5
6use std::collections::{BTreeMap, BTreeSet, HashMap};
7use std::path::Path;
8
9use anyhow::{Context, Result};
10use serde::Deserialize;
11
12use crate::model::{Edge, EdgeKind, Symbol, SymbolKind};
13
14/// Edge kinds the gold set covers; other kinds are ignored on both sides.
15pub const EVAL_KINDS: &[&str] = &[
16    "imports",
17    "calls",
18    "inherits",
19    "implements",
20    "instantiates",
21    "serves",
22    "consumes",
23    "publishes",
24    "subscribes",
25    "reads",
26    "writes",
27    "defines",
28];
29
30#[derive(Debug, Clone, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
31pub struct GoldEdge {
32    pub case: String,
33    pub kind: String,
34    pub src: String,
35    pub dst: String,
36}
37
38#[derive(Debug, Deserialize)]
39struct Gold {
40    edge: Vec<GoldEdge>,
41}
42
43#[derive(Debug, Default, Clone, Copy)]
44pub struct Counts {
45    pub tp: usize,
46    pub fp: usize,
47    pub fn_: usize,
48}
49
50impl Counts {
51    pub fn precision(&self) -> f64 {
52        ratio(self.tp, self.tp + self.fp)
53    }
54    pub fn recall(&self) -> f64 {
55        ratio(self.tp, self.tp + self.fn_)
56    }
57}
58
59fn ratio(a: usize, b: usize) -> f64 {
60    if b == 0 {
61        1.0
62    } else {
63        a as f64 / b as f64
64    }
65}
66
67/// A produced edge, in gold-key form: (kind, src, dst).
68pub type KeyedEdge = (String, String, String);
69
70#[derive(Debug, Default)]
71pub struct EvalReport {
72    pub total: Counts,
73    pub by_kind: BTreeMap<String, Counts>,
74    /// case -> (found, expected)
75    pub by_case: BTreeMap<String, (usize, usize)>,
76    pub missed: Vec<GoldEdge>,
77    pub spurious: Vec<KeyedEdge>,
78    /// References answered by SCIP rather than tree-sitter resolution.
79    pub scip_answers: usize,
80}
81
82/// Parse + resolve every repo under `fixture` and score the result against its `expected.toml`.
83pub fn eval_fixture(fixture: &Path) -> Result<EvalReport> {
84    eval_with(fixture, None, None)
85}
86
87/// Like [`eval_fixture`], limited to `only` repos (gold filtered to edges among them), with
88/// `<scip>/<repo>.scip` indexes attached where present.
89pub fn eval_with(fixture: &Path, only: Option<&[&str]>, scip: Option<&Path>) -> Result<EvalReport> {
90    let gold_path = fixture.join("expected.toml");
91    let gold: Gold = toml::from_str(
92        &std::fs::read_to_string(&gold_path)
93            .with_context(|| format!("read {}", gold_path.display()))?,
94    )
95    .with_context(|| format!("parse {}", gold_path.display()))?;
96
97    let keep = |name: &str| only.is_none_or(|o| o.contains(&name));
98    let mut repos: Vec<_> = std::fs::read_dir(fixture)?
99        .filter_map(|e| e.ok())
100        .filter(|e| e.path().is_dir())
101        .filter(|e| keep(&e.file_name().to_string_lossy()))
102        .collect();
103    repos.sort_by_key(|e| e.file_name());
104
105    let mut symbols = Vec::new();
106    let mut edges = Vec::new();
107    let mut facts = Vec::new();
108    for r in repos {
109        let name = r.file_name().to_string_lossy().to_string();
110        let mut out = crate::parse::parse_repo(&r.path(), &name)?;
111        if let Some(index) = scip.map(|d| d.join(format!("{name}.scip"))) {
112            if index.is_file() {
113                crate::scip::attach(&index, "", &name, &out.symbols, &mut out.facts)?;
114            }
115        }
116        symbols.extend(out.symbols);
117        edges.extend(out.edges);
118        facts.extend(out.facts);
119    }
120    let resolved = crate::resolve::resolve(&symbols, &edges, &facts);
121    symbols.extend(resolved.symbols.iter().cloned());
122    let repo_of = |k: &str| k.split(':').next().unwrap_or_default().to_string();
123    let gold: Vec<GoldEdge> = gold
124        .edge
125        .into_iter()
126        .filter(|g| {
127            keep(&repo_of(&g.src)) && (g.dst.starts_with("contract:") || keep(&repo_of(&g.dst)))
128        })
129        .collect();
130    let mut report = score(&gold, &keyed_edges(&symbols, &resolved.edges));
131    report.scip_answers = resolved.stats.scip_answers;
132    Ok(report)
133}
134
135/// Render resolved edges of the evaluated kinds as gold keys.
136pub fn keyed_edges(symbols: &[Symbol], edges: &[Edge]) -> BTreeSet<KeyedEdge> {
137    let by_id: HashMap<String, &Symbol> = symbols.iter().map(|s| (s.id.render(), s)).collect();
138    let parent: HashMap<String, String> = edges
139        .iter()
140        .filter(|e| e.kind == EdgeKind::Contains)
141        .map(|e| (e.dst.render(), e.src.render()))
142        .collect();
143    let key = |id: &str| -> String {
144        let Some(sym) = by_id.get(id) else {
145            return format!("?{id}");
146        };
147        if sym.kind == SymbolKind::Contract {
148            return format!("contract:{}", sym.name);
149        }
150        let mut names = Vec::new();
151        let mut cur = Some(id.to_string());
152        while let Some(c) = cur {
153            match by_id.get(&c) {
154                Some(s) if s.kind != SymbolKind::File => names.push(s.name.clone()),
155                _ => break,
156            }
157            cur = parent.get(&c).cloned();
158        }
159        names.reverse();
160        if names.is_empty() {
161            format!("{}:{}", sym.repo.0, sym.file_path)
162        } else {
163            format!("{}:{}::{}", sym.repo.0, sym.file_path, names.join("."))
164        }
165    };
166    edges
167        .iter()
168        .filter(|e| EVAL_KINDS.contains(&e.kind.as_str()))
169        .map(|e| {
170            (
171                e.kind.as_str().to_string(),
172                key(&e.src.render()),
173                key(&e.dst.render()),
174            )
175        })
176        .collect()
177}
178
179pub fn score(gold: &[GoldEdge], produced: &BTreeSet<KeyedEdge>) -> EvalReport {
180    let mut r = EvalReport::default();
181    let gold_keys: BTreeSet<KeyedEdge> = gold
182        .iter()
183        .map(|g| (g.kind.clone(), g.src.clone(), g.dst.clone()))
184        .collect();
185    for g in gold {
186        let hit = produced.contains(&(g.kind.clone(), g.src.clone(), g.dst.clone()));
187        let c = r.by_kind.entry(g.kind.clone()).or_default();
188        let case = r.by_case.entry(g.case.clone()).or_default();
189        case.1 += 1;
190        if hit {
191            c.tp += 1;
192            case.0 += 1;
193        } else {
194            c.fn_ += 1;
195            r.missed.push(g.clone());
196        }
197    }
198    for p in produced.difference(&gold_keys) {
199        r.by_kind.entry(p.0.clone()).or_default().fp += 1;
200        r.spurious.push(p.clone());
201    }
202    for c in r.by_kind.values() {
203        r.total.tp += c.tp;
204        r.total.fp += c.fp;
205        r.total.fn_ += c.fn_;
206    }
207    r
208}
209
210#[cfg(test)]
211mod tests {
212    use super::*;
213
214    #[test]
215    fn score_counts_hits_misses_and_spurious() {
216        let gold = vec![
217            GoldEdge {
218                case: "a".into(),
219                kind: "calls".into(),
220                src: "r:x.py::f".into(),
221                dst: "r:x.py::g".into(),
222            },
223            GoldEdge {
224                case: "b".into(),
225                kind: "imports".into(),
226                src: "r:x.py".into(),
227                dst: "r:y.py::h".into(),
228            },
229        ];
230        let produced: BTreeSet<KeyedEdge> = [
231            ("calls".into(), "r:x.py::f".into(), "r:x.py::g".into()),
232            ("calls".into(), "r:x.py::f".into(), "r:x.py::zzz".into()),
233        ]
234        .into_iter()
235        .collect();
236        let r = score(&gold, &produced);
237        assert_eq!((r.total.tp, r.total.fp, r.total.fn_), (1, 1, 1));
238        assert_eq!(r.by_case["a"], (1, 1));
239        assert_eq!(r.by_case["b"], (0, 1));
240    }
241}
comindly_core/eval.rs

comindly_core/
eval.rs
