the_code_graph_eval/
report.rs

1use serde::Serialize;
2use std::io::Write;
3
4#[derive(Debug, Clone, Serialize)]
5pub struct SuiteResult {
6    pub search: Option<SearchSuiteResult>,
7    pub impact: Option<ImpactSuiteResult>,
8}
9
10#[derive(Debug, Clone, Serialize)]
11pub struct CategoryMrr {
12    pub category: String,
13    pub queries: usize,
14    pub mrr: f64,
15}
16
17#[derive(Debug, Clone, Serialize)]
18pub struct SearchSuiteResult {
19    pub repos: usize,
20    pub queries: usize,
21    pub mrr: f64,
22    pub precision_at_5: f64,
23    pub precision_at_10: f64,
24    pub mrr_target: f64,
25    pub mrr_passed: bool,
26    pub per_category: Vec<CategoryMrr>,
27}
28
29#[derive(Debug, Clone, Serialize)]
30pub struct ImpactSuiteResult {
31    pub repos: usize,
32    pub scenarios: usize,
33    pub precision: f64,
34    pub recall: f64,
35    pub f1: f64,
36    pub precision_target: f64,
37    pub precision_passed: bool,
38}
39
40impl SuiteResult {
41    /// Returns true if all quality targets are met.
42    pub fn all_passed(&self) -> bool {
43        let search_ok = self.search.as_ref().is_none_or(|s| s.mrr_passed);
44        let impact_ok = self.impact.as_ref().is_none_or(|i| i.precision_passed);
45        search_ok && impact_ok
46    }
47
48    /// Write compact human-readable output matching the SPEC format.
49    pub fn fmt_compact(&self, w: &mut dyn Write) -> std::io::Result<()> {
50        if let Some(search) = &self.search {
51            let status = if search.mrr_passed { "PASS" } else { "FAIL" };
52            writeln!(
53                w,
54                "Search Suite — {} repos, {} queries",
55                search.repos, search.queries
56            )?;
57            writeln!(
58                w,
59                "  MRR:          {:.2} (target: >={:.2}) {}",
60                search.mrr, search.mrr_target, status
61            )?;
62            writeln!(w, "  Precision@5:  {:.2}", search.precision_at_5)?;
63            writeln!(w, "  Precision@10: {:.2}", search.precision_at_10)?;
64            if !search.per_category.is_empty() {
65                writeln!(w, "  Per-category MRR:")?;
66                for cat in &search.per_category {
67                    writeln!(
68                        w,
69                        "    {:12} {} queries  MRR: {:.2}",
70                        cat.category, cat.queries, cat.mrr
71                    )?;
72                }
73            }
74        }
75        if let Some(impact) = &self.impact {
76            let status = if impact.precision_passed {
77                "PASS"
78            } else {
79                "FAIL"
80            };
81            if self.search.is_some() {
82                writeln!(w)?;
83            }
84            writeln!(
85                w,
86                "Impact Suite — {} repos, {} scenarios",
87                impact.repos, impact.scenarios
88            )?;
89            writeln!(
90                w,
91                "  Precision:    {:.2} (target: >={:.2}) {}",
92                impact.precision, impact.precision_target, status
93            )?;
94            writeln!(w, "  Recall:       {:.2}", impact.recall)?;
95            writeln!(w, "  F1:           {:.2}", impact.f1)?;
96        }
97        Ok(())
98    }
99
100    /// Write tabular breakdown of all metrics.
101    pub fn fmt_table(&self, w: &mut dyn Write) -> std::io::Result<()> {
102        writeln!(w, "Suite   | Metric       | Value | Target | Status")?;
103        writeln!(w, "--------+--------------+-------+--------+-------")?;
104        if let Some(search) = &self.search {
105            let status = if search.mrr_passed { "PASS" } else { "FAIL" };
106            writeln!(
107                w,
108                "Search  | MRR          | {:.2}  | >{:.2}  | {}",
109                search.mrr, search.mrr_target, status
110            )?;
111            writeln!(
112                w,
113                "Search  | Precision@5  | {:.2}  |        |",
114                search.precision_at_5
115            )?;
116            writeln!(
117                w,
118                "Search  | Precision@10 | {:.2}  |        |",
119                search.precision_at_10
120            )?;
121            for cat in &search.per_category {
122                writeln!(
123                    w,
124                    "Search  | MRR/{:<8} | {:.2}  |        |",
125                    cat.category, cat.mrr
126                )?;
127            }
128        }
129        if let Some(impact) = &self.impact {
130            let status = if impact.precision_passed {
131                "PASS"
132            } else {
133                "FAIL"
134            };
135            writeln!(
136                w,
137                "Impact  | Precision    | {:.2}  | >{:.2}  | {}",
138                impact.precision, impact.precision_target, status
139            )?;
140            writeln!(
141                w,
142                "Impact  | Recall       | {:.2}  |        |",
143                impact.recall
144            )?;
145            writeln!(w, "Impact  | F1           | {:.2}  |        |", impact.f1)?;
146        }
147        Ok(())
148    }
149
150    /// Write JSON representation of all results.
151    pub fn fmt_json(&self, w: &mut dyn Write) -> std::io::Result<()> {
152        let json = serde_json::to_string_pretty(self).map_err(std::io::Error::other)?;
153        writeln!(w, "{json}")
154    }
155}
156
157#[cfg(test)]
158mod tests {
159    use super::*;
160
161    fn sample_search() -> SearchSuiteResult {
162        SearchSuiteResult {
163            repos: 5,
164            queries: 52,
165            mrr: 0.62,
166            precision_at_5: 0.71,
167            precision_at_10: 0.58,
168            mrr_target: 0.30,
169            mrr_passed: true,
170            per_category: vec![
171                CategoryMrr {
172                    category: "exact".into(),
173                    queries: 20,
174                    mrr: 0.80,
175                },
176                CategoryMrr {
177                    category: "semantic".into(),
178                    queries: 20,
179                    mrr: 0.50,
180                },
181                CategoryMrr {
182                    category: "partial".into(),
183                    queries: 12,
184                    mrr: 0.45,
185                },
186            ],
187        }
188    }
189
190    fn sample_impact() -> ImpactSuiteResult {
191        ImpactSuiteResult {
192            repos: 5,
193            scenarios: 24,
194            precision: 0.61,
195            recall: 0.48,
196            f1: 0.54,
197            precision_target: 0.40,
198            precision_passed: true,
199        }
200    }
201
202    #[test]
203    fn suite_result_compact_search_only() {
204        let result = SuiteResult {
205            search: Some(sample_search()),
206            impact: None,
207        };
208        let mut buf = Vec::new();
209        result.fmt_compact(&mut buf).unwrap();
210        let output = String::from_utf8(buf).unwrap();
211        assert!(output.contains("Search Suite — 5 repos, 52 queries"));
212        assert!(output.contains("MRR:          0.62 (target: >=0.30) PASS"));
213        assert!(output.contains("Precision@5:  0.71"));
214        assert!(output.contains("Precision@10: 0.58"));
215        assert!(!output.contains("Impact Suite"));
216    }
217
218    #[test]
219    fn suite_result_compact_impact_only() {
220        let result = SuiteResult {
221            search: None,
222            impact: Some(sample_impact()),
223        };
224        let mut buf = Vec::new();
225        result.fmt_compact(&mut buf).unwrap();
226        let output = String::from_utf8(buf).unwrap();
227        assert!(output.contains("Impact Suite — 5 repos, 24 scenarios"));
228        assert!(output.contains("Precision:    0.61 (target: >=0.40) PASS"));
229        assert!(output.contains("Recall:       0.48"));
230        assert!(output.contains("F1:           0.54"));
231        assert!(!output.contains("Search Suite"));
232    }
233
234    #[test]
235    fn suite_result_compact_all() {
236        let result = SuiteResult {
237            search: Some(sample_search()),
238            impact: Some(sample_impact()),
239        };
240        let mut buf = Vec::new();
241        result.fmt_compact(&mut buf).unwrap();
242        let output = String::from_utf8(buf).unwrap();
243        assert!(output.contains("Search Suite"));
244        assert!(output.contains("Impact Suite"));
245        // Verify both sections appear and a blank line separates them
246        let search_pos = output.find("Search Suite").unwrap();
247        let impact_pos = output.find("Impact Suite").unwrap();
248        assert!(
249            search_pos < impact_pos,
250            "Search Suite should appear before Impact Suite"
251        );
252        // The blank line separator must exist somewhere between the two sections
253        assert!(
254            output.contains("\n\nImpact Suite"),
255            "expected blank line before Impact Suite"
256        );
257    }
258
259    #[test]
260    fn suite_result_table_format() {
261        let result = SuiteResult {
262            search: Some(sample_search()),
263            impact: Some(sample_impact()),
264        };
265        let mut buf = Vec::new();
266        result.fmt_table(&mut buf).unwrap();
267        let output = String::from_utf8(buf).unwrap();
268        assert!(output.contains("Suite   | Metric       | Value | Target | Status"));
269        assert!(output.contains("--------+--------------+-------+--------+-------"));
270        assert!(output.contains("Search  | MRR"));
271        assert!(output.contains("Search  | Precision@5"));
272        assert!(output.contains("Search  | Precision@10"));
273        assert!(output.contains("Impact  | Precision"));
274        assert!(output.contains("Impact  | Recall"));
275        assert!(output.contains("Impact  | F1"));
276    }
277
278    #[test]
279    fn suite_result_json_format() {
280        let result = SuiteResult {
281            search: Some(sample_search()),
282            impact: Some(sample_impact()),
283        };
284        let mut buf = Vec::new();
285        result.fmt_json(&mut buf).unwrap();
286        let output = String::from_utf8(buf).unwrap();
287        // Must be valid JSON
288        let parsed: serde_json::Value = serde_json::from_str(output.trim()).unwrap();
289        assert!(parsed.get("search").is_some());
290        assert!(parsed.get("impact").is_some());
291        let search = parsed.get("search").unwrap();
292        assert_eq!(search.get("mrr").unwrap().as_f64().unwrap(), 0.62);
293        assert_eq!(search.get("repos").unwrap().as_u64().unwrap(), 5);
294    }
295
296    #[test]
297    fn quality_gate_all_pass() {
298        let result = SuiteResult {
299            search: Some(sample_search()),
300            impact: Some(sample_impact()),
301        };
302        assert!(result.all_passed());
303    }
304
305    #[test]
306    fn quality_gate_mrr_fail() {
307        let mut search = sample_search();
308        search.mrr_passed = false;
309        let result = SuiteResult {
310            search: Some(search),
311            impact: Some(sample_impact()),
312        };
313        assert!(!result.all_passed());
314    }
315
316    #[test]
317    fn quality_gate_precision_fail() {
318        let mut impact = sample_impact();
319        impact.precision_passed = false;
320        let result = SuiteResult {
321            search: Some(sample_search()),
322            impact: Some(impact),
323        };
324        assert!(!result.all_passed());
325    }
326}
the_code_graph_eval/report.rs

the_code_graph_eval/
report.rs
