vtcode_eval/
report.rs

1use crate::{EvalMetric, task::EvalCategory};
2use serde::Serialize;
3
4#[derive(Debug, Clone, Serialize)]
5pub struct TaskReport {
6    task_id: String,
7    pub(crate) category: String,
8    pub(crate) metric: EvalMetric,
9}
10
11#[derive(Debug, Clone, Serialize)]
12pub struct SuiteReport {
13    pub(crate) suite_id: String,
14    pub(crate) suite_name: String,
15    pub(crate) task_reports: Vec<TaskReport>,
16    pub(crate) aggregate: EvalMetric,
17    pub(crate) capability_metrics: EvalMetric,
18    pub(crate) regression_metrics: EvalMetric,
19}
20
21#[derive(Debug, Clone, Serialize)]
22pub struct EvalReport {
23    pub(crate) generated_at: String,
24    pub(crate) suites: Vec<SuiteReport>,
25}
26
27impl EvalReport {
28    pub fn to_markdown(&self) -> String {
29        let mut out = String::new();
30        out.push_str("# Eval Report\n\n");
31        for s in &self.suites {
32            out.push_str(&format!("## {}\n\n", s.suite_name));
33            out.push_str(&format!("- Aggregate: pass@k={:.1}%\n", s.aggregate.pass_at_k * 100.0));
34            out.push_str("| Task | Category | pass@k | passed/total |\n");
35            out.push_str("|------|----------|--------|-------------|\n");
36            for t in &s.task_reports {
37                out.push_str(&format!(
38                    "| {} | {} | {:.1}% | {}/{} |\n",
39                    t.task_id,
40                    t.category.as_str(),
41                    t.metric.pass_at_k * 100.0,
42                    t.metric.passed_runs,
43                    t.metric.total_runs
44                ));
45            }
46            out.push('\n');
47        }
48        out
49    }
50}
51
52pub fn build_task_report(task_id: &str, _name: &str, category: EvalCategory, metric: EvalMetric) -> TaskReport {
53    TaskReport {
54        task_id: task_id.into(),
55        category: category.label().into(),
56        metric,
57    }
58}
59
60#[cfg(test)]
61mod tests {
62    use super::*;
63    use crate::metric::EvalMetric;
64    use crate::task::EvalCategory;
65
66    #[test]
67    fn to_markdown_renders_tasks_and_aggregate() {
68        let report = EvalReport {
69            generated_at: "2026-01-01".into(),
70            suites: vec![SuiteReport {
71                suite_id: "s1".into(),
72                suite_name: "demo".into(),
73                task_reports: vec![TaskReport {
74                    task_id: "t1".into(),
75                    category: "Capability".into(),
76                    metric: EvalMetric {
77                        pass_at_k: 0.5,
78                        pass_all_k: 0.0,
79                        total_runs: 2,
80                        passed_runs: 1,
81                        task_id: "t1".into(),
82                    },
83                }],
84                aggregate: EvalMetric {
85                    pass_at_k: 0.5,
86                    pass_all_k: 0.0,
87                    total_runs: 2,
88                    passed_runs: 1,
89                    task_id: "aggregate".into(),
90                },
91                capability_metrics: EvalMetric {
92                    pass_at_k: 0.5,
93                    pass_all_k: 0.0,
94                    total_runs: 2,
95                    passed_runs: 1,
96                    task_id: "cap".into(),
97                },
98                regression_metrics: EvalMetric {
99                    pass_at_k: 0.0,
100                    pass_all_k: 0.0,
101                    total_runs: 0,
102                    passed_runs: 0,
103                    task_id: "reg".into(),
104                },
105            }],
106        };
107        let md = report.to_markdown();
108        assert!(md.contains("# Eval Report"));
109        assert!(md.contains("demo"));
110        assert!(md.contains("t1"));
111        assert!(md.contains("Capability"));
112    }
113
114    #[test]
115    fn build_task_report_maps_category() {
116        let tr = build_task_report(
117            "t1",
118            "name",
119            EvalCategory::Regression,
120            EvalMetric {
121                pass_at_k: 1.0,
122                pass_all_k: 1.0,
123                total_runs: 1,
124                passed_runs: 1,
125                task_id: "t1".into(),
126            },
127        );
128        assert_eq!(tr.task_id, "t1");
129        assert_eq!(tr.category, "Regression");
130    }
131}
vtcode_eval/report.rs

vtcode_eval/
report.rs
