vtcode_eval/
report.rs

1use crate::{EvalMetric, task::EvalCategory};
2use serde::Serialize;
3
4#[derive(Debug, Clone, Serialize)]
5pub struct TaskReport {
6    pub task_id: String,
7    pub category: String,
8    pub metric: EvalMetric,
9}
10
11#[derive(Debug, Clone, Serialize)]
12pub struct SuiteReport {
13    pub suite_id: String,
14    pub suite_name: String,
15    pub task_reports: Vec<TaskReport>,
16    pub aggregate: EvalMetric,
17    pub capability_metrics: EvalMetric,
18    pub regression_metrics: EvalMetric,
19}
20
21#[derive(Debug, Clone, Serialize)]
22pub struct EvalReport {
23    pub generated_at: String,
24    pub suites: Vec<SuiteReport>,
25}
26
27impl EvalReport {
28    pub fn to_markdown(&self) -> String {
29        let mut out = String::new();
30        out.push_str("# Eval Report\n\n");
31        for s in &self.suites {
32            out.push_str(&format!("## {}\n\n", s.suite_name));
33            out.push_str(&format!("- Aggregate: pass@k={:.1}%\n", s.aggregate.pass_at_k * 100.0));
34            out.push_str("| Task | Category | pass@k | passed/total |\n");
35            out.push_str("|------|----------|--------|-------------|\n");
36            for t in &s.task_reports {
37                out.push_str(&format!(
38                    "| {} | {} | {:.1}% | {}/{} |\n",
39                    t.task_id,
40                    t.category.as_str(),
41                    t.metric.pass_at_k * 100.0,
42                    t.metric.passed_runs,
43                    t.metric.total_runs
44                ));
45            }
46            out.push('\n');
47        }
48        out
49    }
50}
51
52pub fn build_task_report(
53    task_id: &str,
54    _name: &str,
55    category: EvalCategory,
56    metric: EvalMetric,
57) -> TaskReport {
58    TaskReport {
59        task_id: task_id.into(),
60        category: category.label().into(),
61        metric,
62    }
63}
64
65#[cfg(test)]
66mod tests {
67    use super::*;
68    use crate::metric::EvalMetric;
69    use crate::task::EvalCategory;
70
71    #[test]
72    fn to_markdown_renders_tasks_and_aggregate() {
73        let report = EvalReport {
74            generated_at: "2026-01-01".into(),
75            suites: vec![SuiteReport {
76                suite_id: "s1".into(),
77                suite_name: "demo".into(),
78                task_reports: vec![TaskReport {
79                    task_id: "t1".into(),
80                    category: "Capability".into(),
81                    metric: EvalMetric {
82                        pass_at_k: 0.5,
83                        pass_all_k: 0.0,
84                        total_runs: 2,
85                        passed_runs: 1,
86                        task_id: "t1".into(),
87                    },
88                }],
89                aggregate: EvalMetric {
90                    pass_at_k: 0.5,
91                    pass_all_k: 0.0,
92                    total_runs: 2,
93                    passed_runs: 1,
94                    task_id: "aggregate".into(),
95                },
96                capability_metrics: EvalMetric {
97                    pass_at_k: 0.5,
98                    pass_all_k: 0.0,
99                    total_runs: 2,
100                    passed_runs: 1,
101                    task_id: "cap".into(),
102                },
103                regression_metrics: EvalMetric {
104                    pass_at_k: 0.0,
105                    pass_all_k: 0.0,
106                    total_runs: 0,
107                    passed_runs: 0,
108                    task_id: "reg".into(),
109                },
110            }],
111        };
112        let md = report.to_markdown();
113        assert!(md.contains("# Eval Report"));
114        assert!(md.contains("demo"));
115        assert!(md.contains("t1"));
116        assert!(md.contains("Capability"));
117    }
118
119    #[test]
120    fn build_task_report_maps_category() {
121        let tr = build_task_report(
122            "t1",
123            "name",
124            EvalCategory::Regression,
125            EvalMetric {
126                pass_at_k: 1.0,
127                pass_all_k: 1.0,
128                total_runs: 1,
129                passed_runs: 1,
130                task_id: "t1".into(),
131            },
132        );
133        assert_eq!(tr.task_id, "t1");
134        assert_eq!(tr.category, "Regression");
135    }
136}
vtcode_eval/report.rs

vtcode_eval/
report.rs
