vtcode_eval/
report.rs

1use crate::{EvalMetric, task::EvalCategory};
2use serde::Serialize;
3
4#[derive(Debug, Clone, Serialize)]
5pub struct TaskReport {
6    pub task_id: String,
7    pub category: String,
8    pub metric: EvalMetric,
9}
10
11#[derive(Debug, Clone, Serialize)]
12pub struct SuiteReport {
13    pub suite_id: String,
14    pub suite_name: String,
15    pub task_reports: Vec<TaskReport>,
16    pub aggregate: EvalMetric,
17    pub capability_metrics: EvalMetric,
18    pub regression_metrics: EvalMetric,
19}
20
21#[derive(Debug, Clone, Serialize)]
22pub struct EvalReport {
23    pub generated_at: String,
24    pub suites: Vec<SuiteReport>,
25}
26
27impl EvalReport {
28    pub fn to_markdown(&self) -> String {
29        let mut out = String::new();
30        out.push_str("# Eval Report\n\n");
31        for s in &self.suites {
32            out.push_str(&format!("## {}\n\n", s.suite_name));
33            out.push_str(&format!(
34                "- Aggregate: pass@k={:.1}%\n",
35                s.aggregate.pass_at_k * 100.0
36            ));
37            out.push_str("| Task | Category | pass@k | passed/total |\n");
38            out.push_str("|------|----------|--------|-------------|\n");
39            for t in &s.task_reports {
40                out.push_str(&format!(
41                    "| {} | {} | {:.1}% | {}/{} |\n",
42                    t.task_id,
43                    t.category.as_str(),
44                    t.metric.pass_at_k * 100.0,
45                    t.metric.passed_runs,
46                    t.metric.total_runs
47                ));
48            }
49            out.push('\n');
50        }
51        out
52    }
53}
54
55pub fn build_task_report(
56    task_id: &str,
57    _name: &str,
58    category: EvalCategory,
59    metric: EvalMetric,
60) -> TaskReport {
61    TaskReport {
62        task_id: task_id.into(),
63        category: category.label().into(),
64        metric,
65    }
66}
67
68#[cfg(test)]
69mod tests {
70    use super::*;
71    use crate::metric::EvalMetric;
72    use crate::task::EvalCategory;
73
74    #[test]
75    fn to_markdown_renders_tasks_and_aggregate() {
76        let report = EvalReport {
77            generated_at: "2026-01-01".into(),
78            suites: vec![SuiteReport {
79                suite_id: "s1".into(),
80                suite_name: "demo".into(),
81                task_reports: vec![TaskReport {
82                    task_id: "t1".into(),
83                    category: "Capability".into(),
84                    metric: EvalMetric {
85                        pass_at_k: 0.5,
86                        pass_all_k: 0.0,
87                        total_runs: 2,
88                        passed_runs: 1,
89                        task_id: "t1".into(),
90                    },
91                }],
92                aggregate: EvalMetric {
93                    pass_at_k: 0.5,
94                    pass_all_k: 0.0,
95                    total_runs: 2,
96                    passed_runs: 1,
97                    task_id: "aggregate".into(),
98                },
99                capability_metrics: EvalMetric {
100                    pass_at_k: 0.5,
101                    pass_all_k: 0.0,
102                    total_runs: 2,
103                    passed_runs: 1,
104                    task_id: "cap".into(),
105                },
106                regression_metrics: EvalMetric {
107                    pass_at_k: 0.0,
108                    pass_all_k: 0.0,
109                    total_runs: 0,
110                    passed_runs: 0,
111                    task_id: "reg".into(),
112                },
113            }],
114        };
115        let md = report.to_markdown();
116        assert!(md.contains("# Eval Report"));
117        assert!(md.contains("demo"));
118        assert!(md.contains("t1"));
119        assert!(md.contains("Capability"));
120    }
121
122    #[test]
123    fn build_task_report_maps_category() {
124        let tr = build_task_report(
125            "t1",
126            "name",
127            EvalCategory::Regression,
128            EvalMetric {
129                pass_at_k: 1.0,
130                pass_all_k: 1.0,
131                total_runs: 1,
132                passed_runs: 1,
133                task_id: "t1".into(),
134            },
135        );
136        assert_eq!(tr.task_id, "t1");
137        assert_eq!(tr.category, "Regression");
138    }
139}
vtcode_eval/report.rs

vtcode_eval/
report.rs
