Skip to main content

visual_rubric/
report.rs

1//! Page-level and aggregate visual rubric reports.
2
3use std::collections::BTreeMap;
4use std::fmt::Write as _;
5use std::path::{Path, PathBuf};
6use std::time::{SystemTime, UNIX_EPOCH};
7
8use serde::Serialize;
9
10use crate::RubricVerdict;
11
12/// Result from evaluating one page screenshot.
13#[derive(Clone, Debug, Serialize)]
14pub struct PageResult {
15    /// Human-readable label (e.g. `search`, `entity_plan`).
16    pub label: String,
17    /// Route or URL that was screenshotted.
18    pub route: Option<String>,
19    /// Viewport dimensions (width, height) when set.
20    pub viewport: Option<(u64, u64)>,
21    /// Absolute path to the saved PNG screenshot.
22    pub screenshot_path: PathBuf,
23    /// Text description produced by the vision model.
24    pub vision_description: String,
25    /// Rubric verdict from the ACP model.
26    pub verdict: RubricVerdict,
27}
28
29/// LLM-optimized report aggregating multiple page evaluations.
30#[derive(Clone, Debug, Serialize)]
31pub struct RubricReport {
32    /// ISO 8601 timestamp when the report was generated.
33    pub generated_at: String,
34    /// Project-level description context.
35    pub context: String,
36    /// Sorted list of page-level results.
37    pub pages: Vec<PageResult>,
38    /// Number of passed pages.
39    pub passed: usize,
40    /// Number of failed pages.
41    pub failed: usize,
42    /// Total pages evaluated.
43    pub total: usize,
44}
45
46impl RubricReport {
47    /// Build a report from page results and render it.
48    #[must_use]
49    pub fn from_pages(context: &str, pages: Vec<PageResult>) -> Self {
50        let total = pages.len();
51        let passed = pages.iter().filter(|p| p.verdict.verdict.is_pass()).count();
52        let failed = total - passed;
53        let generated_at = {
54            let dur = SystemTime::now()
55                .duration_since(UNIX_EPOCH)
56                .unwrap_or_default();
57            dur.as_secs().to_string()
58        };
59        Self {
60            generated_at,
61            context: context.to_string(),
62            pages,
63            passed,
64            failed,
65            total,
66        }
67    }
68
69    /// Render the report as LLM-optimized structured markdown.
70    #[must_use]
71    pub fn to_markdown(&self) -> String {
72        let mut md = String::with_capacity(1024 + self.pages.len() * 512);
73        md.push_str("# Visual Rubric Report\n\n");
74        let _ = write!(md, "Generated: {}\n\n", self.generated_at);
75        let _ = write!(md, "{}\n\n", self.context);
76        md.push_str("## Summary\n\n");
77        let _ = writeln!(md, "- **Total pages**: {}", self.total);
78        let _ = writeln!(md, "- **Passed**: {}", self.passed);
79        let _ = writeln!(md, "- **Failed**: {}", self.failed);
80        let _ = write!(
81            md,
82            "- **Pass rate**: {}%\n\n",
83            if self.total > 0 {
84                (self.passed * 100) / self.total
85            } else {
86                0
87            }
88        );
89
90        self.push_defect_patterns(&mut md);
91        self.push_page_details(&mut md);
92        md
93    }
94
95    fn push_defect_patterns(&self, md: &mut String) {
96        let mut anomaly_counts: BTreeMap<String, Vec<&str>> = BTreeMap::new();
97        for page in &self.pages {
98            for anomaly in &page.verdict.anomalies {
99                let key = anomaly.split('.').next().unwrap_or(anomaly).to_string();
100                anomaly_counts.entry(key).or_default().push(&page.label);
101            }
102        }
103        if anomaly_counts.is_empty() {
104            return;
105        }
106
107        md.push_str("### Defect Patterns\n\n");
108        md.push_str("| Pattern | Count | Pages |\n");
109        md.push_str("|---------|-------|-------|\n");
110        for (pattern, labels) in &anomaly_counts {
111            let _ = write!(md, "| {} | {} | ", pattern, labels.len());
112            for (index, label) in labels.iter().enumerate() {
113                if index > 0 {
114                    md.push_str(", ");
115                }
116                let _ = write!(md, "`{label}`");
117            }
118            md.push_str(" |\n");
119        }
120        md.push('\n');
121    }
122
123    fn push_page_details(&self, md: &mut String) {
124        for page in &self.pages {
125            let status = if page.verdict.verdict.is_pass() {
126                "PASS"
127            } else {
128                "FAIL"
129            };
130            let _ = write!(md, "---\n\n### {}: {}\n\n", status, page.label);
131            if let Some(ref route) = page.route {
132                let _ = write!(md, "**Route:** `{route}`\n\n");
133            }
134            let _ = write!(
135                md,
136                "**Screenshot:** `{}`\n\n",
137                page.screenshot_path.display()
138            );
139            if let Some((w, h)) = page.viewport {
140                let _ = write!(md, "**Viewport:** {w}×{h}\n\n");
141            }
142            md.push_str("**Vision Description:**\n\n");
143            let _ = write!(
144                md,
145                "> {}\n\n",
146                page.vision_description.replace('\n', "\n> ")
147            );
148            let _ = write!(md, "**Rubric Result:** {status}\n\n");
149            let _ = write!(md, "**Reason:** {}\n\n", page.verdict.reason);
150            if !page.verdict.anomalies.is_empty() {
151                md.push_str("**Anomalies:**\n");
152                for anomaly in &page.verdict.anomalies {
153                    let _ = writeln!(md, "- {anomaly}");
154                }
155                md.push('\n');
156            }
157        }
158    }
159
160    /// Render the report as JSON.
161    pub fn to_json(&self) -> serde_json::Result<String> {
162        serde_json::to_string_pretty(self)
163    }
164
165    /// Write markdown report to `path`.
166    pub fn save_markdown(&self, path: &Path) -> std::io::Result<()> {
167        std::fs::write(path, self.to_markdown())
168    }
169
170    /// Write JSON report to `path`.
171    pub fn save_json(&self, path: &Path) -> std::io::Result<()> {
172        let json = self
173            .to_json()
174            .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?;
175        std::fs::write(path, json)
176    }
177}