harn_cli/
test_report.rs

1//! Machine-readable test reports for `harn test`.
2//!
3//! User and conformance suites both feed the same writer so the JUnit
4//! XML and `--json-out` payloads have a single source of truth. CI
5//! systems get a uniform schema; performance audits get per-file and
6//! per-test timing without scraping ANSI-coloured terminal output.
7//!
8//! The writers fail loudly: a missing or unwritable destination
9//! returns an error so the CLI can exit non-zero rather than silently
10//! succeed (issue #2146).
11
12use std::fs;
13use std::path::{Path, PathBuf};
14
15use serde::Serialize;
16
17pub const USER_TEST_REPORT_SCHEMA_VERSION: u32 = 1;
18
19#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
20#[serde(rename_all = "snake_case")]
21pub enum TestOutcome {
22    Passed,
23    Failed,
24    TimedOut,
25    Skipped,
26}
27
28impl TestOutcome {
29    fn is_failure(self) -> bool {
30        matches!(self, TestOutcome::Failed | TestOutcome::TimedOut)
31    }
32
33    fn is_skipped(self) -> bool {
34        matches!(self, TestOutcome::Skipped)
35    }
36}
37
38#[derive(Debug, Clone, Serialize)]
39pub struct TestCaseReport {
40    pub name: String,
41    pub file: String,
42    pub classname: String,
43    pub outcome: TestOutcome,
44    pub duration_ms: u64,
45    #[serde(skip_serializing_if = "Option::is_none")]
46    pub message: Option<String>,
47}
48
49#[derive(Debug, Clone, Default, Serialize)]
50pub struct TestReportSummary {
51    pub total: u64,
52    pub passed: u64,
53    pub failed: u64,
54    pub timed_out: u64,
55    pub skipped: u64,
56}
57
58impl TestReportSummary {
59    fn record(&mut self, outcome: TestOutcome) {
60        self.total += 1;
61        match outcome {
62            TestOutcome::Passed => self.passed += 1,
63            TestOutcome::Failed => self.failed += 1,
64            TestOutcome::TimedOut => self.timed_out += 1,
65            TestOutcome::Skipped => self.skipped += 1,
66        }
67    }
68}
69
70#[derive(Debug, Clone, Serialize)]
71pub struct TestReport {
72    #[serde(rename = "schemaVersion")]
73    pub schema_version: u32,
74    pub suite: String,
75    pub root: Option<String>,
76    pub duration_ms: u64,
77    pub summary: TestReportSummary,
78    pub cases: Vec<TestCaseReport>,
79}
80
81impl TestReport {
82    pub fn new(suite: impl Into<String>, root: Option<&Path>) -> Self {
83        Self {
84            schema_version: USER_TEST_REPORT_SCHEMA_VERSION,
85            suite: suite.into(),
86            root: root.map(|p| p.display().to_string()),
87            duration_ms: 0,
88            summary: TestReportSummary::default(),
89            cases: Vec::new(),
90        }
91    }
92
93    pub fn push(&mut self, case: TestCaseReport) {
94        self.summary.record(case.outcome);
95        self.cases.push(case);
96    }
97
98    pub fn set_duration_ms(&mut self, duration_ms: u64) {
99        self.duration_ms = duration_ms;
100    }
101}
102
103fn xml_escape(s: &str) -> String {
104    s.replace('&', "&amp;")
105        .replace('<', "&lt;")
106        .replace('>', "&gt;")
107        .replace('"', "&quot;")
108        .replace('\'', "&apos;")
109}
110
111fn ensure_parent_writable(path: &Path) -> Result<(), String> {
112    let parent = path.parent().filter(|p| !p.as_os_str().is_empty());
113    if let Some(parent) = parent {
114        if !parent.exists() {
115            return Err(format!(
116                "report directory does not exist: {}",
117                parent.display()
118            ));
119        }
120        if !parent.is_dir() {
121            return Err(format!(
122                "report directory is not a directory: {}",
123                parent.display()
124            ));
125        }
126    }
127    Ok(())
128}
129
130pub fn write_junit(path: &str, report: &TestReport) -> Result<PathBuf, String> {
131    let path_buf = PathBuf::from(path);
132    ensure_parent_writable(&path_buf)?;
133
134    let suite_time = report.duration_ms as f64 / 1000.0;
135    let suite_name = xml_escape(&report.suite);
136    let tests = report.summary.total;
137    let failures = report.summary.failed + report.summary.timed_out;
138    let skipped = report.summary.skipped;
139
140    let mut xml = String::new();
141    xml.push_str("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
142    xml.push_str(&format!(
143        "<testsuites name=\"{suite_name}\" tests=\"{tests}\" failures=\"{failures}\" skipped=\"{skipped}\" time=\"{suite_time:.3}\">\n"
144    ));
145    xml.push_str(&format!(
146        "  <testsuite name=\"{suite_name}\" tests=\"{tests}\" failures=\"{failures}\" skipped=\"{skipped}\" time=\"{suite_time:.3}\">\n"
147    ));
148    for case in &report.cases {
149        let time = case.duration_ms as f64 / 1000.0;
150        let escaped_name = xml_escape(&case.name);
151        let escaped_classname = xml_escape(&case.classname);
152        let escaped_file = xml_escape(&case.file);
153        xml.push_str(&format!(
154            "    <testcase name=\"{escaped_name}\" classname=\"{escaped_classname}\" file=\"{escaped_file}\" time=\"{time:.3}\""
155        ));
156        if case.outcome == TestOutcome::Passed {
157            xml.push_str(" />\n");
158            continue;
159        }
160        xml.push_str(">\n");
161        let body = case.message.as_deref().unwrap_or_default();
162        let escaped_body = xml_escape(body);
163        if case.outcome.is_skipped() {
164            xml.push_str(&format!("      <skipped message=\"{escaped_body}\" />\n"));
165        } else if case.outcome.is_failure() {
166            let kind = if matches!(case.outcome, TestOutcome::TimedOut) {
167                "timeout"
168            } else {
169                "AssertionError"
170            };
171            xml.push_str(&format!(
172                "      <failure type=\"{kind}\" message=\"test failed\">{escaped_body}</failure>\n"
173            ));
174        }
175        xml.push_str("    </testcase>\n");
176    }
177    xml.push_str("  </testsuite>\n");
178    xml.push_str("</testsuites>\n");
179
180    fs::write(&path_buf, &xml).map_err(|error| {
181        format!(
182            "failed to write JUnit XML to {}: {error}",
183            path_buf.display()
184        )
185    })?;
186    Ok(path_buf)
187}
188
189pub fn write_json(path: &str, report: &TestReport) -> Result<PathBuf, String> {
190    let path_buf = PathBuf::from(path);
191    ensure_parent_writable(&path_buf)?;
192    let rendered = serde_json::to_string_pretty(report)
193        .map_err(|error| format!("failed to serialize test report JSON: {error}"))?;
194    fs::write(&path_buf, rendered).map_err(|error| {
195        format!(
196            "failed to write JSON report to {}: {error}",
197            path_buf.display()
198        )
199    })?;
200    Ok(path_buf)
201}
202
203#[cfg(test)]
204mod tests {
205    use super::*;
206
207    fn sample_report() -> TestReport {
208        let mut report = TestReport::new("user", None);
209        report.push(TestCaseReport {
210            name: "test_alpha".into(),
211            file: "suite/a.harn".into(),
212            classname: "suite/a.harn".into(),
213            outcome: TestOutcome::Passed,
214            duration_ms: 12,
215            message: None,
216        });
217        report.push(TestCaseReport {
218            name: "test_beta".into(),
219            file: "suite/b.harn".into(),
220            classname: "suite/b.harn".into(),
221            outcome: TestOutcome::Failed,
222            duration_ms: 34,
223            message: Some("expected 1 == 2".into()),
224        });
225        report.push(TestCaseReport {
226            name: "test_gamma".into(),
227            file: "suite/c.harn".into(),
228            classname: "suite/c.harn".into(),
229            outcome: TestOutcome::TimedOut,
230            duration_ms: 30_000,
231            message: Some("timed out after 30000ms".into()),
232        });
233        report.push(TestCaseReport {
234            name: "test_delta".into(),
235            file: "suite/d.harn".into(),
236            classname: "suite/d.harn".into(),
237            outcome: TestOutcome::Skipped,
238            duration_ms: 0,
239            message: Some("xfail: flaky".into()),
240        });
241        report.set_duration_ms(100);
242        report
243    }
244
245    #[test]
246    fn summary_counts_outcomes() {
247        let report = sample_report();
248        assert_eq!(report.summary.total, 4);
249        assert_eq!(report.summary.passed, 1);
250        assert_eq!(report.summary.failed, 1);
251        assert_eq!(report.summary.timed_out, 1);
252        assert_eq!(report.summary.skipped, 1);
253    }
254
255    #[test]
256    fn write_junit_renders_failure_and_skip() {
257        let temp = tempfile::tempdir().unwrap();
258        let path = temp.path().join("report.xml");
259        let report = sample_report();
260        write_junit(path.to_str().unwrap(), &report).unwrap();
261        let xml = std::fs::read_to_string(&path).unwrap();
262        assert!(xml.contains("<testsuites"));
263        assert!(xml.contains(r#"tests="4" failures="2" skipped="1""#));
264        assert!(xml.contains(r#"name="test_alpha""#));
265        assert!(xml.contains(r#"<failure type="AssertionError""#));
266        assert!(xml.contains(r#"<failure type="timeout""#));
267        assert!(xml.contains("<skipped"));
268    }
269
270    #[test]
271    fn write_json_round_trips_through_serde() {
272        let temp = tempfile::tempdir().unwrap();
273        let path = temp.path().join("report.json");
274        let report = sample_report();
275        write_json(path.to_str().unwrap(), &report).unwrap();
276        let text = std::fs::read_to_string(&path).unwrap();
277        let value: serde_json::Value = serde_json::from_str(&text).unwrap();
278        assert_eq!(value["schemaVersion"], USER_TEST_REPORT_SCHEMA_VERSION);
279        assert_eq!(value["summary"]["total"], 4);
280        assert_eq!(value["summary"]["passed"], 1);
281        assert_eq!(value["summary"]["failed"], 1);
282        assert_eq!(value["summary"]["timed_out"], 1);
283        assert_eq!(value["summary"]["skipped"], 1);
284        let cases = value["cases"].as_array().unwrap();
285        assert_eq!(cases.len(), 4);
286        assert_eq!(cases[0]["outcome"], "passed");
287        assert_eq!(cases[1]["outcome"], "failed");
288        assert_eq!(cases[2]["outcome"], "timed_out");
289        assert_eq!(cases[3]["outcome"], "skipped");
290    }
291
292    #[test]
293    fn missing_parent_directory_returns_error() {
294        let temp = tempfile::tempdir().unwrap();
295        let path = temp.path().join("does/not/exist/report.xml");
296        let err = write_junit(path.to_str().unwrap(), &sample_report()).unwrap_err();
297        assert!(
298            err.contains("report directory does not exist"),
299            "unexpected error: {err}"
300        );
301        let err = write_json(path.to_str().unwrap(), &sample_report()).unwrap_err();
302        assert!(
303            err.contains("report directory does not exist"),
304            "unexpected error: {err}"
305        );
306    }
307
308    #[test]
309    fn parent_must_be_a_directory() {
310        let temp = tempfile::tempdir().unwrap();
311        let parent = temp.path().join("notadir");
312        std::fs::write(&parent, "x").unwrap();
313        let path = parent.join("report.xml");
314        let err = write_junit(path.to_str().unwrap(), &sample_report()).unwrap_err();
315        assert!(
316            err.contains("is not a directory"),
317            "unexpected error: {err}"
318        );
319    }
320}
harn_cli/test_report.rs

harn_cli/
test_report.rs
