harn_cli/
test_report.rs

1//! Machine-readable test reports for `harn test`.
2//!
3//! User and conformance suites both feed the same writer so the JUnit
4//! XML and `--json-out` payloads have a single source of truth. CI
5//! systems get a uniform schema; performance audits get per-file and
6//! per-test timing without scraping ANSI-coloured terminal output.
7//!
8//! The writers fail loudly: a missing or unwritable destination
9//! returns an error so the CLI can exit non-zero rather than silently
10//! succeed (issue #2146).
11
12use std::fs;
13use std::path::{Path, PathBuf};
14
15use serde::Serialize;
16
17use crate::test_runner::{AggregateTimings, PhaseTimings, TestTimeout};
18use crate::test_timing::DurationSummary;
19
20pub const USER_TEST_REPORT_SCHEMA_VERSION: u32 = 3;
21
22#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
23#[serde(rename_all = "snake_case")]
24pub enum TestOutcome {
25    Passed,
26    Failed,
27    TimedOut,
28    Skipped,
29}
30
31impl TestOutcome {
32    fn is_failure(self) -> bool {
33        matches!(self, TestOutcome::Failed | TestOutcome::TimedOut)
34    }
35
36    fn is_skipped(self) -> bool {
37        matches!(self, TestOutcome::Skipped)
38    }
39}
40
41#[derive(Debug, Clone, Serialize)]
42pub struct TestCaseReport {
43    pub name: String,
44    pub file: String,
45    pub classname: String,
46    pub outcome: TestOutcome,
47    pub duration_ms: u64,
48    #[serde(skip_serializing_if = "Option::is_none")]
49    pub timeout: Option<TestTimeout>,
50    #[serde(skip_serializing_if = "Option::is_none")]
51    pub phases: Option<PhaseTimings>,
52    #[serde(skip_serializing_if = "Option::is_none")]
53    pub message: Option<String>,
54    /// Everything the case wrote via `log`/`print`/`println`/etc, present
55    /// regardless of outcome — a passing case's probes are as valid a
56    /// reason to want this in a JSON/JUnit consumer as a failing case's.
57    #[serde(skip_serializing_if = "Option::is_none")]
58    pub captured_output: Option<String>,
59}
60
61#[derive(Debug, Clone, Default, Serialize)]
62pub struct TestReportSummary {
63    pub total: u64,
64    pub passed: u64,
65    pub failed: u64,
66    pub timed_out: u64,
67    pub skipped: u64,
68}
69
70impl TestReportSummary {
71    fn record(&mut self, outcome: TestOutcome) {
72        self.total += 1;
73        match outcome {
74            TestOutcome::Passed => self.passed += 1,
75            TestOutcome::Failed => self.failed += 1,
76            TestOutcome::TimedOut => self.timed_out += 1,
77            TestOutcome::Skipped => self.skipped += 1,
78        }
79    }
80}
81
82#[derive(Debug, Clone, Serialize)]
83pub struct TestReport {
84    #[serde(rename = "schemaVersion")]
85    pub schema_version: u32,
86    pub suite: String,
87    pub root: Option<String>,
88    pub duration_ms: u64,
89    #[serde(skip_serializing_if = "Option::is_none")]
90    pub timing: Option<DurationSummary>,
91    #[serde(skip_serializing_if = "Option::is_none")]
92    pub aggregate: Option<AggregateTimings>,
93    pub summary: TestReportSummary,
94    pub cases: Vec<TestCaseReport>,
95}
96
97impl TestReport {
98    pub fn new(suite: impl Into<String>, root: Option<&Path>) -> Self {
99        Self {
100            schema_version: USER_TEST_REPORT_SCHEMA_VERSION,
101            suite: suite.into(),
102            root: root.map(|p| p.display().to_string()),
103            duration_ms: 0,
104            timing: None,
105            aggregate: None,
106            summary: TestReportSummary::default(),
107            cases: Vec::new(),
108        }
109    }
110
111    pub fn set_execution_metrics(&mut self, timing: DurationSummary, aggregate: AggregateTimings) {
112        self.timing = Some(timing);
113        self.aggregate = Some(aggregate);
114    }
115
116    pub fn push(&mut self, case: TestCaseReport) {
117        self.summary.record(case.outcome);
118        self.cases.push(case);
119    }
120
121    pub fn set_duration_ms(&mut self, duration_ms: u64) {
122        self.duration_ms = duration_ms;
123    }
124}
125
126fn ensure_parent_writable(path: &Path) -> Result<(), String> {
127    let parent = path.parent().filter(|p| !p.as_os_str().is_empty());
128    if let Some(parent) = parent {
129        if !parent.exists() {
130            return Err(format!(
131                "report directory does not exist: {}",
132                parent.display()
133            ));
134        }
135        if !parent.is_dir() {
136            return Err(format!(
137                "report directory is not a directory: {}",
138                parent.display()
139            ));
140        }
141    }
142    Ok(())
143}
144
145pub fn write_junit(path: &str, report: &TestReport) -> Result<PathBuf, String> {
146    let path_buf = PathBuf::from(path);
147    ensure_parent_writable(&path_buf)?;
148
149    let suite_time = report.duration_ms as f64 / 1000.0;
150    let suite_name = crate::format::escape_html(&report.suite);
151    let tests = report.summary.total;
152    let failures = report.summary.failed + report.summary.timed_out;
153    let skipped = report.summary.skipped;
154
155    let mut xml = String::new();
156    xml.push_str("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
157    xml.push_str(&format!(
158        "<testsuites name=\"{suite_name}\" tests=\"{tests}\" failures=\"{failures}\" skipped=\"{skipped}\" time=\"{suite_time:.3}\">\n"
159    ));
160    xml.push_str(&format!(
161        "  <testsuite name=\"{suite_name}\" tests=\"{tests}\" failures=\"{failures}\" skipped=\"{skipped}\" time=\"{suite_time:.3}\">\n"
162    ));
163    for case in &report.cases {
164        let time = case.duration_ms as f64 / 1000.0;
165        let escaped_name = crate::format::escape_html(&case.name);
166        let escaped_classname = crate::format::escape_html(&case.classname);
167        let escaped_file = crate::format::escape_html(&case.file);
168        xml.push_str(&format!(
169            "    <testcase name=\"{escaped_name}\" classname=\"{escaped_classname}\" file=\"{escaped_file}\" time=\"{time:.3}\""
170        ));
171        if case.outcome == TestOutcome::Passed {
172            xml.push_str(" />\n");
173            continue;
174        }
175        xml.push_str(">\n");
176        let body = case.message.as_deref().unwrap_or_default();
177        let escaped_body = crate::format::escape_html(body);
178        if case.outcome.is_skipped() {
179            xml.push_str(&format!("      <skipped message=\"{escaped_body}\" />\n"));
180        } else if case.outcome.is_failure() {
181            let kind = if matches!(case.outcome, TestOutcome::TimedOut) {
182                "timeout"
183            } else {
184                "AssertionError"
185            };
186            xml.push_str(&format!(
187                "      <failure type=\"{kind}\" message=\"test failed\">{escaped_body}</failure>\n"
188            ));
189            if let Some(output) = &case.captured_output {
190                let escaped_output = crate::format::escape_html(output);
191                xml.push_str(&format!(
192                    "      <system-out>{escaped_output}</system-out>\n"
193                ));
194            }
195        }
196        xml.push_str("    </testcase>\n");
197    }
198    xml.push_str("  </testsuite>\n");
199    xml.push_str("</testsuites>\n");
200
201    fs::write(&path_buf, &xml).map_err(|error| {
202        format!(
203            "failed to write JUnit XML to {}: {error}",
204            path_buf.display()
205        )
206    })?;
207    Ok(path_buf)
208}
209
210pub fn write_json(path: &str, report: &TestReport) -> Result<PathBuf, String> {
211    let path_buf = PathBuf::from(path);
212    ensure_parent_writable(&path_buf)?;
213    let rendered = serde_json::to_string_pretty(report)
214        .map_err(|error| format!("failed to serialize test report JSON: {error}"))?;
215    fs::write(&path_buf, rendered).map_err(|error| {
216        format!(
217            "failed to write JSON report to {}: {error}",
218            path_buf.display()
219        )
220    })?;
221    Ok(path_buf)
222}
223
224#[cfg(test)]
225mod tests {
226    use super::*;
227
228    fn sample_report() -> TestReport {
229        let mut report = TestReport::new("user", None);
230        report.push(TestCaseReport {
231            name: "test_alpha".into(),
232            file: "suite/a.harn".into(),
233            classname: "suite/a.harn".into(),
234            outcome: TestOutcome::Passed,
235            duration_ms: 12,
236            timeout: None,
237            phases: None,
238            message: None,
239            captured_output: None,
240        });
241        report.push(TestCaseReport {
242            name: "test_beta".into(),
243            file: "suite/b.harn".into(),
244            classname: "suite/b.harn".into(),
245            outcome: TestOutcome::Failed,
246            duration_ms: 34,
247            timeout: None,
248            phases: None,
249            message: Some("expected 1 == 2".into()),
250            captured_output: Some("[harn] probing state\n".into()),
251        });
252        report.push(TestCaseReport {
253            name: "test_gamma".into(),
254            file: "suite/c.harn".into(),
255            classname: "suite/c.harn".into(),
256            outcome: TestOutcome::TimedOut,
257            duration_ms: 30_000,
258            timeout: Some(TestTimeout {
259                phase: crate::test_runner::TestPhase::Execute,
260                limit_ms: 30_000,
261            }),
262            phases: Some(PhaseTimings {
263                execute_ms: 30_000,
264                ..PhaseTimings::default()
265            }),
266            message: Some("timed out after 30000ms".into()),
267            captured_output: None,
268        });
269        report.push(TestCaseReport {
270            name: "test_delta".into(),
271            file: "suite/d.harn".into(),
272            classname: "suite/d.harn".into(),
273            outcome: TestOutcome::Skipped,
274            duration_ms: 0,
275            timeout: None,
276            phases: None,
277            message: Some("xfail: flaky".into()),
278            captured_output: None,
279        });
280        report.set_duration_ms(100);
281        report
282    }
283
284    #[test]
285    fn summary_counts_outcomes() {
286        let report = sample_report();
287        assert_eq!(report.summary.total, 4);
288        assert_eq!(report.summary.passed, 1);
289        assert_eq!(report.summary.failed, 1);
290        assert_eq!(report.summary.timed_out, 1);
291        assert_eq!(report.summary.skipped, 1);
292    }
293
294    #[test]
295    fn write_junit_renders_failure_and_skip() {
296        let temp = tempfile::tempdir().unwrap();
297        let path = temp.path().join("report.xml");
298        let report = sample_report();
299        write_junit(path.to_str().unwrap(), &report).unwrap();
300        let xml = std::fs::read_to_string(&path).unwrap();
301        assert!(xml.contains("<testsuites"));
302        assert!(xml.contains(r#"tests="4" failures="2" skipped="1""#));
303        assert!(xml.contains(r#"name="test_alpha""#));
304        assert!(xml.contains(r#"<failure type="AssertionError""#));
305        assert!(xml.contains(r#"<failure type="timeout""#));
306        assert!(xml.contains("<skipped"));
307        assert!(xml.contains("<system-out>[harn] probing state\n</system-out>"));
308        // The timed-out case carries no captured output in this fixture;
309        // its `<testcase>` must not grow a stray `<system-out>`.
310        assert_eq!(xml.matches("<system-out>").count(), 1);
311    }
312
313    #[test]
314    fn write_json_round_trips_through_serde() {
315        let temp = tempfile::tempdir().unwrap();
316        let path = temp.path().join("report.json");
317        let report = sample_report();
318        write_json(path.to_str().unwrap(), &report).unwrap();
319        let text = std::fs::read_to_string(&path).unwrap();
320        let value: serde_json::Value = serde_json::from_str(&text).unwrap();
321        assert_eq!(value["schemaVersion"], USER_TEST_REPORT_SCHEMA_VERSION);
322        assert_eq!(value["summary"]["total"], 4);
323        assert_eq!(value["summary"]["passed"], 1);
324        assert_eq!(value["summary"]["failed"], 1);
325        assert_eq!(value["summary"]["timed_out"], 1);
326        assert_eq!(value["summary"]["skipped"], 1);
327        let cases = value["cases"].as_array().unwrap();
328        assert_eq!(cases.len(), 4);
329        assert_eq!(cases[0]["outcome"], "passed");
330        assert_eq!(cases[1]["outcome"], "failed");
331        assert_eq!(cases[1]["captured_output"], "[harn] probing state\n");
332        assert_eq!(cases[2]["outcome"], "timed_out");
333        assert_eq!(cases[2]["timeout"]["phase"], "execute");
334        assert_eq!(cases[2]["timeout"]["limit_ms"], 30_000);
335        assert_eq!(cases[2]["phases"]["execute_ms"], 30_000);
336        assert_eq!(cases[3]["outcome"], "skipped");
337    }
338
339    #[test]
340    fn missing_parent_directory_returns_error() {
341        let temp = tempfile::tempdir().unwrap();
342        let path = temp.path().join("does/not/exist/report.xml");
343        let err = write_junit(path.to_str().unwrap(), &sample_report()).unwrap_err();
344        assert!(
345            err.contains("report directory does not exist"),
346            "unexpected error: {err}"
347        );
348        let err = write_json(path.to_str().unwrap(), &sample_report()).unwrap_err();
349        assert!(
350            err.contains("report directory does not exist"),
351            "unexpected error: {err}"
352        );
353    }
354
355    #[test]
356    fn parent_must_be_a_directory() {
357        let temp = tempfile::tempdir().unwrap();
358        let parent = temp.path().join("notadir");
359        std::fs::write(&parent, "x").unwrap();
360        let path = parent.join("report.xml");
361        let err = write_junit(path.to_str().unwrap(), &sample_report()).unwrap_err();
362        assert!(
363            err.contains("is not a directory"),
364            "unexpected error: {err}"
365        );
366    }
367}
harn_cli/test_report.rs

harn_cli/
test_report.rs
