harn_cli/test_runner/
reporting.rs

1//! Test result contracts, timing rollups, and diagnostic rendering.
2
3use serde::Serialize;
4
5use crate::test_timing::DurationSummary;
6
7#[derive(Clone, Debug, Serialize)]
8pub struct TestResult {
9    pub name: String,
10    pub file: String,
11    pub passed: bool,
12    pub error: Option<String>,
13    /// Everything the case wrote via `log`/`print`/`println`/etc, in
14    /// execution order. `None` when nothing was written — keeps quiet,
15    /// passing cases from padding reports. Discovery and worker-start
16    /// errors never reach a VM and always leave this absent.
17    #[serde(skip_serializing_if = "Option::is_none")]
18    pub captured_output: Option<String>,
19    /// Typed timeout metadata. Consumers must use this instead of parsing
20    /// human-readable error text.
21    pub timeout: Option<TestTimeout>,
22    pub duration_ms: u64,
23    /// Per-phase timings for an executed case. Discovery and worker-start
24    /// errors have no execution timeline and leave this absent.
25    #[serde(skip_serializing_if = "Option::is_none")]
26    pub phases: Option<PhaseTimings>,
27}
28
29#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
30pub struct TestTimeout {
31    pub phase: TestPhase,
32    pub limit_ms: u64,
33}
34
35#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
36#[serde(rename_all = "snake_case")]
37pub enum TestPhase {
38    Execute,
39}
40
41#[derive(Clone, Debug, Serialize)]
42pub struct TestSummary {
43    pub results: Vec<TestResult>,
44    pub passed: usize,
45    pub failed: usize,
46    pub total: usize,
47    pub duration_ms: u64,
48    /// Distribution of per-test wall-clock durations.
49    pub timing: DurationSummary,
50    /// Aggregated phase costs across the entire run.
51    pub aggregate: AggregateTimings,
52}
53
54/// Wall-clock cost of each phase of a single test execution.
55///
56/// Sums to the test's `duration_ms` modulo measurement overhead. Surfaced
57/// so consumers can attribute cold-start vs assertion cost without
58/// having to instrument the runner externally.
59#[derive(Debug, Default, Clone, Copy, Serialize)]
60pub struct PhaseTimings {
61    /// VM construction + stdlib/hostlib registration + skill install +
62    /// runtime extension install + manifest hooks/triggers install.
63    pub setup_ms: u64,
64    /// `Compiler::compile_named` time for this test's chunk.
65    pub compile_ms: u64,
66    /// `vm.execute(chunk)` wall time, i.e. the actual user-test body.
67    pub execute_ms: u64,
68    /// VM/LocalSet task cancellation and `reset_thread_local_state` between tests.
69    pub teardown_ms: u64,
70    /// Module attribution overlapping setup and execute. These values are
71    /// diagnostic subtotals and must not be added to the top-level phases.
72    pub modules: harn_vm::ModulePhaseStats,
73}
74
75/// Cumulative worker-time across the run. Mirrors [`PhaseTimings`] plus the
76/// suite-level collection cost (discover + parse). Parallel case phases overlap,
77/// so these totals may exceed suite wall time.
78#[derive(Debug, Default, Clone, Copy, Serialize)]
79pub struct AggregateTimings {
80    pub collection_ms: u64,
81    pub setup_ms: u64,
82    pub compile_ms: u64,
83    pub execute_ms: u64,
84    pub teardown_ms: u64,
85    /// Sum of per-case module attribution. Overlaps setup and execute.
86    pub modules: harn_vm::ModulePhaseStats,
87}
88
89impl AggregateTimings {
90    pub(super) fn from_results(collection_ms: u64, results: &[TestResult]) -> Self {
91        results.iter().filter_map(|result| result.phases).fold(
92            Self {
93                collection_ms,
94                ..Self::default()
95            },
96            |acc, phases| Self {
97                collection_ms: acc.collection_ms,
98                setup_ms: acc.setup_ms.saturating_add(phases.setup_ms),
99                compile_ms: acc.compile_ms.saturating_add(phases.compile_ms),
100                execute_ms: acc.execute_ms.saturating_add(phases.execute_ms),
101                teardown_ms: acc.teardown_ms.saturating_add(phases.teardown_ms),
102                modules: acc.modules.saturating_add(phases.modules),
103            },
104        )
105    }
106}
107
108impl TestResult {
109    /// Emit a one-line phase breakdown to stderr. Driven by `--diagnose`
110    /// / `HARN_TEST_DIAGNOSE=1`. The format is intentionally
111    /// machine-readable so downstream eval pipelines can grep it.
112    pub(super) fn emit_diagnose(&self) {
113        let outcome = if self.passed { "ok" } else { "FAIL" };
114        let phases = self
115            .phases
116            .expect("diagnostics are emitted only for executed cases");
117        eprintln!(
118            "[harn test diag] {} {} setup={}ms compile={}ms execute={}ms teardown={}ms module_compile={}ms module_load={}ms modules_compiled={} modules_loaded={} total={}ms",
119            outcome,
120            self.name,
121            phases.setup_ms,
122            phases.compile_ms,
123            phases.execute_ms,
124            phases.teardown_ms,
125            phases.modules.module_compile_ms,
126            phases.modules.module_load_ms,
127            phases.modules.modules_compiled,
128            phases.modules.modules_loaded,
129            self.duration_ms,
130        );
131    }
132}
harn_cli/test_runner/reporting.rs

harn_cli/test_runner/
reporting.rs
