harn_cli/test_runner/
reporting.rs

1//! Test result contracts, timing rollups, and diagnostic rendering.
2
3use serde::Serialize;
4
5use crate::test_timing::DurationSummary;
6
7#[derive(Clone, Debug, Serialize)]
8pub struct TestResult {
9    pub name: String,
10    pub file: String,
11    pub passed: bool,
12    pub error: Option<String>,
13    /// Everything the case wrote via `log`/`print`/`println`/etc, in
14    /// execution order. `None` when nothing was written — keeps quiet,
15    /// passing cases from padding reports. Discovery and worker-start
16    /// errors never reach a VM and always leave this absent.
17    #[serde(skip_serializing_if = "Option::is_none")]
18    pub captured_output: Option<String>,
19    /// Typed timeout metadata. Consumers must use this instead of parsing
20    /// human-readable error text.
21    pub timeout: Option<TestTimeout>,
22    pub duration_ms: u64,
23    /// Per-phase timings for an executed case. Discovery and worker-start
24    /// errors have no execution timeline and leave this absent.
25    #[serde(skip_serializing_if = "Option::is_none")]
26    pub phases: Option<PhaseTimings>,
27}
28
29#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
30pub struct TestTimeout {
31    pub phase: TestPhase,
32    pub limit_ms: u64,
33}
34
35#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
36#[serde(rename_all = "snake_case")]
37pub enum TestPhase {
38    Execute,
39}
40
41#[derive(Clone, Debug, Serialize)]
42pub struct TestSummary {
43    pub results: Vec<TestResult>,
44    pub passed: usize,
45    pub failed: usize,
46    pub total: usize,
47    pub duration_ms: u64,
48    /// Distribution of per-test wall-clock durations.
49    pub timing: DurationSummary,
50    /// Aggregated phase costs across the entire run.
51    pub aggregate: AggregateTimings,
52}
53
54/// Wall-clock cost of each phase of a single test execution.
55///
56/// Sums to the test's `duration_ms` modulo measurement overhead. Surfaced
57/// so consumers can attribute cold-start vs assertion cost without
58/// having to instrument the runner externally.
59#[derive(Debug, Default, Clone, Copy, Serialize)]
60pub struct PhaseTimings {
61    /// VM construction + stdlib/hostlib registration + skill install +
62    /// runtime extension install + manifest hooks/triggers install.
63    pub setup_ms: u64,
64    /// `Compiler::compile_named` time for this test's chunk.
65    pub compile_ms: u64,
66    /// `vm.execute(chunk)` wall time, i.e. the actual user-test body.
67    pub execute_ms: u64,
68    /// VM/LocalSet task cancellation and `reset_thread_local_state` between tests.
69    pub teardown_ms: u64,
70    /// Module attribution overlapping setup and execute. These values are
71    /// diagnostic subtotals and must not be added to the top-level phases.
72    pub modules: harn_vm::ModulePhaseStats,
73}
74
75/// Cumulative worker-time across the run. Mirrors [`PhaseTimings`] plus
76/// suite-level collection and import-graph preparation. Parallel case phases
77/// overlap, so these totals may exceed suite wall time.
78#[derive(Debug, Default, Clone, Copy, Serialize)]
79pub struct AggregateTimings {
80    pub collection_ms: u64,
81    pub setup_ms: u64,
82    pub compile_ms: u64,
83    pub execute_ms: u64,
84    pub teardown_ms: u64,
85    /// Suite preparation plus per-case module attribution. Overlaps compile,
86    /// setup, and execute.
87    pub modules: harn_vm::ModulePhaseStats,
88}
89
90#[derive(Clone, Copy, Debug, Default)]
91pub(super) struct SuiteModulePreparation {
92    pub duration_ms: u64,
93    pub modules: harn_vm::ModulePhaseStats,
94}
95
96impl AggregateTimings {
97    pub(super) fn from_results(
98        collection_ms: u64,
99        module_preparation: SuiteModulePreparation,
100        results: &[TestResult],
101    ) -> Self {
102        results.iter().filter_map(|result| result.phases).fold(
103            Self {
104                collection_ms,
105                compile_ms: module_preparation.duration_ms,
106                modules: module_preparation.modules,
107                ..Self::default()
108            },
109            |acc, phases| Self {
110                collection_ms: acc.collection_ms,
111                setup_ms: acc.setup_ms.saturating_add(phases.setup_ms),
112                compile_ms: acc.compile_ms.saturating_add(phases.compile_ms),
113                execute_ms: acc.execute_ms.saturating_add(phases.execute_ms),
114                teardown_ms: acc.teardown_ms.saturating_add(phases.teardown_ms),
115                modules: acc.modules.saturating_add(phases.modules),
116            },
117        )
118    }
119}
120
121impl TestResult {
122    /// Emit a one-line phase breakdown to stderr. Driven by `--diagnose`
123    /// / `HARN_TEST_DIAGNOSE=1`. The format is intentionally
124    /// machine-readable so downstream eval pipelines can grep it.
125    pub(super) fn emit_diagnose(&self) {
126        let outcome = if self.passed { "ok" } else { "FAIL" };
127        let phases = self
128            .phases
129            .expect("diagnostics are emitted only for executed cases");
130        eprintln!(
131            "[harn test diag] {} {} setup={}ms compile={}ms execute={}ms teardown={}ms module_compile={}ms module_load={}ms modules_compiled={} modules_loaded={} total={}ms",
132            outcome,
133            self.name,
134            phases.setup_ms,
135            phases.compile_ms,
136            phases.execute_ms,
137            phases.teardown_ms,
138            phases.modules.module_compile_ms,
139            phases.modules.module_load_ms,
140            phases.modules.modules_compiled,
141            phases.modules.modules_loaded,
142            self.duration_ms,
143        );
144    }
145}
harn_cli/test_runner/reporting.rs

harn_cli/test_runner/
reporting.rs
