harn_test_runner/
reporting.rs

1//! Test result contracts, timing rollups, and diagnostic rendering.
2
3use serde::Serialize;
4
5use crate::timing::DurationSummary;
6
7#[derive(Clone, Debug, Serialize)]
8pub struct TestResult {
9    pub name: String,
10    pub file: String,
11    pub passed: bool,
12    pub error: Option<String>,
13    /// Everything the case wrote via `log`/`print`/`println`/etc, in
14    /// execution order. `None` when nothing was written — keeps quiet,
15    /// passing cases from padding reports. Discovery and worker-start
16    /// errors never reach a VM and always leave this absent.
17    #[serde(skip_serializing_if = "Option::is_none")]
18    pub captured_output: Option<String>,
19    /// Typed timeout metadata. Consumers must use this instead of parsing
20    /// human-readable error text.
21    pub timeout: Option<TestTimeout>,
22    pub duration_ms: u64,
23    /// Per-phase timings for an executed case. Discovery and worker-start
24    /// errors have no execution timeline and leave this absent.
25    #[serde(skip_serializing_if = "Option::is_none")]
26    pub phases: Option<PhaseTimings>,
27}
28
29#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
30pub struct TestTimeout {
31    pub phase: TestPhase,
32    pub limit_ms: u64,
33}
34
35#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
36#[serde(rename_all = "snake_case")]
37pub enum TestPhase {
38    Execute,
39}
40
41#[derive(Clone, Debug, Serialize)]
42pub struct TestSummary {
43    pub results: Vec<TestResult>,
44    pub passed: usize,
45    pub failed: usize,
46    pub total: usize,
47    pub duration_ms: u64,
48    /// Distribution of per-test wall-clock durations.
49    pub timing: DurationSummary,
50    /// Aggregated phase costs across the entire run.
51    pub aggregate: AggregateTimings,
52}
53
54/// Wall-clock cost of each phase of a single test execution.
55///
56/// Sums to the test's `duration_ms` modulo measurement overhead. Surfaced
57/// so consumers can attribute cold-start vs assertion cost without
58/// having to instrument the runner externally.
59#[derive(Debug, Default, Clone, Copy, Serialize)]
60pub struct PhaseTimings {
61    /// VM construction + stdlib/hostlib registration + skill install +
62    /// runtime extension install + manifest hooks/triggers install.
63    pub setup_ms: u64,
64    /// `Compiler::compile_named` time for this test's chunk.
65    pub compile_ms: u64,
66    /// `vm.execute(chunk)` wall time, i.e. the actual user-test body.
67    pub execute_ms: u64,
68    /// VM/LocalSet task cancellation and `reset_thread_local_state` between tests.
69    pub teardown_ms: u64,
70    /// Module attribution overlapping setup and execute. These values are
71    /// diagnostic subtotals and must not be added to the top-level phases.
72    pub modules: harn_vm::ModulePhaseStats,
73}
74
75/// Cumulative worker-time across the run. Mirrors [`PhaseTimings`] plus
76/// suite-level collection and import-graph preparation. Parallel case phases
77/// overlap, so these totals may exceed suite wall time.
78#[derive(Debug, Default, Clone, Copy, Serialize)]
79pub struct AggregateTimings {
80    pub collection_ms: u64,
81    pub setup_ms: u64,
82    pub compile_ms: u64,
83    /// Suite-level lowering of selected test-file entries. This is included in
84    /// `compile_ms` and exposed separately so callers can verify compile-once
85    /// behavior without inferring it from case timings.
86    pub test_file_compile_ms: u64,
87    pub test_files_compiled: usize,
88    pub test_entries_compiled: usize,
89    pub execute_ms: u64,
90    pub teardown_ms: u64,
91    /// Suite preparation plus per-case module attribution. Overlaps compile,
92    /// setup, and execute.
93    pub modules: harn_vm::ModulePhaseStats,
94}
95
96#[derive(Clone, Copy, Debug, Default)]
97#[doc(hidden)]
98pub struct SuiteModulePreparation {
99    pub duration_ms: u64,
100    pub modules: harn_vm::ModulePhaseStats,
101}
102
103#[derive(Clone, Copy, Debug, Default)]
104#[doc(hidden)]
105pub struct SuiteCallablePreparation {
106    pub duration_ms: u64,
107    pub files: usize,
108    pub entries: usize,
109}
110
111impl AggregateTimings {
112    #[doc(hidden)]
113    pub fn from_results(
114        collection_ms: u64,
115        module_preparation: SuiteModulePreparation,
116        callable_preparation: SuiteCallablePreparation,
117        results: &[TestResult],
118    ) -> Self {
119        results.iter().filter_map(|result| result.phases).fold(
120            Self {
121                collection_ms,
122                compile_ms: module_preparation
123                    .duration_ms
124                    .saturating_add(callable_preparation.duration_ms),
125                test_file_compile_ms: callable_preparation.duration_ms,
126                test_files_compiled: callable_preparation.files,
127                test_entries_compiled: callable_preparation.entries,
128                modules: module_preparation.modules,
129                ..Self::default()
130            },
131            |acc, phases| Self {
132                collection_ms: acc.collection_ms,
133                setup_ms: acc.setup_ms.saturating_add(phases.setup_ms),
134                compile_ms: acc.compile_ms.saturating_add(phases.compile_ms),
135                test_file_compile_ms: acc.test_file_compile_ms,
136                test_files_compiled: acc.test_files_compiled,
137                test_entries_compiled: acc.test_entries_compiled,
138                execute_ms: acc.execute_ms.saturating_add(phases.execute_ms),
139                teardown_ms: acc.teardown_ms.saturating_add(phases.teardown_ms),
140                modules: acc.modules.saturating_add(phases.modules),
141            },
142        )
143    }
144}
145
146impl TestResult {
147    /// Emit a one-line phase breakdown to stderr. Driven by `--diagnose`
148    /// / `HARN_TEST_DIAGNOSE=1`. The format is intentionally
149    /// machine-readable so downstream eval pipelines can grep it.
150    #[doc(hidden)]
151    pub fn emit_diagnose(&self) {
152        let outcome = if self.passed { "ok" } else { "FAIL" };
153        let phases = self
154            .phases
155            .expect("diagnostics are emitted only for executed cases");
156        eprintln!(
157            "[harn test diag] {} {} setup={}ms compile={}ms execute={}ms teardown={}ms module_compile={}ms module_load={}ms modules_compiled={} modules_loaded={} total={}ms",
158            outcome,
159            self.name,
160            phases.setup_ms,
161            phases.compile_ms,
162            phases.execute_ms,
163            phases.teardown_ms,
164            phases.modules.module_compile_ms,
165            phases.modules.module_load_ms,
166            phases.modules.modules_compiled,
167            phases.modules.modules_loaded,
168            self.duration_ms,
169        );
170    }
171}
harn_test_runner/reporting.rs

harn_test_runner/
reporting.rs
