harn_cli/test_runner/
reporting.rs

1//! Test result contracts, timing rollups, and diagnostic rendering.
2
3use serde::Serialize;
4
5use crate::test_timing::DurationSummary;
6
7#[derive(Clone, Debug, Serialize)]
8pub struct TestResult {
9    pub name: String,
10    pub file: String,
11    pub passed: bool,
12    pub error: Option<String>,
13    /// Everything the case wrote via `log`/`print`/`println`/etc, in
14    /// execution order. `None` when nothing was written — keeps quiet,
15    /// passing cases from padding reports. Discovery and worker-start
16    /// errors never reach a VM and always leave this absent.
17    #[serde(skip_serializing_if = "Option::is_none")]
18    pub captured_output: Option<String>,
19    /// Typed timeout metadata. Consumers must use this instead of parsing
20    /// human-readable error text.
21    pub timeout: Option<TestTimeout>,
22    pub duration_ms: u64,
23    /// Per-phase timings for an executed case. Discovery and worker-start
24    /// errors have no execution timeline and leave this absent.
25    #[serde(skip_serializing_if = "Option::is_none")]
26    pub phases: Option<PhaseTimings>,
27}
28
29#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
30pub struct TestTimeout {
31    pub phase: TestPhase,
32    pub limit_ms: u64,
33}
34
35#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
36#[serde(rename_all = "snake_case")]
37pub enum TestPhase {
38    Execute,
39}
40
41#[derive(Clone, Debug, Serialize)]
42pub struct TestSummary {
43    pub results: Vec<TestResult>,
44    pub passed: usize,
45    pub failed: usize,
46    pub total: usize,
47    pub duration_ms: u64,
48    /// Distribution of per-test wall-clock durations.
49    pub timing: DurationSummary,
50    /// Aggregated phase costs across the entire run.
51    pub aggregate: AggregateTimings,
52}
53
54/// Wall-clock cost of each phase of a single test execution.
55///
56/// Sums to the test's `duration_ms` modulo measurement overhead. Surfaced
57/// so consumers can attribute cold-start vs assertion cost without
58/// having to instrument the runner externally.
59#[derive(Debug, Default, Clone, Copy, Serialize)]
60pub struct PhaseTimings {
61    /// VM construction + stdlib/hostlib registration + skill install +
62    /// runtime extension install + manifest hooks/triggers install.
63    pub setup_ms: u64,
64    /// `Compiler::compile_named` time for this test's chunk.
65    pub compile_ms: u64,
66    /// `vm.execute(chunk)` wall time, i.e. the actual user-test body.
67    pub execute_ms: u64,
68    /// VM/LocalSet task cancellation and `reset_thread_local_state` between tests.
69    pub teardown_ms: u64,
70    /// Module attribution overlapping setup and execute. These values are
71    /// diagnostic subtotals and must not be added to the top-level phases.
72    pub modules: harn_vm::ModulePhaseStats,
73}
74
75/// Cumulative worker-time across the run. Mirrors [`PhaseTimings`] plus
76/// suite-level collection and import-graph preparation. Parallel case phases
77/// overlap, so these totals may exceed suite wall time.
78#[derive(Debug, Default, Clone, Copy, Serialize)]
79pub struct AggregateTimings {
80    pub collection_ms: u64,
81    pub setup_ms: u64,
82    pub compile_ms: u64,
83    /// Suite-level lowering of selected test-file entries. This is included in
84    /// `compile_ms` and exposed separately so callers can verify compile-once
85    /// behavior without inferring it from case timings.
86    pub test_file_compile_ms: u64,
87    pub test_files_compiled: usize,
88    pub test_entries_compiled: usize,
89    pub execute_ms: u64,
90    pub teardown_ms: u64,
91    /// Suite preparation plus per-case module attribution. Overlaps compile,
92    /// setup, and execute.
93    pub modules: harn_vm::ModulePhaseStats,
94}
95
96#[derive(Clone, Copy, Debug, Default)]
97pub(super) struct SuiteModulePreparation {
98    pub duration_ms: u64,
99    pub modules: harn_vm::ModulePhaseStats,
100}
101
102#[derive(Clone, Copy, Debug, Default)]
103pub(super) struct SuiteCallablePreparation {
104    pub duration_ms: u64,
105    pub files: usize,
106    pub entries: usize,
107}
108
109impl AggregateTimings {
110    pub(super) fn from_results(
111        collection_ms: u64,
112        module_preparation: SuiteModulePreparation,
113        callable_preparation: SuiteCallablePreparation,
114        results: &[TestResult],
115    ) -> Self {
116        results.iter().filter_map(|result| result.phases).fold(
117            Self {
118                collection_ms,
119                compile_ms: module_preparation
120                    .duration_ms
121                    .saturating_add(callable_preparation.duration_ms),
122                test_file_compile_ms: callable_preparation.duration_ms,
123                test_files_compiled: callable_preparation.files,
124                test_entries_compiled: callable_preparation.entries,
125                modules: module_preparation.modules,
126                ..Self::default()
127            },
128            |acc, phases| Self {
129                collection_ms: acc.collection_ms,
130                setup_ms: acc.setup_ms.saturating_add(phases.setup_ms),
131                compile_ms: acc.compile_ms.saturating_add(phases.compile_ms),
132                test_file_compile_ms: acc.test_file_compile_ms,
133                test_files_compiled: acc.test_files_compiled,
134                test_entries_compiled: acc.test_entries_compiled,
135                execute_ms: acc.execute_ms.saturating_add(phases.execute_ms),
136                teardown_ms: acc.teardown_ms.saturating_add(phases.teardown_ms),
137                modules: acc.modules.saturating_add(phases.modules),
138            },
139        )
140    }
141}
142
143impl TestResult {
144    /// Emit a one-line phase breakdown to stderr. Driven by `--diagnose`
145    /// / `HARN_TEST_DIAGNOSE=1`. The format is intentionally
146    /// machine-readable so downstream eval pipelines can grep it.
147    pub(super) fn emit_diagnose(&self) {
148        let outcome = if self.passed { "ok" } else { "FAIL" };
149        let phases = self
150            .phases
151            .expect("diagnostics are emitted only for executed cases");
152        eprintln!(
153            "[harn test diag] {} {} setup={}ms compile={}ms execute={}ms teardown={}ms module_compile={}ms module_load={}ms modules_compiled={} modules_loaded={} total={}ms",
154            outcome,
155            self.name,
156            phases.setup_ms,
157            phases.compile_ms,
158            phases.execute_ms,
159            phases.teardown_ms,
160            phases.modules.module_compile_ms,
161            phases.modules.module_load_ms,
162            phases.modules.modules_compiled,
163            phases.modules.modules_loaded,
164            self.duration_ms,
165        );
166    }
167}
harn_cli/test_runner/reporting.rs

harn_cli/test_runner/
reporting.rs
