llm_browser_testkit/events.rs
1//! Typed run events emitted by the runner.
2//!
3//! The runner no longer prints directly: every interesting point in a run
4//! (test start/end, step start/end, LLM call, budget warning) is emitted as a
5//! [`TestEvent`]. Sinks attached to the [`crate::reporting::Reporter`] render
6//! these events for humans (console), machines (NDJSON log file, JUnit XML),
7//! CI systems (GitHub workflow commands) and profilers (Perfetto trace).
8//!
9//! The event stream doubles as the run's source of truth: JUnit and trace
10//! files are derived from it, and a `--log-file` NDJSON capture alone is
11//! enough to replay or analyze a run.
12
13use serde::Serialize;
14
15/// Outcome of a single step.
16#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
17#[serde(rename_all = "snake_case")]
18pub enum StepStatus {
19 /// Step executed successfully and all assertions passed.
20 Passed,
21 /// Step execution or assertion failed.
22 Failed,
23 /// Step was skipped.
24 Skipped,
25}
26
27/// One event in the run's lifecycle.
28///
29/// Serialized as a flat JSON object with a `type` discriminator, e.g.
30/// `{"type":"step_finished","test":"login","status":"passed",...}`. The
31/// reporter adds a `ts` (epoch milliseconds) field on emission.
32#[derive(Debug, Clone, Serialize)]
33#[serde(tag = "type", rename_all = "snake_case")]
34pub enum TestEvent {
35 /// The run is about to execute `total_tests` tests.
36 RunStarted {
37 /// Number of tests in the run.
38 total_tests: u32,
39 },
40 /// A test started.
41 TestStarted {
42 /// Test name (includes viewport-matrix suffix when expanded).
43 test: String,
44 },
45 /// A step inside a test started.
46 StepStarted {
47 /// Enclosing test name.
48 test: String,
49 /// Zero-based step index within the test.
50 index: u32,
51 /// Human-readable label, e.g. `[click] the sign-in button`.
52 label: String,
53 },
54 /// A step finished.
55 StepFinished {
56 /// Enclosing test name.
57 test: String,
58 /// Zero-based step index within the test.
59 index: u32,
60 /// Human-readable label, e.g. `[click] the sign-in button`.
61 label: String,
62 /// Outcome of the step.
63 status: StepStatus,
64 /// Wall-clock duration of the step.
65 duration_ms: u64,
66 /// Human-readable result message (includes the page-state excerpt
67 /// and the failure message for failed steps).
68 message: String,
69 /// Multi-line failure diagnostics (URL, title, visible text, alerts)
70 /// — `None` for non-failed steps.
71 diagnostics: Option<String>,
72 /// Path of the failure screenshot, if one was written.
73 screenshot: Option<String>,
74 },
75 /// An LLM call started (one per attempted endpoint chain).
76 LlmCallStarted {
77 /// Enclosing test name.
78 test: String,
79 /// Zero-based step index within the test.
80 index: u32,
81 /// Endpoint name (or chain entry) being called.
82 endpoint: String,
83 /// Model name sent in the request.
84 model: String,
85 /// Call purpose: `targeting` or `assertion`.
86 purpose: String,
87 },
88 /// An LLM call finished (success or exhaustion of all attempts).
89 LlmCallFinished {
90 /// Enclosing test name.
91 test: String,
92 /// Zero-based step index within the test.
93 index: u32,
94 /// Endpoint that answered (or the last one tried on failure).
95 endpoint: String,
96 /// Model name used.
97 model: String,
98 /// Call purpose: `targeting` or `assertion`.
99 purpose: String,
100 /// Whether an answer was obtained.
101 ok: bool,
102 /// Wall-clock duration of the call including retries.
103 duration_ms: u64,
104 /// Input (prompt) tokens billed.
105 input_tokens: u64,
106 /// Output (completion) tokens billed.
107 output_tokens: u64,
108 /// Computed cost in USD.
109 cost: f64,
110 /// Error message when `ok` is false.
111 error: Option<String>,
112 },
113 /// A test finished.
114 TestFinished {
115 /// Test name.
116 test: String,
117 /// Number of passed steps.
118 passed: u32,
119 /// Number of failed steps.
120 failed: u32,
121 /// Number of skipped steps.
122 skipped: u32,
123 /// Wall-clock duration of the whole test.
124 duration_ms: u64,
125 /// Total cost of the test in USD.
126 cost: f64,
127 /// Total tokens consumed by the test.
128 tokens: u64,
129 /// Total LLM/MCP/agent calls made by the test.
130 calls: u64,
131 },
132 /// The whole run finished.
133 RunFinished {
134 /// Tests that passed.
135 tests_passed: u32,
136 /// Tests that failed.
137 tests_failed: u32,
138 /// Steps that passed.
139 steps_passed: u32,
140 /// Steps that failed.
141 steps_failed: u32,
142 /// Steps that were skipped.
143 steps_skipped: u32,
144 /// Total cost in USD across all tests.
145 total_cost: f64,
146 /// Total tokens consumed.
147 total_tokens: u64,
148 /// Total calls made.
149 total_calls: u64,
150 },
151 /// A non-fatal warning (budget soft-exceeded, feature not enabled, ...).
152 Warning {
153 /// Human-readable warning text.
154 message: String,
155 },
156}
157
158#[cfg(test)]
159mod tests {
160 use super::{StepStatus, TestEvent};
161
162 #[test]
163 fn test_step_status_serializes_snake_case() {
164 assert_eq!(
165 serde_json::to_value(StepStatus::Passed).unwrap(),
166 serde_json::json!("passed")
167 );
168 assert_eq!(
169 serde_json::to_value(StepStatus::Failed).unwrap(),
170 serde_json::json!("failed")
171 );
172 assert_eq!(
173 serde_json::to_value(StepStatus::Skipped).unwrap(),
174 serde_json::json!("skipped")
175 );
176 }
177
178 #[test]
179 fn test_event_serializes_with_type_tag() {
180 let event = TestEvent::StepFinished {
181 test: "login".into(),
182 index: 2,
183 label: "[click] the button".into(),
184 status: StepStatus::Failed,
185 duration_ms: 1234,
186 message: "element #btn not found".into(),
187 diagnostics: Some(" │ url: http://x".into()),
188 screenshot: Some("artifacts/x.png".into()),
189 };
190 let value = serde_json::to_value(&event).unwrap();
191 assert_eq!(value["type"], "step_finished");
192 assert_eq!(value["test"], "login");
193 assert_eq!(value["index"], 2);
194 assert_eq!(value["label"], "[click] the button");
195 assert_eq!(value["status"], "failed");
196 assert_eq!(value["duration_ms"], 1234);
197 assert_eq!(value["message"], "element #btn not found");
198 assert_eq!(value["diagnostics"], " │ url: http://x");
199 assert_eq!(value["screenshot"], "artifacts/x.png");
200 }
201
202 #[test]
203 fn test_run_started_shape() {
204 let value = serde_json::to_value(TestEvent::RunStarted { total_tests: 3 }).unwrap();
205 assert_eq!(value["type"], "run_started");
206 assert_eq!(value["total_tests"], 3);
207 }
208
209 #[test]
210 fn test_llm_call_finished_shape() {
211 let value = serde_json::to_value(TestEvent::LlmCallFinished {
212 test: "login".into(),
213 index: 0,
214 endpoint: "default".into(),
215 model: "deepseek".into(),
216 purpose: "targeting".into(),
217 ok: false,
218 duration_ms: 900,
219 input_tokens: 100,
220 output_tokens: 0,
221 cost: 0.0012,
222 error: Some("HTTP 429: slow down".into()),
223 })
224 .unwrap();
225 assert_eq!(value["type"], "llm_call_finished");
226 assert_eq!(value["endpoint"], "default");
227 assert_eq!(value["model"], "deepseek");
228 assert_eq!(value["purpose"], "targeting");
229 assert_eq!(value["ok"], false);
230 assert_eq!(value["cost"], 0.0012);
231 assert_eq!(value["error"], "HTTP 429: slow down");
232 }
233
234 #[test]
235 fn test_run_finished_shape() {
236 let value = serde_json::to_value(TestEvent::RunFinished {
237 tests_passed: 2,
238 tests_failed: 1,
239 steps_passed: 9,
240 steps_failed: 1,
241 steps_skipped: 2,
242 total_cost: 0.05,
243 total_tokens: 5000,
244 total_calls: 7,
245 })
246 .unwrap();
247 assert_eq!(value["type"], "run_finished");
248 assert_eq!(value["tests_passed"], 2);
249 assert_eq!(value["steps_failed"], 1);
250 assert_eq!(value["total_cost"], 0.05);
251 }
252
253 #[test]
254 fn test_warning_shape() {
255 let value = serde_json::to_value(TestEvent::Warning {
256 message: "budget soft".into(),
257 })
258 .unwrap();
259 assert_eq!(value["type"], "warning");
260 assert_eq!(value["message"], "budget soft");
261 }
262}