llm_browser_testkit/events.rs
1//! Typed run events emitted by the runner.
2//!
3//! The runner no longer prints directly: every interesting point in a run
4//! (test start/end, step start/end, LLM call, budget warning) is emitted as a
5//! [`TestEvent`]. Sinks attached to the [`crate::reporting::Reporter`] render
6//! these events for humans (console), machines (NDJSON log file, JUnit XML),
7//! CI systems (GitHub workflow commands) and profilers (Perfetto trace).
8//!
9//! The event stream doubles as the run's source of truth: JUnit and trace
10//! files are derived from it, and a `--log-file` NDJSON capture alone is
11//! enough to replay or analyze a run.
12
13use serde::Serialize;
14
15/// Outcome of a single step.
16#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
17#[serde(rename_all = "snake_case")]
18pub enum StepStatus {
19 /// Step executed successfully and all assertions passed.
20 Passed,
21 /// Step execution or assertion failed.
22 Failed,
23 /// Step was skipped.
24 Skipped,
25}
26
27/// One event in the run's lifecycle.
28///
29/// Serialized as a flat JSON object with a `type` discriminator, e.g.
30/// `{"type":"step_finished","test":"login","status":"passed",...}`. The
31/// reporter adds a `ts` (epoch milliseconds) field on emission.
32#[derive(Debug, Clone, Serialize)]
33#[serde(tag = "type", rename_all = "snake_case")]
34pub enum TestEvent {
35 /// The run is about to execute `total_tests` tests.
36 RunStarted {
37 /// Number of tests in the run.
38 total_tests: u32,
39 },
40 /// A test started.
41 TestStarted {
42 /// Test name (includes viewport-matrix suffix when expanded).
43 test: String,
44 },
45 /// A step inside a test started.
46 StepStarted {
47 /// Enclosing test name.
48 test: String,
49 /// Zero-based step index within the test.
50 index: u32,
51 /// Human-readable label, e.g. `[click] the sign-in button`.
52 label: String,
53 },
54 /// A step finished.
55 StepFinished {
56 /// Enclosing test name.
57 test: String,
58 /// Zero-based step index within the test.
59 index: u32,
60 /// Human-readable label, e.g. `[click] the sign-in button`.
61 label: String,
62 /// Outcome of the step.
63 status: StepStatus,
64 /// Wall-clock duration of the step.
65 duration_ms: u64,
66 /// Human-readable result message (includes the page-state excerpt
67 /// and the failure message for failed steps).
68 message: String,
69 /// Multi-line failure diagnostics (URL, title, visible text, alerts)
70 /// — `None` for non-failed steps.
71 diagnostics: Option<String>,
72 /// Path of the failure screenshot, if one was written.
73 screenshot: Option<String>,
74 },
75 /// An LLM call started (one per attempted endpoint chain).
76 LlmCallStarted {
77 /// Enclosing test name.
78 test: String,
79 /// Zero-based step index within the test.
80 index: u32,
81 /// Endpoint name (or chain entry) being called.
82 endpoint: String,
83 /// Model name sent in the request.
84 model: String,
85 /// Call purpose: `targeting` or `assertion`.
86 purpose: String,
87 },
88 /// An LLM call finished (success or exhaustion of all attempts).
89 LlmCallFinished {
90 /// Enclosing test name.
91 test: String,
92 /// Zero-based step index within the test.
93 index: u32,
94 /// Endpoint that answered (or the last one tried on failure).
95 endpoint: String,
96 /// Model name used.
97 model: String,
98 /// Call purpose: `targeting` or `assertion`.
99 purpose: String,
100 /// Whether an answer was obtained.
101 ok: bool,
102 /// Wall-clock duration of the call including retries.
103 duration_ms: u64,
104 /// Input (prompt) tokens billed.
105 input_tokens: u64,
106 /// Output (completion) tokens billed.
107 output_tokens: u64,
108 /// Input tokens served from the provider's prompt cache.
109 cached_input_tokens: u64,
110 /// Computed cost in USD.
111 cost: f64,
112 /// Error message when `ok` is false.
113 error: Option<String>,
114 },
115 /// A test finished.
116 TestFinished {
117 /// Test name.
118 test: String,
119 /// Number of passed steps.
120 passed: u32,
121 /// Number of failed steps.
122 failed: u32,
123 /// Number of skipped steps.
124 skipped: u32,
125 /// Wall-clock duration of the whole test.
126 duration_ms: u64,
127 /// Total cost of the test in USD.
128 cost: f64,
129 /// Total tokens consumed by the test.
130 tokens: u64,
131 /// Input (prompt) tokens consumed by the test.
132 input_tokens: u64,
133 /// Output (completion) tokens consumed by the test.
134 output_tokens: u64,
135 /// Input tokens served from provider prompt caches.
136 cached_input_tokens: u64,
137 /// Models used by the test, sorted and deduplicated.
138 models: Vec<String>,
139 /// Total LLM/MCP/agent calls made by the test.
140 calls: u64,
141 },
142 /// The whole run finished.
143 RunFinished {
144 /// Tests that passed.
145 tests_passed: u32,
146 /// Tests that failed.
147 tests_failed: u32,
148 /// Steps that passed.
149 steps_passed: u32,
150 /// Steps that failed.
151 steps_failed: u32,
152 /// Steps that were skipped.
153 steps_skipped: u32,
154 /// Total cost in USD across all tests.
155 total_cost: f64,
156 /// Total tokens consumed.
157 total_tokens: u64,
158 /// Total input (prompt) tokens consumed.
159 total_input_tokens: u64,
160 /// Total output (completion) tokens consumed.
161 total_output_tokens: u64,
162 /// Total input tokens served from provider prompt caches.
163 total_cached_input_tokens: u64,
164 /// Models used across the run, sorted and deduplicated.
165 models: Vec<String>,
166 /// Total calls made.
167 total_calls: u64,
168 },
169 /// A non-fatal warning (budget soft-exceeded, feature not enabled, ...).
170 Warning {
171 /// Human-readable warning text.
172 message: String,
173 },
174}
175
176#[cfg(test)]
177mod tests {
178 use super::{StepStatus, TestEvent};
179
180 #[test]
181 fn test_step_status_serializes_snake_case() {
182 assert_eq!(
183 serde_json::to_value(StepStatus::Passed).unwrap(),
184 serde_json::json!("passed")
185 );
186 assert_eq!(
187 serde_json::to_value(StepStatus::Failed).unwrap(),
188 serde_json::json!("failed")
189 );
190 assert_eq!(
191 serde_json::to_value(StepStatus::Skipped).unwrap(),
192 serde_json::json!("skipped")
193 );
194 }
195
196 #[test]
197 fn test_event_serializes_with_type_tag() {
198 let event = TestEvent::StepFinished {
199 test: "login".into(),
200 index: 2,
201 label: "[click] the button".into(),
202 status: StepStatus::Failed,
203 duration_ms: 1234,
204 message: "element #btn not found".into(),
205 diagnostics: Some(" │ url: http://x".into()),
206 screenshot: Some("artifacts/x.png".into()),
207 };
208 let value = serde_json::to_value(&event).unwrap();
209 assert_eq!(value["type"], "step_finished");
210 assert_eq!(value["test"], "login");
211 assert_eq!(value["index"], 2);
212 assert_eq!(value["label"], "[click] the button");
213 assert_eq!(value["status"], "failed");
214 assert_eq!(value["duration_ms"], 1234);
215 assert_eq!(value["message"], "element #btn not found");
216 assert_eq!(value["diagnostics"], " │ url: http://x");
217 assert_eq!(value["screenshot"], "artifacts/x.png");
218 }
219
220 #[test]
221 fn test_run_started_shape() {
222 let value = serde_json::to_value(TestEvent::RunStarted { total_tests: 3 }).unwrap();
223 assert_eq!(value["type"], "run_started");
224 assert_eq!(value["total_tests"], 3);
225 }
226
227 #[test]
228 fn test_llm_call_finished_shape() {
229 let value = serde_json::to_value(TestEvent::LlmCallFinished {
230 test: "login".into(),
231 index: 0,
232 endpoint: "default".into(),
233 model: "deepseek".into(),
234 purpose: "targeting".into(),
235 ok: false,
236 duration_ms: 900,
237 input_tokens: 100,
238 output_tokens: 0,
239 cached_input_tokens: 40,
240 cost: 0.0012,
241 error: Some("HTTP 429: slow down".into()),
242 })
243 .unwrap();
244 assert_eq!(value["type"], "llm_call_finished");
245 assert_eq!(value["endpoint"], "default");
246 assert_eq!(value["model"], "deepseek");
247 assert_eq!(value["purpose"], "targeting");
248 assert_eq!(value["ok"], false);
249 assert_eq!(value["cached_input_tokens"], 40);
250 assert_eq!(value["cost"], 0.0012);
251 assert_eq!(value["error"], "HTTP 429: slow down");
252 }
253
254 #[test]
255 fn test_run_finished_shape() {
256 let value = serde_json::to_value(TestEvent::RunFinished {
257 tests_passed: 2,
258 tests_failed: 1,
259 steps_passed: 9,
260 steps_failed: 1,
261 steps_skipped: 2,
262 total_cost: 0.05,
263 total_tokens: 5000,
264 total_input_tokens: 4000,
265 total_output_tokens: 1000,
266 total_cached_input_tokens: 500,
267 models: vec!["deepseek".into()],
268 total_calls: 7,
269 })
270 .unwrap();
271 assert_eq!(value["type"], "run_finished");
272 assert_eq!(value["tests_passed"], 2);
273 assert_eq!(value["steps_failed"], 1);
274 assert_eq!(value["total_cost"], 0.05);
275 assert_eq!(value["total_input_tokens"], 4000);
276 assert_eq!(value["total_cached_input_tokens"], 500);
277 assert_eq!(value["models"][0], "deepseek");
278 }
279
280 #[test]
281 fn test_warning_shape() {
282 let value = serde_json::to_value(TestEvent::Warning {
283 message: "budget soft".into(),
284 })
285 .unwrap();
286 assert_eq!(value["type"], "warning");
287 assert_eq!(value["message"], "budget soft");
288 }
289}