Skip to main content

llm_browser_testkit/
scenario.rs

1//! Scenario types for human-readable browser test case definitions.
2//!
3//! Scenarios are written in [TOML](https://toml.io) and describe groups of
4//! browser interaction tests with reusable assertion definitions and
5//! configurable test-level overrides.
6//!
7//! # Structure
8//!
9//! ```toml
10//! [config]                         # Global defaults
11//! start_url = "/dashboard"
12//!
13//! [[definitions]]                  # Reusable assertion definitions
14//! name = "no_errors"
15//! preset = "no_error_on_page"
16//!
17//! [[test]]                         # Test group
18//! name = "Dashboard Smoke"
19//! start_url = "/dashboard"         # Override global start_url (optional)
20//!
21//! [[test.steps]]                   # Ordered steps — the `kind` field
22//! kind = "navigate"                # determines which other fields apply
23//! url = "/dashboard"
24//!
25//! [[test.steps]]
26//! kind = "click"
27//! target = "the Login button"      # Natural language — LLM resolves to selector
28//!
29//! [[test.steps]]
30//! kind = "assert"
31//! definition = "no_errors"
32//! ```
33//!
34//! ## Step Kinds
35//!
36//! | `kind`        | Required fields  | Optional fields                          |
37//! |---------------|-----------------|------------------------------------------|
38//! | `navigate`    | `url`           | `wait_after_ms`                          |
39//! | `click`       | `target`        | `selector`, `wait_after_ms`              |
40//! | `type`        | `target`, `text`| `selector`, `wait_after_ms`              |
41//! | `wait`        | `target`        | `selector`, `timeout_ms`                 |
42//! | `assert`      | *one of below*  | —                                        |
43//! | `screenshot`  | —               | `path`                                   |
44//! | `agent`       | `agent`, `task` | —                                        |
45//! | `mcp`         | `server`, `tool`| `args`                                   |
46//!
47//! Assert steps require one of: `definition` (references a named
48//! `[[definitions]]` entry), `preset` (built-in preset name), or `prompt`
49//! (custom LLM evaluation prompt).
50
51use std::collections::HashMap;
52
53use serde::Deserialize;
54use serde_json::Value;
55
56/// Top-level scenario file, deserialized from TOML.
57#[derive(Debug, Deserialize)]
58pub struct Scenario {
59    /// Global configuration (overridable per test).
60    #[serde(default)]
61    pub config: ScenarioConfig,
62
63    /// Reusable assertion definitions referenced by name in `assert` steps.
64    #[serde(default)]
65    pub definitions: Vec<AssertDefinition>,
66
67    /// Ordered test groups to execute.
68    #[serde(default)]
69    pub test: Vec<TestGroup>,
70}
71
72/// Global scenario configuration with per-test overridable fields.
73#[derive(Debug, Deserialize, Default, Clone)]
74pub struct ScenarioConfig {
75    /// Base URL for relative navigation.
76    #[serde(default)]
77    pub base_url: Option<String>,
78    /// LLM server base URL (deprecated; prefer `[config.endpoints]`).
79    #[serde(default)]
80    pub llm_url: Option<String>,
81    /// LLM model name (deprecated; prefer `[config.endpoints]`).
82    #[serde(default)]
83    pub llm_model: Option<String>,
84    /// LLM API key (Bearer token).
85    #[serde(default)]
86    pub llm_api_key: Option<String>,
87    /// Custom HTTP headers as JSON key-value pairs.
88    #[serde(default, deserialize_with = "deserialize_headers")]
89    pub llm_headers: HashMap<String, String>,
90    /// Run browser in headless mode.
91    #[serde(default)]
92    pub browser_headless: Option<bool>,
93    /// HTTP / browser action timeout in seconds.
94    #[serde(default)]
95    pub timeout_secs: Option<u64>,
96    /// Browser viewport width.
97    #[serde(default)]
98    pub viewport_width: Option<u32>,
99    /// Browser viewport height.
100    #[serde(default)]
101    pub viewport_height: Option<u32>,
102    /// Default URL every test auto-navigates to before running its steps.
103    #[serde(default)]
104    pub start_url: Option<String>,
105    /// Whether to auto-navigate to `start_url` before test steps.
106    ///
107    /// Disable when a test starts with click-based navigation.
108    #[serde(default = "default_auto_navigate")]
109    pub auto_navigate: bool,
110    /// LLM temperature (0.0–1.0). Lower = more deterministic.
111    #[serde(default = "default_temperature")]
112    pub temperature: f64,
113    /// Enable thinking/reasoning tokens. `None` means the provider default
114    /// is used (no `thinking` key is sent). Set to `true`/`false` to
115    /// explicitly enable or disable.
116    #[serde(default)]
117    pub thinking: Option<bool>,
118    /// Provider-specific model parameters merged into the chat completion
119    /// request body (e.g. `effort = "high"` for Anthropic).
120    #[serde(default, deserialize_with = "deserialize_model_params")]
121    pub model_params: HashMap<String, Value>,
122    /// Named endpoints (LLM, MCP, A2A agents) with pricing.
123    #[serde(default)]
124    pub endpoints: HashMap<String, EndpointConfig>,
125    /// Global and per-test budgets for cost/token/call limits.
126    #[serde(default)]
127    pub budgets: BudgetsConfig,
128    /// MCP server exposure configuration.
129    #[serde(default)]
130    pub mcp_server: Option<McpServerConfig>,
131    /// A2A agent server exposure configuration.
132    #[serde(default)]
133    pub a2a_server: Option<A2aServerConfig>,
134    /// Whether to continue running the remaining steps of a test after a
135    /// step fails. Default `false` = fail fast: the first failed step ends
136    /// the test and the rest are reported as skipped. Set to `true` to run
137    /// every step (more diagnostics, more LLM cost on broken apps).
138    #[serde(default)]
139    pub continue_on_failure: bool,
140    /// Longest edge (px) of screenshots attached to `screenshot = true`
141    /// assert steps, before they are JPEG-encoded and sent to the vision
142    /// endpoint. Downscaling keeps vision token cost/quality sane.
143    /// Default: 1400.
144    #[serde(default)]
145    pub screenshot_max_dimension: Option<u32>,
146    /// Directory for failure artifacts (screenshots, page snapshots).
147    /// Defaults to `artifacts`.
148    #[serde(default)]
149    pub artifacts_dir: Option<String>,
150    /// Optional viewport matrix: when set, every test in the scenario is
151    /// expanded into one variant per viewport (e.g. mobile/tablet/desktop).
152    /// Each variant overrides the test's viewport and gets a ` — <name>`
153    /// suffix on the test name. Per-test budgets apply per variant.
154    #[serde(default)]
155    pub viewport_matrix: Option<ViewportMatrix>,
156}
157
158/// A list of named viewports a scenario is expanded across.
159#[derive(Debug, Deserialize, Clone, Default)]
160pub struct ViewportMatrix {
161    /// The viewport variants (`{name, width, height}`).
162    #[serde(default)]
163    pub viewports: Vec<ViewportDef>,
164}
165
166/// One named viewport size in a matrix.
167#[derive(Debug, Deserialize, Clone)]
168pub struct ViewportDef {
169    /// Human-readable variant name (appended to test names, e.g.
170    /// `— mobile`).
171    pub name: String,
172    /// Browser viewport width in pixels.
173    pub width: u32,
174    /// Browser viewport height in pixels.
175    pub height: u32,
176}
177
178/// A named endpoint definition with pricing.
179#[derive(Debug, Deserialize, Clone, Default)]
180pub struct EndpointConfig {
181    /// Endpoint type: `llm`, `mcp`, or `a2a`.
182    #[serde(rename = "type")]
183    pub endpoint_type: EndpointType,
184    /// Base URL for the endpoint.
185    #[serde(default)]
186    pub url: Option<String>,
187    /// Model name (LLM endpoints only).
188    #[serde(default)]
189    pub model: Option<String>,
190    /// API key / bearer token.
191    #[serde(default)]
192    pub api_key: Option<String>,
193    /// Custom HTTP headers as JSON key-value pairs.
194    #[serde(default, deserialize_with = "deserialize_headers")]
195    pub headers: HashMap<String, String>,
196    /// Pricing configuration.
197    #[serde(default)]
198    pub pricing: Option<PricingConfig>,
199    /// Task types this endpoint serves by default
200    /// (e.g. `["targeting", "assertion"]`).
201    #[serde(default)]
202    pub default_for: Vec<String>,
203    /// Command to launch an MCP server subprocess (stdio transport).
204    #[serde(default)]
205    pub command: Option<String>,
206    /// Arguments for the MCP server command.
207    #[serde(default)]
208    pub args: Vec<String>,
209    /// Whether this LLM endpoint accepts image parts (vision) in addition
210    /// to text. `assert` steps with `screenshot = true` require a vision
211    /// endpoint.
212    #[serde(default)]
213    pub vision: bool,
214}
215
216/// Type discriminator for endpoint configuration.
217#[derive(Debug, Deserialize, Clone, PartialEq, Eq, Default)]
218#[serde(rename_all = "lowercase")]
219pub enum EndpointType {
220    /// OpenAI-compatible LLM API.
221    #[default]
222    Llm,
223    /// Model Context Protocol server.
224    Mcp,
225    /// Agent-to-Agent protocol agent.
226    A2a,
227}
228
229/// Pricing configuration for an endpoint.
230#[derive(Debug, Deserialize, Clone, Default)]
231pub struct PricingConfig {
232    /// Cost per 1M input tokens (USD).
233    #[serde(default)]
234    pub input_per_1m_tokens: f64,
235    /// Cost per 1M output tokens (USD).
236    #[serde(default)]
237    pub output_per_1m_tokens: f64,
238    /// Flat cost per call (USD), used for MCP/agent endpoints.
239    #[serde(default)]
240    pub per_call: f64,
241}
242
243/// Budget limits for test execution.
244#[derive(Debug, Deserialize, Clone, Default)]
245pub struct BudgetsConfig {
246    /// Global budget across all tests in the scenario.
247    #[serde(default)]
248    pub global: Option<BudgetDef>,
249    /// Default per-test budget. Individual tests can override.
250    #[serde(default)]
251    pub per_test_default: Option<BudgetDef>,
252}
253
254/// A budget definition with limits and enforcement mode.
255#[derive(Debug, Deserialize, Clone)]
256pub struct BudgetDef {
257    /// Maximum cost in USD.
258    #[serde(default)]
259    pub max_cost: Option<f64>,
260    /// Maximum total tokens (input + output).
261    #[serde(default)]
262    pub max_tokens: Option<u64>,
263    /// Maximum number of calls (LLM, MCP, agent combined).
264    #[serde(default)]
265    pub max_calls: Option<u64>,
266    /// Enforcement mode: `hard` (abort) or `soft` (warn and continue).
267    #[serde(default)]
268    pub enforcement: Option<BudgetEnforcement>,
269}
270
271/// Budget enforcement strategy.
272#[derive(Debug, Deserialize, Clone, PartialEq, Eq)]
273#[serde(rename_all = "lowercase")]
274pub enum BudgetEnforcement {
275    /// Abort the test or run when budget is exceeded.
276    Hard,
277    /// Log a warning but continue execution.
278    Soft,
279}
280
281/// MCP server exposure configuration.
282#[derive(Debug, Deserialize, Clone)]
283pub struct McpServerConfig {
284    /// Whether to enable the embedded MCP server.
285    #[serde(default)]
286    pub enabled: bool,
287    /// Port to listen on.
288    #[serde(default = "default_mcp_port")]
289    pub port: u16,
290}
291
292const fn default_mcp_port() -> u16 {
293    3000
294}
295
296/// A2A agent server exposure configuration.
297#[derive(Debug, Deserialize, Clone)]
298pub struct A2aServerConfig {
299    /// Whether to enable the embedded A2A agent server.
300    #[serde(default)]
301    pub enabled: bool,
302    /// Port to listen on.
303    #[serde(default = "default_a2a_port")]
304    pub port: u16,
305}
306
307const fn default_a2a_port() -> u16 {
308    3100
309}
310
311fn deserialize_headers<'de, D>(deserializer: D) -> Result<HashMap<String, String>, D::Error>
312where
313    D: serde::Deserializer<'de>,
314{
315    let raw: Option<serde_json::Value> = Option::deserialize(deserializer)?;
316    let Some(json) = raw else {
317        return Ok(HashMap::new());
318    };
319    let serde_json::Value::Object(obj) = json else {
320        return Ok(HashMap::new());
321    };
322    Ok(obj
323        .into_iter()
324        .filter_map(|(k, v)| v.as_str().map(|s| (k, s.to_owned())))
325        .collect())
326}
327
328const fn default_auto_navigate() -> bool {
329    true
330}
331
332fn default_login_url() -> String {
333    "/auth/login".to_owned()
334}
335
336const fn default_temperature() -> f64 {
337    0.0
338}
339
340fn deserialize_model_params<'de, D>(deserializer: D) -> Result<HashMap<String, Value>, D::Error>
341where
342    D: serde::Deserializer<'de>,
343{
344    #[derive(Deserialize)]
345    #[serde(untagged)]
346    enum Raw {
347        Map(HashMap<String, Value>),
348        Table(HashMap<String, Value>),
349    }
350    let raw: Option<Raw> = Option::deserialize(deserializer)?;
351    Ok(match raw {
352        Some(Raw::Map(m) | Raw::Table(m)) => m,
353        None => HashMap::new(),
354    })
355}
356
357/// Reusable assertion definition referenced by name from `assert` steps.
358///
359/// Definitions can either reference a built-in preset via `preset`, supply a
360/// custom LLM `prompt`, or define a **custom preset** by providing both
361/// `system` and `user_template`. Custom presets support the same template
362/// variables as built-in presets: `{url}`, `{title}`, `{content}`,
363/// `{expected_text}`, `{description}`.
364#[derive(Debug, Deserialize, Clone)]
365pub struct AssertDefinition {
366    /// Unique name used to reference this definition from steps.
367    pub name: String,
368    /// Predefined assertion preset name
369    /// (e.g. `no_error_on_page`, `text_visible`).
370    #[serde(default)]
371    pub preset: Option<String>,
372    /// Custom LLM prompt for assertion evaluation.
373    #[serde(default)]
374    pub prompt: Option<String>,
375    /// System prompt for a custom preset.
376    #[serde(default)]
377    pub system: Option<String>,
378    /// User template (with `{placeholders}`) for a custom preset.
379    #[serde(default)]
380    pub user_template: Option<String>,
381    /// Text that the `text_visible` preset checks for, or the
382    /// `{expected_text}` placeholder value for custom presets.
383    #[serde(default)]
384    pub assert_text: Option<String>,
385    /// Agent endpoint to call for this assertion.
386    #[serde(default)]
387    pub agent: Option<String>,
388    /// Agent task template for this assertion.
389    #[serde(default)]
390    pub task_template: Option<String>,
391}
392
393/// A group of steps that form a single test scenario.
394#[derive(Debug, Deserialize, Clone)]
395pub struct TestGroup {
396    /// Human-readable test name.
397    pub name: String,
398    /// Override the global `start_url` for this test.
399    #[serde(default)]
400    pub start_url: Option<String>,
401    /// Override the global `auto_navigate` for this test.
402    #[serde(default)]
403    pub auto_navigate: Option<bool>,
404    /// Override the global `base_url` for this test.
405    #[serde(default)]
406    pub base_url: Option<String>,
407    /// Override the global `timeout_secs` for this test.
408    #[serde(default)]
409    pub timeout_secs: Option<u64>,
410    /// Override the global `browser_headless` for this test.
411    #[serde(default)]
412    pub browser_headless: Option<bool>,
413    /// Override the global viewport width for this test (applied via CDP
414    /// `Emulation.setDeviceMetricsOverride` before the test runs).
415    #[serde(default)]
416    pub viewport_width: Option<u32>,
417    /// Override the global viewport height for this test.
418    #[serde(default)]
419    pub viewport_height: Option<u32>,
420    /// Per-test budget override.
421    #[serde(default)]
422    pub budget: Option<BudgetDef>,
423    /// Endpoint to use for all steps in this test (can be overridden
424    /// per-step).
425    #[serde(default)]
426    pub endpoint: Option<String>,
427    /// Ordered steps to execute.
428    #[serde(default)]
429    pub steps: Vec<TestStep>,
430}
431
432/// A single step in a test. The `kind` field determines which variant is
433/// deserialized and which field constraints apply.
434#[derive(Debug, Deserialize, Clone)]
435#[serde(tag = "kind")]
436pub enum TestStep {
437    /// Navigate the browser to a URL.
438    #[serde(rename = "navigate")]
439    Navigate {
440        /// URL to navigate to (absolute, or relative to the test's
441        /// base URL).
442        url: String,
443        /// Milliseconds to wait after navigation completes.
444        #[serde(default)]
445        wait_after_ms: Option<u64>,
446    },
447
448    /// Idempotent login: navigate to the login URL; if the app is
449    /// already authenticated (no login form rendered), pass silently.
450    /// Otherwise fill the form, wait for the bot-protection token,
451    /// submit, and wait for the authenticated shell. Built for
452    /// scenarios that repeat login steps across viewport-matrix
453    /// variants in one browser session.
454    #[serde(rename = "login")]
455    Login {
456        /// Login page URL (relative to the base URL).
457        #[serde(default = "default_login_url")]
458        url: String,
459        /// Email / username to enter.
460        email: String,
461        /// Password to enter.
462        password: String,
463        /// Milliseconds to wait after the authenticated shell appears.
464        #[serde(default)]
465        wait_after_ms: Option<u64>,
466    },
467
468    /// Click an element described in natural language.
469    #[serde(rename = "click")]
470    Click {
471        /// Natural language description of the element. The LLM resolves
472        /// this to a CSS selector at runtime.
473        target: String,
474        /// Explicit CSS selector override (bypasses LLM resolution).
475        #[serde(default)]
476        selector: Option<String>,
477        /// Milliseconds to wait after the click.
478        #[serde(default)]
479        wait_after_ms: Option<u64>,
480        /// Endpoint to use for LLM element targeting.
481        #[serde(default)]
482        endpoint: Option<String>,
483    },
484
485    /// Type text into an input element.
486    #[serde(rename = "type")]
487    Type {
488        /// Natural language description of the target input element.
489        target: String,
490        /// Text to type into the element.
491        text: String,
492        /// Explicit CSS selector override (bypasses LLM resolution).
493        #[serde(default)]
494        selector: Option<String>,
495        /// Milliseconds to wait after typing.
496        #[serde(default)]
497        wait_after_ms: Option<u64>,
498        /// Endpoint to use for LLM element targeting.
499        #[serde(default)]
500        endpoint: Option<String>,
501    },
502
503    /// Wait for an element to appear on the page.
504    #[serde(rename = "wait")]
505    Wait {
506        /// Natural language description of the element to wait for.
507        target: String,
508        /// Explicit CSS selector override (bypasses LLM resolution).
509        #[serde(default)]
510        selector: Option<String>,
511        /// Wait until the page's visible text contains this substring
512        /// (alternative to `selector`; either or both may be set — both are
513        /// required to hold when both are set).
514        #[serde(default)]
515        text: Option<String>,
516        /// Maximum milliseconds to wait (default: 10000).
517        #[serde(default)]
518        timeout_ms: Option<u64>,
519        /// Endpoint to use for LLM element targeting.
520        #[serde(default)]
521        endpoint: Option<String>,
522    },
523
524    /// Evaluate an assertion against the current page content.
525    #[serde(rename = "assert")]
526    Assert {
527        /// Reference to a named `[[definitions]]` entry.
528        #[serde(default)]
529        definition: Option<String>,
530        /// Inline predefined assertion preset (e.g. `no_error_on_page`).
531        #[serde(default)]
532        preset: Option<String>,
533        /// Inline custom LLM prompt for assertion evaluation.
534        #[serde(default)]
535        prompt: Option<String>,
536        /// Text that the `text_visible` preset checks for.
537        #[serde(default)]
538        assert_text: Option<String>,
539        /// Endpoint to use for this assertion's LLM call.
540        #[serde(default)]
541        endpoint: Option<String>,
542        /// Attach a screenshot of the current viewport to the assertion so
543        /// the LLM can evaluate visuals (overlaps, clipping, layout).
544        /// Requires the resolved endpoint to declare `vision = true`.
545        #[serde(default)]
546        screenshot: bool,
547    },
548
549    /// Take a screenshot of the current page.
550    #[serde(rename = "screenshot")]
551    Screenshot {
552        /// File path to save the screenshot (default: `screenshot.png`).
553        #[serde(default)]
554        path: Option<String>,
555    },
556
557    /// Call an A2A agent with a task.
558    #[serde(rename = "agent")]
559    Agent {
560        /// Name of the agent endpoint to call.
561        agent: String,
562        /// Task description / prompt for the agent.
563        task: String,
564        /// Optional definition name with a task template.
565        #[serde(default)]
566        definition: Option<String>,
567    },
568
569    /// Call an MCP server tool.
570    #[serde(rename = "mcp")]
571    Mcp {
572        /// Name of the MCP server endpoint.
573        server: String,
574        /// Tool name to invoke on the server.
575        tool: String,
576        /// Tool arguments as JSON.
577        #[serde(default)]
578        args: Option<serde_json::Value>,
579    },
580}