Skip to main content

llm_browser_testkit/
scenario.rs

1//! Scenario types for human-readable browser test case definitions.
2//!
3//! Scenarios are written in [TOML](https://toml.io) and describe groups of
4//! browser interaction tests with reusable assertion definitions and
5//! configurable test-level overrides.
6//!
7//! # Structure
8//!
9//! ```toml
10//! [config]                         # Global defaults
11//! start_url = "/dashboard"
12//!
13//! [[definitions]]                  # Reusable assertion definitions
14//! name = "no_errors"
15//! preset = "no_error_on_page"
16//!
17//! [[test]]                         # Test group
18//! name = "Dashboard Smoke"
19//! start_url = "/dashboard"         # Override global start_url (optional)
20//!
21//! [[test.steps]]                   # Ordered steps — the `kind` field
22//! kind = "navigate"                # determines which other fields apply
23//! url = "/dashboard"
24//!
25//! [[test.steps]]
26//! kind = "click"
27//! target = "the Login button"      # Natural language — LLM resolves to selector
28//!
29//! [[test.steps]]
30//! kind = "assert"
31//! definition = "no_errors"
32//! ```
33//!
34//! ## Step Kinds
35//!
36//! | `kind`        | Required fields  | Optional fields                          |
37//! |---------------|-----------------|------------------------------------------|
38//! | `navigate`    | `url`           | `wait_after_ms`                          |
39//! | `click`       | `target`        | `selector`, `wait_after_ms`              |
40//! | `type`        | `target`, `text`| `selector`, `wait_after_ms`              |
41//! | `wait`        | `target`        | `selector`, `timeout_ms`                 |
42//! | `assert`      | *one of below*  | —                                        |
43//! | `screenshot`  | —               | `path`                                   |
44//! | `agent`       | `agent`, `task` | —                                        |
45//! | `mcp`         | `server`, `tool`| `args`                                   |
46//!
47//! Assert steps require one of: `definition` (references a named
48//! `[[definitions]]` entry), `preset` (built-in preset name), or `prompt`
49//! (custom LLM evaluation prompt).
50
51use std::collections::HashMap;
52
53use serde::Deserialize;
54use serde_json::Value;
55
56/// Top-level scenario file, deserialized from TOML.
57#[derive(Debug, Deserialize)]
58pub struct Scenario {
59    /// Global configuration (overridable per test).
60    #[serde(default)]
61    pub config: ScenarioConfig,
62
63    /// Reusable assertion definitions referenced by name in `assert` steps.
64    #[serde(default)]
65    pub definitions: Vec<AssertDefinition>,
66
67    /// Ordered test groups to execute.
68    #[serde(default)]
69    pub test: Vec<TestGroup>,
70}
71
72/// Global scenario configuration with per-test overridable fields.
73#[derive(Debug, Deserialize, Default, Clone)]
74pub struct ScenarioConfig {
75    /// Base URL for relative navigation.
76    #[serde(default)]
77    pub base_url: Option<String>,
78    /// LLM server base URL (deprecated; prefer `[config.endpoints]`).
79    #[serde(default)]
80    pub llm_url: Option<String>,
81    /// LLM model name (deprecated; prefer `[config.endpoints]`).
82    #[serde(default)]
83    pub llm_model: Option<String>,
84    /// LLM API key (Bearer token).
85    #[serde(default)]
86    pub llm_api_key: Option<String>,
87    /// Custom HTTP headers as JSON key-value pairs.
88    #[serde(default, deserialize_with = "deserialize_headers")]
89    pub llm_headers: HashMap<String, String>,
90    /// Run browser in headless mode.
91    #[serde(default)]
92    pub browser_headless: Option<bool>,
93    /// HTTP / browser action timeout in seconds.
94    #[serde(default)]
95    pub timeout_secs: Option<u64>,
96    /// Browser viewport width.
97    #[serde(default)]
98    pub viewport_width: Option<u32>,
99    /// Browser viewport height.
100    #[serde(default)]
101    pub viewport_height: Option<u32>,
102    /// Default URL every test auto-navigates to before running its steps.
103    #[serde(default)]
104    pub start_url: Option<String>,
105    /// Whether to auto-navigate to `start_url` before test steps.
106    ///
107    /// Disable when a test starts with click-based navigation.
108    #[serde(default = "default_auto_navigate")]
109    pub auto_navigate: bool,
110    /// LLM temperature (0.0–1.0). Lower = more deterministic.
111    #[serde(default = "default_temperature")]
112    pub temperature: f64,
113    /// Enable thinking/reasoning tokens. `None` means the provider default
114    /// is used (no `thinking` key is sent). Set to `true`/`false` to
115    /// explicitly enable or disable.
116    #[serde(default)]
117    pub thinking: Option<bool>,
118    /// Provider-specific model parameters merged into the chat completion
119    /// request body (e.g. `effort = "high"` for Anthropic).
120    #[serde(default, deserialize_with = "deserialize_model_params")]
121    pub model_params: HashMap<String, Value>,
122    /// Named endpoints (LLM, MCP, A2A agents) with pricing.
123    #[serde(default)]
124    pub endpoints: HashMap<String, EndpointConfig>,
125    /// Global and per-test budgets for cost/token/call limits.
126    #[serde(default)]
127    pub budgets: BudgetsConfig,
128    /// MCP server exposure configuration.
129    #[serde(default)]
130    pub mcp_server: Option<McpServerConfig>,
131    /// A2A agent server exposure configuration.
132    #[serde(default)]
133    pub a2a_server: Option<A2aServerConfig>,
134    /// Whether to continue running the remaining steps of a test after a
135    /// step fails. Default `false` = fail fast: the first failed step ends
136    /// the test and the rest are reported as skipped. Set to `true` to run
137    /// every step (more diagnostics, more LLM cost on broken apps).
138    #[serde(default)]
139    pub continue_on_failure: bool,
140    /// Directory for failure artifacts (screenshots, page snapshots).
141    /// Defaults to `artifacts`.
142    #[serde(default)]
143    pub artifacts_dir: Option<String>,
144}
145
146/// A named endpoint definition with pricing.
147#[derive(Debug, Deserialize, Clone, Default)]
148pub struct EndpointConfig {
149    /// Endpoint type: `llm`, `mcp`, or `a2a`.
150    #[serde(rename = "type")]
151    pub endpoint_type: EndpointType,
152    /// Base URL for the endpoint.
153    #[serde(default)]
154    pub url: Option<String>,
155    /// Model name (LLM endpoints only).
156    #[serde(default)]
157    pub model: Option<String>,
158    /// API key / bearer token.
159    #[serde(default)]
160    pub api_key: Option<String>,
161    /// Custom HTTP headers as JSON key-value pairs.
162    #[serde(default, deserialize_with = "deserialize_headers")]
163    pub headers: HashMap<String, String>,
164    /// Pricing configuration.
165    #[serde(default)]
166    pub pricing: Option<PricingConfig>,
167    /// Task types this endpoint serves by default
168    /// (e.g. `["targeting", "assertion"]`).
169    #[serde(default)]
170    pub default_for: Vec<String>,
171    /// Command to launch an MCP server subprocess (stdio transport).
172    #[serde(default)]
173    pub command: Option<String>,
174    /// Arguments for the MCP server command.
175    #[serde(default)]
176    pub args: Vec<String>,
177}
178
179/// Type discriminator for endpoint configuration.
180#[derive(Debug, Deserialize, Clone, PartialEq, Eq, Default)]
181#[serde(rename_all = "lowercase")]
182pub enum EndpointType {
183    /// OpenAI-compatible LLM API.
184    #[default]
185    Llm,
186    /// Model Context Protocol server.
187    Mcp,
188    /// Agent-to-Agent protocol agent.
189    A2a,
190}
191
192/// Pricing configuration for an endpoint.
193#[derive(Debug, Deserialize, Clone, Default)]
194pub struct PricingConfig {
195    /// Cost per 1M input tokens (USD).
196    #[serde(default)]
197    pub input_per_1m_tokens: f64,
198    /// Cost per 1M output tokens (USD).
199    #[serde(default)]
200    pub output_per_1m_tokens: f64,
201    /// Flat cost per call (USD), used for MCP/agent endpoints.
202    #[serde(default)]
203    pub per_call: f64,
204}
205
206/// Budget limits for test execution.
207#[derive(Debug, Deserialize, Clone, Default)]
208pub struct BudgetsConfig {
209    /// Global budget across all tests in the scenario.
210    #[serde(default)]
211    pub global: Option<BudgetDef>,
212    /// Default per-test budget. Individual tests can override.
213    #[serde(default)]
214    pub per_test_default: Option<BudgetDef>,
215}
216
217/// A budget definition with limits and enforcement mode.
218#[derive(Debug, Deserialize, Clone)]
219pub struct BudgetDef {
220    /// Maximum cost in USD.
221    #[serde(default)]
222    pub max_cost: Option<f64>,
223    /// Maximum total tokens (input + output).
224    #[serde(default)]
225    pub max_tokens: Option<u64>,
226    /// Maximum number of calls (LLM, MCP, agent combined).
227    #[serde(default)]
228    pub max_calls: Option<u64>,
229    /// Enforcement mode: `hard` (abort) or `soft` (warn and continue).
230    #[serde(default)]
231    pub enforcement: Option<BudgetEnforcement>,
232}
233
234/// Budget enforcement strategy.
235#[derive(Debug, Deserialize, Clone, PartialEq, Eq)]
236#[serde(rename_all = "lowercase")]
237pub enum BudgetEnforcement {
238    /// Abort the test or run when budget is exceeded.
239    Hard,
240    /// Log a warning but continue execution.
241    Soft,
242}
243
244/// MCP server exposure configuration.
245#[derive(Debug, Deserialize, Clone)]
246pub struct McpServerConfig {
247    /// Whether to enable the embedded MCP server.
248    #[serde(default)]
249    pub enabled: bool,
250    /// Port to listen on.
251    #[serde(default = "default_mcp_port")]
252    pub port: u16,
253}
254
255const fn default_mcp_port() -> u16 {
256    3000
257}
258
259/// A2A agent server exposure configuration.
260#[derive(Debug, Deserialize, Clone)]
261pub struct A2aServerConfig {
262    /// Whether to enable the embedded A2A agent server.
263    #[serde(default)]
264    pub enabled: bool,
265    /// Port to listen on.
266    #[serde(default = "default_a2a_port")]
267    pub port: u16,
268}
269
270const fn default_a2a_port() -> u16 {
271    3100
272}
273
274fn deserialize_headers<'de, D>(deserializer: D) -> Result<HashMap<String, String>, D::Error>
275where
276    D: serde::Deserializer<'de>,
277{
278    let raw: Option<serde_json::Value> = Option::deserialize(deserializer)?;
279    let Some(json) = raw else {
280        return Ok(HashMap::new());
281    };
282    let serde_json::Value::Object(obj) = json else {
283        return Ok(HashMap::new());
284    };
285    Ok(obj
286        .into_iter()
287        .filter_map(|(k, v)| v.as_str().map(|s| (k, s.to_owned())))
288        .collect())
289}
290
291const fn default_auto_navigate() -> bool {
292    true
293}
294
295const fn default_temperature() -> f64 {
296    0.0
297}
298
299fn deserialize_model_params<'de, D>(deserializer: D) -> Result<HashMap<String, Value>, D::Error>
300where
301    D: serde::Deserializer<'de>,
302{
303    #[derive(Deserialize)]
304    #[serde(untagged)]
305    enum Raw {
306        Map(HashMap<String, Value>),
307        Table(HashMap<String, Value>),
308    }
309    let raw: Option<Raw> = Option::deserialize(deserializer)?;
310    Ok(match raw {
311        Some(Raw::Map(m) | Raw::Table(m)) => m,
312        None => HashMap::new(),
313    })
314}
315
316/// Reusable assertion definition referenced by name from `assert` steps.
317///
318/// Definitions can either reference a built-in preset via `preset`, supply a
319/// custom LLM `prompt`, or define a **custom preset** by providing both
320/// `system` and `user_template`. Custom presets support the same template
321/// variables as built-in presets: `{url}`, `{title}`, `{content}`,
322/// `{expected_text}`, `{description}`.
323#[derive(Debug, Deserialize, Clone)]
324pub struct AssertDefinition {
325    /// Unique name used to reference this definition from steps.
326    pub name: String,
327    /// Predefined assertion preset name
328    /// (e.g. `no_error_on_page`, `text_visible`).
329    #[serde(default)]
330    pub preset: Option<String>,
331    /// Custom LLM prompt for assertion evaluation.
332    #[serde(default)]
333    pub prompt: Option<String>,
334    /// System prompt for a custom preset.
335    #[serde(default)]
336    pub system: Option<String>,
337    /// User template (with `{placeholders}`) for a custom preset.
338    #[serde(default)]
339    pub user_template: Option<String>,
340    /// Text that the `text_visible` preset checks for, or the
341    /// `{expected_text}` placeholder value for custom presets.
342    #[serde(default)]
343    pub assert_text: Option<String>,
344    /// Agent endpoint to call for this assertion.
345    #[serde(default)]
346    pub agent: Option<String>,
347    /// Agent task template for this assertion.
348    #[serde(default)]
349    pub task_template: Option<String>,
350}
351
352/// A group of steps that form a single test scenario.
353#[derive(Debug, Deserialize)]
354pub struct TestGroup {
355    /// Human-readable test name.
356    pub name: String,
357    /// Override the global `start_url` for this test.
358    #[serde(default)]
359    pub start_url: Option<String>,
360    /// Override the global `auto_navigate` for this test.
361    #[serde(default)]
362    pub auto_navigate: Option<bool>,
363    /// Override the global `base_url` for this test.
364    #[serde(default)]
365    pub base_url: Option<String>,
366    /// Override the global `timeout_secs` for this test.
367    #[serde(default)]
368    pub timeout_secs: Option<u64>,
369    /// Override the global `browser_headless` for this test.
370    #[serde(default)]
371    pub browser_headless: Option<bool>,
372    /// Per-test budget override.
373    #[serde(default)]
374    pub budget: Option<BudgetDef>,
375    /// Endpoint to use for all steps in this test (can be overridden
376    /// per-step).
377    #[serde(default)]
378    pub endpoint: Option<String>,
379    /// Ordered steps to execute.
380    #[serde(default)]
381    pub steps: Vec<TestStep>,
382}
383
384/// A single step in a test. The `kind` field determines which variant is
385/// deserialized and which field constraints apply.
386#[derive(Debug, Deserialize)]
387#[serde(tag = "kind")]
388pub enum TestStep {
389    /// Navigate the browser to a URL.
390    #[serde(rename = "navigate")]
391    Navigate {
392        /// URL to navigate to (absolute, or relative to the test's base
393        /// URL).
394        url: String,
395        /// Milliseconds to wait after navigation completes.
396        #[serde(default)]
397        wait_after_ms: Option<u64>,
398    },
399
400    /// Click an element described in natural language.
401    #[serde(rename = "click")]
402    Click {
403        /// Natural language description of the element. The LLM resolves
404        /// this to a CSS selector at runtime.
405        target: String,
406        /// Explicit CSS selector override (bypasses LLM resolution).
407        #[serde(default)]
408        selector: Option<String>,
409        /// Milliseconds to wait after the click.
410        #[serde(default)]
411        wait_after_ms: Option<u64>,
412        /// Endpoint to use for LLM element targeting.
413        #[serde(default)]
414        endpoint: Option<String>,
415    },
416
417    /// Type text into an input element.
418    #[serde(rename = "type")]
419    Type {
420        /// Natural language description of the target input element.
421        target: String,
422        /// Text to type into the element.
423        text: String,
424        /// Explicit CSS selector override (bypasses LLM resolution).
425        #[serde(default)]
426        selector: Option<String>,
427        /// Milliseconds to wait after typing.
428        #[serde(default)]
429        wait_after_ms: Option<u64>,
430        /// Endpoint to use for LLM element targeting.
431        #[serde(default)]
432        endpoint: Option<String>,
433    },
434
435    /// Wait for an element to appear on the page.
436    #[serde(rename = "wait")]
437    Wait {
438        /// Natural language description of the element to wait for.
439        target: String,
440        /// Explicit CSS selector override (bypasses LLM resolution).
441        #[serde(default)]
442        selector: Option<String>,
443        /// Wait until the page's visible text contains this substring
444        /// (alternative to `selector`; either or both may be set — both are
445        /// required to hold when both are set).
446        #[serde(default)]
447        text: Option<String>,
448        /// Maximum milliseconds to wait (default: 10000).
449        #[serde(default)]
450        timeout_ms: Option<u64>,
451        /// Endpoint to use for LLM element targeting.
452        #[serde(default)]
453        endpoint: Option<String>,
454    },
455
456    /// Evaluate an assertion against the current page content.
457    #[serde(rename = "assert")]
458    Assert {
459        /// Reference to a named `[[definitions]]` entry.
460        #[serde(default)]
461        definition: Option<String>,
462        /// Inline predefined assertion preset (e.g. `no_error_on_page`).
463        #[serde(default)]
464        preset: Option<String>,
465        /// Inline custom LLM prompt for assertion evaluation.
466        #[serde(default)]
467        prompt: Option<String>,
468        /// Text that the `text_visible` preset checks for.
469        #[serde(default)]
470        assert_text: Option<String>,
471        /// Endpoint to use for this assertion's LLM call.
472        #[serde(default)]
473        endpoint: Option<String>,
474    },
475
476    /// Take a screenshot of the current page.
477    #[serde(rename = "screenshot")]
478    Screenshot {
479        /// File path to save the screenshot (default: `screenshot.png`).
480        #[serde(default)]
481        path: Option<String>,
482    },
483
484    /// Call an A2A agent with a task.
485    #[serde(rename = "agent")]
486    Agent {
487        /// Name of the agent endpoint to call.
488        agent: String,
489        /// Task description / prompt for the agent.
490        task: String,
491        /// Optional definition name with a task template.
492        #[serde(default)]
493        definition: Option<String>,
494    },
495
496    /// Call an MCP server tool.
497    #[serde(rename = "mcp")]
498    Mcp {
499        /// Name of the MCP server endpoint.
500        server: String,
501        /// Tool name to invoke on the server.
502        tool: String,
503        /// Tool arguments as JSON.
504        #[serde(default)]
505        args: Option<serde_json::Value>,
506    },
507}