Skip to main content

llm_browser_testkit/
scenario.rs

1//! Scenario types for human-readable browser test case definitions.
2//!
3//! Scenarios are written in [TOML](https://toml.io) and describe groups of
4//! browser interaction tests with reusable assertion definitions and
5//! configurable test-level overrides.
6//!
7//! # Structure
8//!
9//! ```toml
10//! [config]                         # Global defaults
11//! start_url = "/dashboard"
12//!
13//! [[definitions]]                  # Reusable assertion definitions
14//! name = "no_errors"
15//! preset = "no_error_on_page"
16//!
17//! [[test]]                         # Test group
18//! name = "Dashboard Smoke"
19//! start_url = "/dashboard"         # Override global start_url (optional)
20//!
21//! [[test.steps]]                   # Ordered steps — the `kind` field
22//! kind = "navigate"                # determines which other fields apply
23//! url = "/dashboard"
24//!
25//! [[test.steps]]
26//! kind = "click"
27//! target = "the Login button"      # Natural language — LLM resolves to selector
28//!
29//! [[test.steps]]
30//! kind = "assert"
31//! definition = "no_errors"
32//! ```
33//!
34//! ## Step Kinds
35//!
36//! | `kind`        | Required fields  | Optional fields                          |
37//! |---------------|-----------------|------------------------------------------|
38//! | `navigate`    | `url`           | `wait_after_ms`                          |
39//! | `click`       | `target`        | `selector`, `wait_after_ms`              |
40//! | `type`        | `target`, `text`| `selector`, `wait_after_ms`              |
41//! | `wait`        | `target`        | `selector`, `timeout_ms`                 |
42//! | `assert`      | *one of below*  | —                                        |
43//! | `screenshot`  | —               | `path`                                   |
44//! | `agent`       | `agent`, `task` | —                                        |
45//! | `mcp`         | `server`, `tool`| `args`                                   |
46//!
47//! Assert steps require one of: `definition` (references a named
48//! `[[definitions]]` entry), `preset` (built-in preset name), or `prompt`
49//! (custom LLM evaluation prompt).
50
51use std::collections::HashMap;
52
53use serde::Deserialize;
54use serde_json::Value;
55
56/// Top-level scenario file, deserialized from TOML.
57#[derive(Debug, Deserialize)]
58pub struct Scenario {
59    /// Global configuration (overridable per test).
60    #[serde(default)]
61    pub config: ScenarioConfig,
62
63    /// Reusable assertion definitions referenced by name in `assert` steps.
64    #[serde(default)]
65    pub definitions: Vec<AssertDefinition>,
66
67    /// Ordered test groups to execute.
68    #[serde(default)]
69    pub test: Vec<TestGroup>,
70}
71
72/// Global scenario configuration with per-test overridable fields.
73#[derive(Debug, Deserialize, Default, Clone)]
74pub struct ScenarioConfig {
75    /// Base URL for relative navigation.
76    #[serde(default)]
77    pub base_url: Option<String>,
78    /// LLM server base URL (deprecated; prefer `[config.endpoints]`).
79    #[serde(default)]
80    pub llm_url: Option<String>,
81    /// LLM model name (deprecated; prefer `[config.endpoints]`).
82    #[serde(default)]
83    pub llm_model: Option<String>,
84    /// LLM API key (Bearer token).
85    #[serde(default)]
86    pub llm_api_key: Option<String>,
87    /// Custom HTTP headers as JSON key-value pairs.
88    #[serde(default, deserialize_with = "deserialize_headers")]
89    pub llm_headers: HashMap<String, String>,
90    /// Run browser in headless mode.
91    #[serde(default)]
92    pub browser_headless: Option<bool>,
93    /// HTTP / browser action timeout in seconds.
94    #[serde(default)]
95    pub timeout_secs: Option<u64>,
96    /// Browser viewport width.
97    #[serde(default)]
98    pub viewport_width: Option<u32>,
99    /// Browser viewport height.
100    #[serde(default)]
101    pub viewport_height: Option<u32>,
102    /// Default URL every test auto-navigates to before running its steps.
103    #[serde(default)]
104    pub start_url: Option<String>,
105    /// Whether to auto-navigate to `start_url` before test steps.
106    ///
107    /// Disable when a test starts with click-based navigation.
108    #[serde(default = "default_auto_navigate")]
109    pub auto_navigate: bool,
110    /// LLM temperature (0.0–1.0). Lower = more deterministic.
111    #[serde(default = "default_temperature")]
112    pub temperature: f64,
113    /// Enable thinking/reasoning tokens. `None` means the provider default
114    /// is used (no `thinking` key is sent). Set to `true`/`false` to
115    /// explicitly enable or disable.
116    #[serde(default)]
117    pub thinking: Option<bool>,
118    /// Provider-specific model parameters merged into the chat completion
119    /// request body (e.g. `effort = "high"` for Anthropic).
120    #[serde(default, deserialize_with = "deserialize_model_params")]
121    pub model_params: HashMap<String, Value>,
122    /// Named endpoints (LLM, MCP, A2A agents) with pricing.
123    #[serde(default)]
124    pub endpoints: HashMap<String, EndpointConfig>,
125    /// Global and per-test budgets for cost/token/call limits.
126    #[serde(default)]
127    pub budgets: BudgetsConfig,
128    /// MCP server exposure configuration.
129    #[serde(default)]
130    pub mcp_server: Option<McpServerConfig>,
131    /// A2A agent server exposure configuration.
132    #[serde(default)]
133    pub a2a_server: Option<A2aServerConfig>,
134}
135
136/// A named endpoint definition with pricing.
137#[derive(Debug, Deserialize, Clone, Default)]
138pub struct EndpointConfig {
139    /// Endpoint type: `llm`, `mcp`, or `a2a`.
140    #[serde(rename = "type")]
141    pub endpoint_type: EndpointType,
142    /// Base URL for the endpoint.
143    #[serde(default)]
144    pub url: Option<String>,
145    /// Model name (LLM endpoints only).
146    #[serde(default)]
147    pub model: Option<String>,
148    /// API key / bearer token.
149    #[serde(default)]
150    pub api_key: Option<String>,
151    /// Custom HTTP headers as JSON key-value pairs.
152    #[serde(default, deserialize_with = "deserialize_headers")]
153    pub headers: HashMap<String, String>,
154    /// Pricing configuration.
155    #[serde(default)]
156    pub pricing: Option<PricingConfig>,
157    /// Task types this endpoint serves by default
158    /// (e.g. `["targeting", "assertion"]`).
159    #[serde(default)]
160    pub default_for: Vec<String>,
161    /// Command to launch an MCP server subprocess (stdio transport).
162    #[serde(default)]
163    pub command: Option<String>,
164    /// Arguments for the MCP server command.
165    #[serde(default)]
166    pub args: Vec<String>,
167}
168
169/// Type discriminator for endpoint configuration.
170#[derive(Debug, Deserialize, Clone, PartialEq, Eq, Default)]
171#[serde(rename_all = "lowercase")]
172pub enum EndpointType {
173    /// OpenAI-compatible LLM API.
174    #[default]
175    Llm,
176    /// Model Context Protocol server.
177    Mcp,
178    /// Agent-to-Agent protocol agent.
179    A2a,
180}
181
182/// Pricing configuration for an endpoint.
183#[derive(Debug, Deserialize, Clone, Default)]
184pub struct PricingConfig {
185    /// Cost per 1M input tokens (USD).
186    #[serde(default)]
187    pub input_per_1m_tokens: f64,
188    /// Cost per 1M output tokens (USD).
189    #[serde(default)]
190    pub output_per_1m_tokens: f64,
191    /// Flat cost per call (USD), used for MCP/agent endpoints.
192    #[serde(default)]
193    pub per_call: f64,
194}
195
196/// Budget limits for test execution.
197#[derive(Debug, Deserialize, Clone, Default)]
198pub struct BudgetsConfig {
199    /// Global budget across all tests in the scenario.
200    #[serde(default)]
201    pub global: Option<BudgetDef>,
202    /// Default per-test budget. Individual tests can override.
203    #[serde(default)]
204    pub per_test_default: Option<BudgetDef>,
205}
206
207/// A budget definition with limits and enforcement mode.
208#[derive(Debug, Deserialize, Clone)]
209pub struct BudgetDef {
210    /// Maximum cost in USD.
211    #[serde(default)]
212    pub max_cost: Option<f64>,
213    /// Maximum total tokens (input + output).
214    #[serde(default)]
215    pub max_tokens: Option<u64>,
216    /// Maximum number of calls (LLM, MCP, agent combined).
217    #[serde(default)]
218    pub max_calls: Option<u64>,
219    /// Enforcement mode: `hard` (abort) or `soft` (warn and continue).
220    #[serde(default)]
221    pub enforcement: Option<BudgetEnforcement>,
222}
223
224/// Budget enforcement strategy.
225#[derive(Debug, Deserialize, Clone, PartialEq, Eq)]
226#[serde(rename_all = "lowercase")]
227pub enum BudgetEnforcement {
228    /// Abort the test or run when budget is exceeded.
229    Hard,
230    /// Log a warning but continue execution.
231    Soft,
232}
233
234/// MCP server exposure configuration.
235#[derive(Debug, Deserialize, Clone)]
236pub struct McpServerConfig {
237    /// Whether to enable the embedded MCP server.
238    #[serde(default)]
239    pub enabled: bool,
240    /// Port to listen on.
241    #[serde(default = "default_mcp_port")]
242    pub port: u16,
243}
244
245const fn default_mcp_port() -> u16 {
246    3000
247}
248
249/// A2A agent server exposure configuration.
250#[derive(Debug, Deserialize, Clone)]
251pub struct A2aServerConfig {
252    /// Whether to enable the embedded A2A agent server.
253    #[serde(default)]
254    pub enabled: bool,
255    /// Port to listen on.
256    #[serde(default = "default_a2a_port")]
257    pub port: u16,
258}
259
260const fn default_a2a_port() -> u16 {
261    3100
262}
263
264fn deserialize_headers<'de, D>(deserializer: D) -> Result<HashMap<String, String>, D::Error>
265where
266    D: serde::Deserializer<'de>,
267{
268    let raw: Option<serde_json::Value> = Option::deserialize(deserializer)?;
269    let Some(json) = raw else {
270        return Ok(HashMap::new());
271    };
272    let serde_json::Value::Object(obj) = json else {
273        return Ok(HashMap::new());
274    };
275    Ok(obj
276        .into_iter()
277        .filter_map(|(k, v)| v.as_str().map(|s| (k, s.to_owned())))
278        .collect())
279}
280
281const fn default_auto_navigate() -> bool {
282    true
283}
284
285const fn default_temperature() -> f64 {
286    0.0
287}
288
289fn deserialize_model_params<'de, D>(deserializer: D) -> Result<HashMap<String, Value>, D::Error>
290where
291    D: serde::Deserializer<'de>,
292{
293    #[derive(Deserialize)]
294    #[serde(untagged)]
295    enum Raw {
296        Map(HashMap<String, Value>),
297        Table(HashMap<String, Value>),
298    }
299    let raw: Option<Raw> = Option::deserialize(deserializer)?;
300    Ok(match raw {
301        Some(Raw::Map(m) | Raw::Table(m)) => m,
302        None => HashMap::new(),
303    })
304}
305
306/// Reusable assertion definition referenced by name from `assert` steps.
307///
308/// Definitions can either reference a built-in preset via `preset`, supply a
309/// custom LLM `prompt`, or define a **custom preset** by providing both
310/// `system` and `user_template`. Custom presets support the same template
311/// variables as built-in presets: `{url}`, `{title}`, `{content}`,
312/// `{expected_text}`, `{description}`.
313#[derive(Debug, Deserialize, Clone)]
314pub struct AssertDefinition {
315    /// Unique name used to reference this definition from steps.
316    pub name: String,
317    /// Predefined assertion preset name
318    /// (e.g. `no_error_on_page`, `text_visible`).
319    #[serde(default)]
320    pub preset: Option<String>,
321    /// Custom LLM prompt for assertion evaluation.
322    #[serde(default)]
323    pub prompt: Option<String>,
324    /// System prompt for a custom preset.
325    #[serde(default)]
326    pub system: Option<String>,
327    /// User template (with `{placeholders}`) for a custom preset.
328    #[serde(default)]
329    pub user_template: Option<String>,
330    /// Text that the `text_visible` preset checks for, or the
331    /// `{expected_text}` placeholder value for custom presets.
332    #[serde(default)]
333    pub assert_text: Option<String>,
334    /// Agent endpoint to call for this assertion.
335    #[serde(default)]
336    pub agent: Option<String>,
337    /// Agent task template for this assertion.
338    #[serde(default)]
339    pub task_template: Option<String>,
340}
341
342/// A group of steps that form a single test scenario.
343#[derive(Debug, Deserialize)]
344pub struct TestGroup {
345    /// Human-readable test name.
346    pub name: String,
347    /// Override the global `start_url` for this test.
348    #[serde(default)]
349    pub start_url: Option<String>,
350    /// Override the global `auto_navigate` for this test.
351    #[serde(default)]
352    pub auto_navigate: Option<bool>,
353    /// Override the global `base_url` for this test.
354    #[serde(default)]
355    pub base_url: Option<String>,
356    /// Override the global `timeout_secs` for this test.
357    #[serde(default)]
358    pub timeout_secs: Option<u64>,
359    /// Override the global `browser_headless` for this test.
360    #[serde(default)]
361    pub browser_headless: Option<bool>,
362    /// Per-test budget override.
363    #[serde(default)]
364    pub budget: Option<BudgetDef>,
365    /// Endpoint to use for all steps in this test (can be overridden
366    /// per-step).
367    #[serde(default)]
368    pub endpoint: Option<String>,
369    /// Ordered steps to execute.
370    #[serde(default)]
371    pub steps: Vec<TestStep>,
372}
373
374/// A single step in a test. The `kind` field determines which variant is
375/// deserialized and which field constraints apply.
376#[derive(Debug, Deserialize)]
377#[serde(tag = "kind")]
378pub enum TestStep {
379    /// Navigate the browser to a URL.
380    #[serde(rename = "navigate")]
381    Navigate {
382        /// URL to navigate to (absolute, or relative to the test's base
383        /// URL).
384        url: String,
385        /// Milliseconds to wait after navigation completes.
386        #[serde(default)]
387        wait_after_ms: Option<u64>,
388    },
389
390    /// Click an element described in natural language.
391    #[serde(rename = "click")]
392    Click {
393        /// Natural language description of the element. The LLM resolves
394        /// this to a CSS selector at runtime.
395        target: String,
396        /// Explicit CSS selector override (bypasses LLM resolution).
397        #[serde(default)]
398        selector: Option<String>,
399        /// Milliseconds to wait after the click.
400        #[serde(default)]
401        wait_after_ms: Option<u64>,
402        /// Endpoint to use for LLM element targeting.
403        #[serde(default)]
404        endpoint: Option<String>,
405    },
406
407    /// Type text into an input element.
408    #[serde(rename = "type")]
409    Type {
410        /// Natural language description of the target input element.
411        target: String,
412        /// Text to type into the element.
413        text: String,
414        /// Explicit CSS selector override (bypasses LLM resolution).
415        #[serde(default)]
416        selector: Option<String>,
417        /// Milliseconds to wait after typing.
418        #[serde(default)]
419        wait_after_ms: Option<u64>,
420        /// Endpoint to use for LLM element targeting.
421        #[serde(default)]
422        endpoint: Option<String>,
423    },
424
425    /// Wait for an element to appear on the page.
426    #[serde(rename = "wait")]
427    Wait {
428        /// Natural language description of the element to wait for.
429        target: String,
430        /// Explicit CSS selector override (bypasses LLM resolution).
431        #[serde(default)]
432        selector: Option<String>,
433        /// Maximum milliseconds to wait (default: 10000).
434        #[serde(default)]
435        timeout_ms: Option<u64>,
436        /// Endpoint to use for LLM element targeting.
437        #[serde(default)]
438        endpoint: Option<String>,
439    },
440
441    /// Evaluate an assertion against the current page content.
442    #[serde(rename = "assert")]
443    Assert {
444        /// Reference to a named `[[definitions]]` entry.
445        #[serde(default)]
446        definition: Option<String>,
447        /// Inline predefined assertion preset (e.g. `no_error_on_page`).
448        #[serde(default)]
449        preset: Option<String>,
450        /// Inline custom LLM prompt for assertion evaluation.
451        #[serde(default)]
452        prompt: Option<String>,
453        /// Text that the `text_visible` preset checks for.
454        #[serde(default)]
455        assert_text: Option<String>,
456        /// Endpoint to use for this assertion's LLM call.
457        #[serde(default)]
458        endpoint: Option<String>,
459    },
460
461    /// Take a screenshot of the current page.
462    #[serde(rename = "screenshot")]
463    Screenshot {
464        /// File path to save the screenshot (default: `screenshot.png`).
465        #[serde(default)]
466        path: Option<String>,
467    },
468
469    /// Call an A2A agent with a task.
470    #[serde(rename = "agent")]
471    Agent {
472        /// Name of the agent endpoint to call.
473        agent: String,
474        /// Task description / prompt for the agent.
475        task: String,
476        /// Optional definition name with a task template.
477        #[serde(default)]
478        definition: Option<String>,
479    },
480
481    /// Call an MCP server tool.
482    #[serde(rename = "mcp")]
483    Mcp {
484        /// Name of the MCP server endpoint.
485        server: String,
486        /// Tool name to invoke on the server.
487        tool: String,
488        /// Tool arguments as JSON.
489        #[serde(default)]
490        args: Option<serde_json::Value>,
491    },
492}