llm_browser_testkit/
scenario.rs

1//! Scenario types for human-readable browser test case definitions.
2//!
3//! Scenarios are written in [TOML](https://toml.io) and describe groups of
4//! browser interaction tests with reusable assertion definitions and
5//! configurable test-level overrides.
6//!
7//! # Structure
8//!
9//! ```toml
10//! [config]                         # Global defaults
11//! start_url = "/dashboard"
12//!
13//! [[definitions]]                  # Reusable assertion definitions
14//! name = "no_errors"
15//! preset = "no_error_on_page"
16//!
17//! [[test]]                         # Test group
18//! name = "Dashboard Smoke"
19//! start_url = "/dashboard"         # Override global start_url (optional)
20//!
21//! [[test.steps]]                   # Ordered steps — the `kind` field
22//! kind = "navigate"                # determines which other fields apply
23//! url = "/dashboard"
24//!
25//! [[test.steps]]
26//! kind = "click"
27//! target = "the Login button"      # Natural language — LLM resolves to selector
28//!
29//! [[test.steps]]
30//! kind = "assert"
31//! definition = "no_errors"
32//! ```
33//!
34//! ## Step Kinds
35//!
36//! | `kind`        | Required fields  | Optional fields                          |
37//! |---------------|-----------------|------------------------------------------|
38//! | `navigate`    | `url`           | `wait_after_ms`                          |
39//! | `click`       | `target`        | `selector`, `wait_after_ms`              |
40//! | `type`        | `target`, `text`| `selector`, `wait_after_ms`              |
41//! | `wait`        | `target`        | `selector`, `timeout_ms`                 |
42//! | `assert`      | *one of below*  | —                                        |
43//! | `screenshot`  | —               | `path`                                   |
44//!
45//! Assert steps require one of: `definition` (references a named
46//! `[[definitions]]` entry), `preset` (built-in preset name), or `prompt`
47//! (custom LLM evaluation prompt).
48
49use std::collections::HashMap;
50
51use serde::Deserialize;
52use serde_json::Value;
53
54/// Top-level scenario file, deserialized from TOML.
55#[derive(Debug, Deserialize)]
56pub struct Scenario {
57    /// Global configuration (overridable per test).
58    #[serde(default)]
59    pub config: ScenarioConfig,
60
61    /// Reusable assertion definitions referenced by name in `assert` steps.
62    #[serde(default)]
63    pub definitions: Vec<AssertDefinition>,
64
65    /// Ordered test groups to execute.
66    #[serde(default)]
67    pub test: Vec<TestGroup>,
68}
69
70/// Global scenario configuration with per-test overridable fields.
71#[derive(Debug, Deserialize, Default, Clone)]
72pub struct ScenarioConfig {
73    /// Base URL for relative navigation.
74    pub base_url: Option<String>,
75    /// LLM server base URL.
76    pub llm_url: Option<String>,
77    /// LLM model name.
78    pub llm_model: Option<String>,
79    /// LLM API key (Bearer token).
80    #[serde(default)]
81    pub llm_api_key: Option<String>,
82    /// Custom HTTP headers as JSON key-value pairs.
83    #[serde(default, deserialize_with = "deserialize_headers")]
84    pub llm_headers: HashMap<String, String>,
85    /// Run browser in headless mode.
86    pub browser_headless: Option<bool>,
87    /// HTTP / browser action timeout in seconds.
88    pub timeout_secs: Option<u64>,
89    /// Browser viewport width.
90    pub viewport_width: Option<u32>,
91    /// Browser viewport height.
92    pub viewport_height: Option<u32>,
93    /// Default URL every test auto-navigates to before running its steps.
94    pub start_url: Option<String>,
95    /// Whether to auto-navigate to `start_url` before test steps.
96    /// Disable when a test starts with click-based navigation.
97    #[serde(default = "default_auto_navigate")]
98    pub auto_navigate: bool,
99    /// LLM temperature (0.0–1.0). Lower = more deterministic.
100    #[serde(default = "default_temperature")]
101    pub temperature: f64,
102    /// Enable thinking/reasoning tokens. `None` means the provider default
103    /// is used (no `thinking` key is sent). Set to `true`/`false` to
104    /// explicitly enable or disable.
105    #[serde(default)]
106    pub thinking: Option<bool>,
107    /// Provider-specific model parameters merged into the chat completion
108    /// request body (e.g. `effort = "high"` for Anthropic).
109    #[serde(default, deserialize_with = "deserialize_model_params")]
110    pub model_params: HashMap<String, Value>,
111}
112
113fn deserialize_headers<'de, D>(deserializer: D) -> Result<HashMap<String, String>, D::Error>
114where
115    D: serde::Deserializer<'de>,
116{
117    let raw: Option<serde_json::Value> = Option::deserialize(deserializer)?;
118    let Some(json) = raw else {
119        return Ok(HashMap::new());
120    };
121    let serde_json::Value::Object(obj) = json else {
122        return Ok(HashMap::new());
123    };
124    Ok(obj
125        .into_iter()
126        .filter_map(|(k, v)| v.as_str().map(|s| (k, s.to_owned())))
127        .collect())
128}
129
130const fn default_auto_navigate() -> bool {
131    true
132}
133
134const fn default_temperature() -> f64 {
135    0.0
136}
137
138fn deserialize_model_params<'de, D>(deserializer: D) -> Result<HashMap<String, Value>, D::Error>
139where
140    D: serde::Deserializer<'de>,
141{
142    #[derive(Deserialize)]
143    #[serde(untagged)]
144    enum Raw {
145        Map(HashMap<String, Value>),
146        Table(HashMap<String, Value>),
147    }
148    let raw: Option<Raw> = Option::deserialize(deserializer)?;
149    Ok(match raw {
150        Some(Raw::Map(m) | Raw::Table(m)) => m,
151        None => HashMap::new(),
152    })
153}
154
155/// Reusable assertion definition referenced by name from `assert` steps.
156///
157/// Definitions can either reference a built-in preset via `preset`, supply a
158/// custom LLM `prompt`, or define a **custom preset** by providing both
159/// `system` and `user_template`. Custom presets support the same template
160/// variables as built-in presets: `{url}`, `{title}`, `{content}`,
161/// `{expected_text}`, `{description}`.
162#[derive(Debug, Deserialize, Clone)]
163pub struct AssertDefinition {
164    /// Unique name used to reference this definition from steps.
165    pub name: String,
166    /// Predefined assertion preset name (e.g. `no_error_on_page`, `text_visible`).
167    #[serde(default)]
168    pub preset: Option<String>,
169    /// Custom LLM prompt for assertion evaluation.
170    #[serde(default)]
171    pub prompt: Option<String>,
172    /// System prompt for a custom preset.
173    #[serde(default)]
174    pub system: Option<String>,
175    /// User template (with `{placeholders}`) for a custom preset.
176    #[serde(default)]
177    pub user_template: Option<String>,
178    /// Text that the `text_visible` preset checks for, or the
179    /// `{expected_text}` placeholder value for custom presets.
180    #[serde(default)]
181    pub assert_text: Option<String>,
182}
183
184/// A group of steps that form a single test scenario.
185#[derive(Debug, Deserialize)]
186pub struct TestGroup {
187    /// Human-readable test name.
188    pub name: String,
189    /// Override the global `start_url` for this test.
190    #[serde(default)]
191    pub start_url: Option<String>,
192    /// Override the global `auto_navigate` for this test.
193    #[serde(default)]
194    pub auto_navigate: Option<bool>,
195    /// Override the global `base_url` for this test.
196    #[serde(default)]
197    pub base_url: Option<String>,
198    /// Override the global `timeout_secs` for this test.
199    #[serde(default)]
200    pub timeout_secs: Option<u64>,
201    /// Override the global `browser_headless` for this test.
202    #[serde(default)]
203    pub browser_headless: Option<bool>,
204    /// Ordered steps to execute.
205    #[serde(default)]
206    pub steps: Vec<TestStep>,
207}
208
209/// A single step in a test. The `kind` field determines which variant is
210/// deserialized and which field constraints apply.
211#[derive(Debug, Deserialize)]
212#[serde(tag = "kind")]
213pub enum TestStep {
214    /// Navigate the browser to a URL.
215    #[serde(rename = "navigate")]
216    Navigate {
217        /// URL to navigate to (absolute, or relative to the test's base URL).
218        url: String,
219        /// Milliseconds to wait after navigation completes.
220        #[serde(default)]
221        wait_after_ms: Option<u64>,
222    },
223
224    /// Click an element described in natural language.
225    #[serde(rename = "click")]
226    Click {
227        /// Natural language description of the element. The LLM resolves this
228        /// to a CSS selector at runtime.
229        target: String,
230        /// Explicit CSS selector override (bypasses LLM resolution).
231        #[serde(default)]
232        selector: Option<String>,
233        /// Milliseconds to wait after the click.
234        #[serde(default)]
235        wait_after_ms: Option<u64>,
236    },
237
238    /// Type text into an input element.
239    #[serde(rename = "type")]
240    Type {
241        /// Natural language description of the target input element.
242        target: String,
243        /// Text to type into the element.
244        text: String,
245        /// Explicit CSS selector override (bypasses LLM resolution).
246        #[serde(default)]
247        selector: Option<String>,
248        /// Milliseconds to wait after typing.
249        #[serde(default)]
250        wait_after_ms: Option<u64>,
251    },
252
253    /// Wait for an element to appear on the page.
254    #[serde(rename = "wait")]
255    Wait {
256        /// Natural language description of the element to wait for.
257        target: String,
258        /// Explicit CSS selector override (bypasses LLM resolution).
259        #[serde(default)]
260        selector: Option<String>,
261        /// Maximum milliseconds to wait (default: 10000).
262        #[serde(default)]
263        timeout_ms: Option<u64>,
264    },
265
266    /// Evaluate an assertion against the current page content.
267    #[serde(rename = "assert")]
268    Assert {
269        /// Reference to a named `[[definitions]]` entry.
270        #[serde(default)]
271        definition: Option<String>,
272        /// Inline predefined assertion preset (e.g. `no_error_on_page`).
273        #[serde(default)]
274        preset: Option<String>,
275        /// Inline custom LLM prompt for assertion evaluation.
276        #[serde(default)]
277        prompt: Option<String>,
278        /// Text that the `text_visible` preset checks for.
279        #[serde(default)]
280        assert_text: Option<String>,
281    },
282
283    /// Take a screenshot of the current page.
284    #[serde(rename = "screenshot")]
285    Screenshot {
286        /// File path to save the screenshot (default: `screenshot.png`).
287        #[serde(default)]
288        path: Option<String>,
289    },
290}
llm_browser_testkit/scenario.rs

llm_browser_testkit/
scenario.rs
