llm_browser_testkit/
scenario.rs

1//! Scenario types for human-readable browser test case definitions.
2//!
3//! Scenarios are written in [TOML](https://toml.io) and describe groups of
4//! browser interaction tests with reusable assertion definitions and
5//! configurable test-level overrides.
6//!
7//! # Structure
8//!
9//! ```toml
10//! [config]                         # Global defaults
11//! start_url = "/dashboard"
12//!
13//! [[definitions]]                  # Reusable assertion definitions
14//! name = "no_errors"
15//! preset = "no_error_on_page"
16//!
17//! [[test]]                         # Test group
18//! name = "Dashboard Smoke"
19//! start_url = "/dashboard"         # Override global start_url (optional)
20//!
21//! [[test.steps]]                   # Ordered steps — the `kind` field
22//! kind = "navigate"                # determines which other fields apply
23//! url = "/dashboard"
24//!
25//! [[test.steps]]
26//! kind = "click"
27//! target = "the Login button"      # Natural language — LLM resolves to selector
28//!
29//! [[test.steps]]
30//! kind = "assert"
31//! definition = "no_errors"
32//! ```
33//!
34//! ## Step Kinds
35//!
36//! | `kind`        | Required fields  | Optional fields                          |
37//! |---------------|-----------------|------------------------------------------|
38//! | `navigate`    | `url`           | `wait_after_ms`                          |
39//! | `click`       | `target`        | `selector`, `wait_after_ms`              |
40//! | `type`        | `target`, `text`| `selector`, `wait_after_ms`              |
41//! | `wait`        | `target`        | `selector`, `timeout_ms`                 |
42//! | `assert`      | *one of below*  | —                                        |
43//! | `screenshot`  | —               | `path`                                   |
44//! | `agent`       | `agent`, `task` | —                                        |
45//! | `mcp`         | `server`, `tool`| `args`                                   |
46//!
47//! Assert steps require one of: `definition` (references a named
48//! `[[definitions]]` entry), `preset` (built-in preset name), or `prompt`
49//! (custom LLM evaluation prompt).
50
51use std::collections::HashMap;
52
53use serde::Deserialize;
54use serde_json::Value;
55
56/// Top-level scenario file, deserialized from TOML.
57#[derive(Debug, Deserialize)]
58pub struct Scenario {
59    /// Global configuration (overridable per test).
60    #[serde(default)]
61    pub config: ScenarioConfig,
62
63    /// Reusable assertion definitions referenced by name in `assert` steps.
64    #[serde(default)]
65    pub definitions: Vec<AssertDefinition>,
66
67    /// Ordered test groups to execute.
68    #[serde(default)]
69    pub test: Vec<TestGroup>,
70}
71
72/// Global scenario configuration with per-test overridable fields.
73#[derive(Debug, Deserialize, Default, Clone)]
74pub struct ScenarioConfig {
75    /// Base URL for relative navigation.
76    #[serde(default)]
77    pub base_url: Option<String>,
78    /// LLM server base URL (deprecated; prefer `[config.endpoints]`).
79    #[serde(default)]
80    pub llm_url: Option<String>,
81    /// LLM model name (deprecated; prefer `[config.endpoints]`).
82    #[serde(default)]
83    pub llm_model: Option<String>,
84    /// LLM API key (Bearer token).
85    #[serde(default)]
86    pub llm_api_key: Option<String>,
87    /// Custom HTTP headers as JSON key-value pairs.
88    #[serde(default, deserialize_with = "deserialize_headers")]
89    pub llm_headers: HashMap<String, String>,
90    /// Run browser in headless mode.
91    #[serde(default)]
92    pub browser_headless: Option<bool>,
93    /// HTTP / browser action timeout in seconds.
94    #[serde(default)]
95    pub timeout_secs: Option<u64>,
96    /// Browser viewport width.
97    #[serde(default)]
98    pub viewport_width: Option<u32>,
99    /// Browser viewport height.
100    #[serde(default)]
101    pub viewport_height: Option<u32>,
102    /// Default URL every test auto-navigates to before running its steps.
103    #[serde(default)]
104    pub start_url: Option<String>,
105    /// Whether to auto-navigate to `start_url` before test steps.
106    ///
107    /// Disable when a test starts with click-based navigation.
108    #[serde(default = "default_auto_navigate")]
109    pub auto_navigate: bool,
110    /// LLM temperature (0.0–1.0). Lower = more deterministic.
111    #[serde(default = "default_temperature")]
112    pub temperature: f64,
113    /// Enable thinking/reasoning tokens. `None` means the provider default
114    /// is used (no `thinking` key is sent). Set to `true`/`false` to
115    /// explicitly enable or disable.
116    #[serde(default)]
117    pub thinking: Option<bool>,
118    /// Provider-specific model parameters merged into the chat completion
119    /// request body (e.g. `effort = "high"` for Anthropic).
120    #[serde(default, deserialize_with = "deserialize_model_params")]
121    pub model_params: HashMap<String, Value>,
122    /// Named endpoints (LLM, MCP, A2A agents) with pricing.
123    #[serde(default)]
124    pub endpoints: HashMap<String, EndpointConfig>,
125    /// Global and per-test budgets for cost/token/call limits.
126    #[serde(default)]
127    pub budgets: BudgetsConfig,
128    /// MCP server exposure configuration.
129    #[serde(default)]
130    pub mcp_server: Option<McpServerConfig>,
131    /// A2A agent server exposure configuration.
132    #[serde(default)]
133    pub a2a_server: Option<A2aServerConfig>,
134    /// Whether to continue running the remaining steps of a test after a
135    /// step fails. Default `false` = fail fast: the first failed step ends
136    /// the test and the rest are reported as skipped. Set to `true` to run
137    /// every step (more diagnostics, more LLM cost on broken apps).
138    #[serde(default)]
139    pub continue_on_failure: bool,
140    /// Longest edge (px) of screenshots attached to `screenshot = true`
141    /// assert steps, before they are JPEG-encoded and sent to the vision
142    /// endpoint. Downscaling keeps vision token cost/quality sane.
143    /// Default: 1400.
144    #[serde(default)]
145    pub screenshot_max_dimension: Option<u32>,
146    /// Height cap (px) of page coverage for screenshots attached to
147    /// `screenshot = true` assert steps. The full scrollable page is
148    /// captured, then split into viewport-tall tiles (each sent as its own
149    /// image part, ordered from the top) covering at most this many pixels
150    /// — below-the-fold content stays visible to the vision model at 1:1
151    /// detail while token cost stays bounded. Accepts an absolute pixel
152    /// count (`2880`) or a viewport multiple (`"20x"` = twenty times the
153    /// currently applied viewport height, which follows viewport-matrix
154    /// and per-test overrides). A value below the viewport height is
155    /// raised to it, so the visible viewport is always fully included
156    /// (`0` / `"0x"` therefore means "viewport only", the pre-full-page
157    /// behavior). Default: `"20x"`.
158    #[serde(default, deserialize_with = "deserialize_screenshot_max_height")]
159    pub screenshot_max_height: Option<ScreenshotHeight>,
160    /// Directory for failure artifacts (screenshots, page snapshots).
161    /// Defaults to `artifacts`.
162    #[serde(default)]
163    pub artifacts_dir: Option<String>,
164    /// Optional viewport matrix: when set, every test in the scenario is
165    /// expanded into one variant per viewport (e.g. mobile/tablet/desktop).
166    /// Each variant overrides the test's viewport and gets a ` — <name>`
167    /// suffix on the test name. Per-test budgets apply per variant.
168    #[serde(default)]
169    pub viewport_matrix: Option<ViewportMatrix>,
170    /// Class-name prefixes the `layout_no_issues` scan skips: elements
171    /// whose class matches any prefix are ignored by the fixed-element,
172    /// text-clipped, and overlap checks. Defaults cover the Angular CDK
173    /// screen-reader helpers (`cdk-visually-hidden`,
174    /// `cdk-describedby-message-container`, `cdk-overlay-container`),
175    /// which are intentionally 1x1 / off-screen.
176    #[serde(default = "default_layout_ignore_classes")]
177    pub layout_ignore_classes: Vec<String>,
178    /// Concurrency group for parallel runs across scenario files.
179    ///
180    /// When several scenario files are run together (`--parallel > 1`),
181    /// files that declare the **same** `concurrency_group` are never
182    /// executed at the same time — use this for files that touch the same
183    /// shared backend state and would interfere if run concurrently. A file
184    /// with no group gets its own implicit group, so distinct files run in
185    /// parallel by default. Only honored when files are passed to the
186    /// runner as a batch; has no effect on the steps within a single file,
187    /// which always run sequentially.
188    #[serde(default)]
189    pub concurrency_group: Option<String>,
190}
191
192/// A list of named viewports a scenario is expanded across.
193#[derive(Debug, Deserialize, Clone, Default)]
194pub struct ViewportMatrix {
195    /// The viewport variants (`{name, width, height}`).
196    #[serde(default)]
197    pub viewports: Vec<ViewportDef>,
198}
199
200/// Height cap for screenshots attached to `screenshot = true` assert
201/// steps: either an absolute pixel count or a multiple of the currently
202/// applied viewport height.
203#[derive(Debug, Clone, PartialEq)]
204pub enum ScreenshotHeight {
205    /// Absolute height in pixels.
206    Pixels(u32),
207    /// Multiple of the active viewport height (e.g. `2x` = twice the
208    /// current viewport's pixel height).
209    ViewportTimes(f64),
210}
211
212/// Ceiling (px) applied when resolving screenshot height specs, so an
213/// absurdly large multiplier can never overflow or produce an unusable
214/// capture.
215const MAX_SCREENSHOT_HEIGHT: u32 = 4_000_000;
216
217impl ScreenshotHeight {
218    /// Resolves this height spec to a pixel value for the given viewport
219    /// height. Multipliers are rounded and clamped to
220    /// [`MAX_SCREENSHOT_HEIGHT`].
221    #[must_use]
222    pub fn to_px(&self, viewport_height: u32) -> u32 {
223        match self {
224            Self::Pixels(px) => *px,
225            Self::ViewportTimes(mult) => {
226                let scaled = f64::from(viewport_height) * mult;
227                #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
228                let px = scaled
229                    .round()
230                    .max(1.0)
231                    .min(f64::from(MAX_SCREENSHOT_HEIGHT)) as u32;
232                px
233            }
234        }
235    }
236}
237
238/// One named viewport size in a matrix.
239#[derive(Debug, Deserialize, Clone)]
240pub struct ViewportDef {
241    /// Human-readable variant name (appended to test names, e.g.
242    /// `— mobile`).
243    pub name: String,
244    /// Browser viewport width in pixels.
245    pub width: u32,
246    /// Browser viewport height in pixels.
247    pub height: u32,
248}
249
250/// A named endpoint definition with pricing.
251#[derive(Debug, Deserialize, Clone, Default)]
252pub struct EndpointConfig {
253    /// Endpoint type: `llm`, `mcp`, or `a2a`.
254    #[serde(rename = "type")]
255    pub endpoint_type: EndpointType,
256    /// Base URL for the endpoint.
257    #[serde(default)]
258    pub url: Option<String>,
259    /// Model name (LLM endpoints only).
260    #[serde(default)]
261    pub model: Option<String>,
262    /// API key / bearer token.
263    #[serde(default)]
264    pub api_key: Option<String>,
265    /// Custom HTTP headers as JSON key-value pairs.
266    #[serde(default, deserialize_with = "deserialize_headers")]
267    pub headers: HashMap<String, String>,
268    /// Pricing configuration.
269    #[serde(default)]
270    pub pricing: Option<PricingConfig>,
271    /// Task types this endpoint serves by default
272    /// (e.g. `["targeting", "assertion"]`).
273    #[serde(default)]
274    pub default_for: Vec<String>,
275    /// Command to launch an MCP server subprocess (stdio transport).
276    #[serde(default)]
277    pub command: Option<String>,
278    /// Arguments for the MCP server command.
279    #[serde(default)]
280    pub args: Vec<String>,
281    /// Whether this LLM endpoint accepts image parts (vision) in addition
282    /// to text. `assert` steps with `screenshot = true` require a vision
283    /// endpoint.
284    #[serde(default)]
285    pub vision: bool,
286    /// How often a single chat completion against this endpoint is retried
287    /// on transient failures (HTTP 429/5xx, empty 200 bodies, network
288    /// errors) before the fallback chain is tried. Default: 3 (override
289    /// globally with `HARNESS_LLM_CALL_ATTEMPTS`).
290    #[serde(default)]
291    pub max_attempts: Option<u32>,
292    /// Ordered names of other endpoints to try when this endpoint exhausts
293    /// its attempts. Only LLM endpoints are eligible. Useful for pairing a
294    /// cheap primary model with a more powerful/expensive fallback.
295    #[serde(default)]
296    pub fallbacks: Vec<String>,
297    /// LLM provider protocol: `openai` (default, OpenAI-compatible chat
298    /// completions), `azure` (`Azure` `OpenAI`), or `bedrock` (AWS Bedrock
299    /// Converse API; requires the `aws` cargo feature).
300    #[serde(default)]
301    pub provider: Provider,
302    /// `Azure` `OpenAI` deployment name (`provider = "azure"`). Defaults to
303    /// `model` when unset.
304    #[serde(default)]
305    pub deployment: Option<String>,
306    /// `Azure` `OpenAI` API version (`provider = "azure"`). Defaults to
307    /// `2024-10-21`.
308    #[serde(default)]
309    pub api_version: Option<String>,
310    /// Authentication configuration for LLM endpoints (API key, token
311    /// command, Entra ID client credentials / managed identity).
312    #[serde(default)]
313    pub auth: AuthConfig,
314    /// Extra HTTP headers produced by running a command per call, keyed by
315    /// header name. The command's stdout (first line) becomes the header
316    /// value. Provider-agnostic — applies to every LLM provider.
317    #[serde(default, deserialize_with = "deserialize_headers")]
318    pub header_commands: HashMap<String, String>,
319    /// AWS credential settings (`provider = "bedrock"`).
320    #[serde(default)]
321    pub aws: AwsConfig,
322}
323
324/// Type discriminator for endpoint configuration.
325#[derive(Debug, Deserialize, Clone, PartialEq, Eq, Default)]
326#[serde(rename_all = "lowercase")]
327pub enum EndpointType {
328    /// OpenAI-compatible LLM API.
329    #[default]
330    Llm,
331    /// Model Context Protocol server.
332    Mcp,
333    /// Agent-to-Agent protocol agent.
334    A2a,
335}
336
337/// LLM provider protocol used by an LLM endpoint.
338#[derive(Debug, Deserialize, Clone, Copy, PartialEq, Eq, Default)]
339#[serde(rename_all = "lowercase")]
340pub enum Provider {
341    /// OpenAI-compatible Chat Completions API
342    /// (`POST <url>/v1/chat/completions`).
343    #[default]
344    Openai,
345    /// `Azure` `OpenAI`
346    /// (`POST <url>/openai/deployments/<deployment>/chat/completions`).
347    Azure,
348    /// AWS Bedrock Converse API (`POST https://bedrock-runtime.<region>
349    /// .amazonaws.com/model/<model>/converse`). Requires the `aws` cargo
350    /// feature; uses the standard AWS credential chain unless overridden.
351    Bedrock,
352}
353
354/// Authentication mode for an LLM endpoint.
355#[derive(Debug, Deserialize, Clone, Copy, PartialEq, Eq, Default)]
356#[serde(rename_all = "kebab-case")]
357pub enum AuthMode {
358    /// Static API key. Sent as `Authorization: Bearer <key>` by default;
359    /// set [`AuthConfig::api_key_header`] (or use the `azure` provider)
360    /// to send it in a different header.
361    #[default]
362    ApiKey,
363    /// Execute a command and use its stdout (first line) as the bearer
364    /// token. Uses the `token_command` field. Provider-agnostic escape
365    /// hatch — e.g. `az account get-access-token …`.
366    TokenCommand,
367    /// Entra ID (`Azure` AD) OAuth 2.0 client-credentials grant: exchanges
368    /// `client_id` + `client_secret` in `tenant_id` for a bearer token.
369    EntraClientCredentials,
370    /// Entra ID managed identity: fetches a bearer token from the IMDS
371    /// endpoint (works on `Azure` VMs / App Service / ACI with a
372    /// system-assigned identity; zero credentials in the config).
373    EntraManagedIdentity,
374}
375
376/// Authentication settings for an LLM endpoint.
377#[derive(Debug, Deserialize, Clone, Default)]
378pub struct AuthConfig {
379    /// Which authentication mode to use. Default: `api-key`.
380    #[serde(default)]
381    pub mode: AuthMode,
382    /// Header name that receives the API key instead of
383    /// `Authorization: Bearer` (`mode = "api-key"`). For example the `Azure`
384    /// `OpenAI` `api-key` header.
385    #[serde(default)]
386    pub api_key_header: Option<String>,
387    /// Command whose stdout (first line) is used as the bearer token
388    /// (`mode = "token-command"`).
389    #[serde(default)]
390    pub token_command: Option<String>,
391    /// Entra tenant id (`mode = "entra-client-credentials"`,
392    /// `"entra-managed-identity"`).
393    #[serde(default)]
394    pub tenant_id: Option<String>,
395    /// Entra client id (`mode = "entra-client-credentials"`).
396    #[serde(default)]
397    pub client_id: Option<String>,
398    /// Entra client secret (`mode = "entra-client-credentials"`).
399    #[serde(default)]
400    pub client_secret: Option<String>,
401    /// Entra token scope. Defaults to
402    /// `https://cognitiveservices.azure.com/.default`.
403    #[serde(default)]
404    pub scope: Option<String>,
405    /// Override for the Entra token endpoint
406    /// (`mode = "entra-client-credentials"`), default
407    /// `https://login.microsoftonline.com`. Also used as the IMDS identity
408    /// endpoint base for `"entra-managed-identity"` (default
409    /// `http://169.254.169.254`). Mainly useful for tests and proxies.
410    #[serde(default)]
411    pub token_url: Option<String>,
412    /// Seconds to reuse a fetched (non-API-key) token before refetching.
413    /// For Entra modes the server-issued expiry is used when available;
414    /// this caps the reuse window. Default 300.
415    #[serde(default)]
416    pub cache_ttl_secs: Option<u64>,
417}
418
419/// AWS credential and region settings for a `bedrock` provider endpoint.
420///
421/// When no explicit access key is given, the standard AWS credential chain
422/// is used (env vars, shared config `~/.aws/config` + `~/.aws/credentials`,
423/// SSO, ECS/IMDS) — the same behavior as the AWS CLI. `profile` selects a
424/// named profile from the shared config, and `region` overrides the chain
425/// default.
426#[derive(Debug, Deserialize, Clone, Default)]
427pub struct AwsConfig {
428    /// Named profile from `~/.aws/config` / `~/.aws/credentials` to use
429    /// (default: the AWS CLI default selection via `AWS_PROFILE` or
430    /// `default`).
431    #[serde(default)]
432    pub profile: Option<String>,
433    /// AWS region (default: `AWS_REGION` env or the profile's region;
434    /// required if neither is set).
435    #[serde(default)]
436    pub region: Option<String>,
437    /// Explicit access key id (bypasses the credential chain).
438    #[serde(default)]
439    pub access_key_id: Option<String>,
440    /// Explicit secret access key (with `access_key_id`).
441    #[serde(default)]
442    pub secret_access_key: Option<String>,
443    /// Optional session token for explicit temporary credentials.
444    #[serde(default)]
445    pub session_token: Option<String>,
446}
447
448/// Pricing configuration for an endpoint.
449#[derive(Debug, Deserialize, Clone, Default)]
450pub struct PricingConfig {
451    /// Cost per 1M input tokens (USD).
452    #[serde(default)]
453    pub input_per_1m_tokens: f64,
454    /// Cost per 1M output tokens (USD).
455    #[serde(default)]
456    pub output_per_1m_tokens: f64,
457    /// Flat cost per call (USD), used for MCP/agent endpoints.
458    #[serde(default)]
459    pub per_call: f64,
460}
461
462/// Budget limits for test execution.
463#[derive(Debug, Deserialize, Clone, Default)]
464pub struct BudgetsConfig {
465    /// Global budget across all tests in the scenario.
466    #[serde(default)]
467    pub global: Option<BudgetDef>,
468    /// Default per-test budget. Individual tests can override.
469    #[serde(default)]
470    pub per_test_default: Option<BudgetDef>,
471}
472
473/// A budget definition with limits and enforcement mode.
474#[derive(Debug, Deserialize, Clone)]
475pub struct BudgetDef {
476    /// Maximum cost in USD.
477    #[serde(default)]
478    pub max_cost: Option<f64>,
479    /// Maximum total tokens (input + output).
480    #[serde(default)]
481    pub max_tokens: Option<u64>,
482    /// Maximum number of calls (LLM, MCP, agent combined).
483    #[serde(default)]
484    pub max_calls: Option<u64>,
485    /// Enforcement mode: `hard` (abort) or `soft` (warn and continue).
486    #[serde(default)]
487    pub enforcement: Option<BudgetEnforcement>,
488}
489
490/// Budget enforcement strategy.
491#[derive(Debug, Deserialize, Clone, PartialEq, Eq)]
492#[serde(rename_all = "lowercase")]
493pub enum BudgetEnforcement {
494    /// Abort the test or run when budget is exceeded.
495    Hard,
496    /// Log a warning but continue execution.
497    Soft,
498}
499
500/// MCP server exposure configuration.
501#[derive(Debug, Deserialize, Clone)]
502pub struct McpServerConfig {
503    /// Whether to enable the embedded MCP server.
504    #[serde(default)]
505    pub enabled: bool,
506    /// Port to listen on.
507    #[serde(default = "default_mcp_port")]
508    pub port: u16,
509}
510
511const fn default_mcp_port() -> u16 {
512    3000
513}
514
515/// A2A agent server exposure configuration.
516#[derive(Debug, Deserialize, Clone)]
517pub struct A2aServerConfig {
518    /// Whether to enable the embedded A2A agent server.
519    #[serde(default)]
520    pub enabled: bool,
521    /// Port to listen on.
522    #[serde(default = "default_a2a_port")]
523    pub port: u16,
524}
525
526const fn default_a2a_port() -> u16 {
527    3100
528}
529
530fn deserialize_headers<'de, D>(deserializer: D) -> Result<HashMap<String, String>, D::Error>
531where
532    D: serde::Deserializer<'de>,
533{
534    let raw: Option<serde_json::Value> = Option::deserialize(deserializer)?;
535    let Some(json) = raw else {
536        return Ok(HashMap::new());
537    };
538    let serde_json::Value::Object(obj) = json else {
539        return Ok(HashMap::new());
540    };
541    Ok(obj
542        .into_iter()
543        .filter_map(|(k, v)| v.as_str().map(|s| (k, s.to_owned())))
544        .collect())
545}
546
547const fn default_auto_navigate() -> bool {
548    true
549}
550
551fn default_layout_ignore_classes() -> Vec<String> {
552    vec![
553        "cdk-visually-hidden".to_owned(),
554        "cdk-describedby-message-container".to_owned(),
555        "cdk-overlay-container".to_owned(),
556    ]
557}
558
559const fn default_temperature() -> f64 {
560    0.0
561}
562
563fn deserialize_model_params<'de, D>(deserializer: D) -> Result<HashMap<String, Value>, D::Error>
564where
565    D: serde::Deserializer<'de>,
566{
567    #[derive(Deserialize)]
568    #[serde(untagged)]
569    enum Raw {
570        Map(HashMap<String, Value>),
571        Table(HashMap<String, Value>),
572    }
573    let raw: Option<Raw> = Option::deserialize(deserializer)?;
574    Ok(match raw {
575        Some(Raw::Map(m) | Raw::Table(m)) => m,
576        None => HashMap::new(),
577    })
578}
579
580fn deserialize_screenshot_max_height<'de, D>(
581    deserializer: D,
582) -> Result<Option<ScreenshotHeight>, D::Error>
583where
584    D: serde::Deserializer<'de>,
585{
586    let raw: Option<serde_json::Value> = Option::deserialize(deserializer)?;
587    match raw {
588        None => Ok(None),
589        Some(value) => match value.as_u64() {
590            // Integer: absolute pixel count.
591            Some(px) if px <= u64::from(MAX_SCREENSHOT_HEIGHT) => {
592                #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
593                Ok(Some(ScreenshotHeight::Pixels(px as u32)))
594            }
595            // String: viewport multiple like "2x" or "1.5x".
596            None => match value.as_str() {
597                Some(s) => {
598                    let lower = s.trim().to_lowercase();
599                    if lower.ends_with('x') {
600                        let num = lower.strip_suffix("x");
601                        if let Some(num) = num {
602                            if let Ok(m) = num.parse::<f64>() {
603                                if m > 0.0 {
604                                    return Ok(Some(ScreenshotHeight::ViewportTimes(m)));
605                                }
606                            }
607                        }
608                    }
609                    Err(serde::de::Error::custom(format_args!(
610                        "screenshot_max_height must be a pixel count (e.g. 2880) or a viewport multiple like \"2x\", found {s:?}"
611                    )))
612                }
613                None => Err(serde::de::Error::custom(format_args!(
614                    "screenshot_max_height must be a pixel count (e.g. 2880) or a viewport multiple like \"2x\", found {value:?}"
615                ))),
616            },
617            Some(_) => Err(serde::de::Error::custom(format_args!(
618                "screenshot_max_height value too large (max {MAX_SCREENSHOT_HEIGHT} px)"
619            ))),
620        },
621    }
622}
623
624/// Reusable assertion definition referenced by name from `assert` steps.
625///
626/// Definitions can either reference a built-in preset via `preset`, supply a
627/// custom LLM `prompt`, or define a **custom preset** by providing both
628/// `system` and `user_template`. Custom presets support the same template
629/// variables as built-in presets: `{url}`, `{title}`, `{content}`,
630/// `{expected_text}`, `{description}`.
631#[derive(Debug, Deserialize, Clone)]
632pub struct AssertDefinition {
633    /// Unique name used to reference this definition from steps.
634    pub name: String,
635    /// Predefined assertion preset name
636    /// (e.g. `no_error_on_page`, `text_visible`).
637    #[serde(default)]
638    pub preset: Option<String>,
639    /// Custom LLM prompt for assertion evaluation.
640    #[serde(default)]
641    pub prompt: Option<String>,
642    /// System prompt for a custom preset.
643    #[serde(default)]
644    pub system: Option<String>,
645    /// User template (with `{placeholders}`) for a custom preset.
646    #[serde(default)]
647    pub user_template: Option<String>,
648    /// Text that the `text_visible` preset checks for, or the
649    /// `{expected_text}` placeholder value for custom presets.
650    #[serde(default)]
651    pub assert_text: Option<String>,
652    /// Agent endpoint to call for this assertion.
653    #[serde(default)]
654    pub agent: Option<String>,
655    /// Agent task template for this assertion.
656    #[serde(default)]
657    pub task_template: Option<String>,
658}
659
660/// A group of steps that form a single test scenario.
661#[derive(Debug, Deserialize, Clone)]
662pub struct TestGroup {
663    /// Human-readable test name.
664    pub name: String,
665    /// Override the global `start_url` for this test.
666    #[serde(default)]
667    pub start_url: Option<String>,
668    /// Override the global `auto_navigate` for this test.
669    #[serde(default)]
670    pub auto_navigate: Option<bool>,
671    /// Override the global `base_url` for this test.
672    #[serde(default)]
673    pub base_url: Option<String>,
674    /// Override the global `timeout_secs` for this test.
675    #[serde(default)]
676    pub timeout_secs: Option<u64>,
677    /// Override the global `browser_headless` for this test.
678    #[serde(default)]
679    pub browser_headless: Option<bool>,
680    /// Override the global viewport width for this test (applied via CDP
681    /// `Emulation.setDeviceMetricsOverride` before the test runs).
682    #[serde(default)]
683    pub viewport_width: Option<u32>,
684    /// Override the global viewport height for this test.
685    #[serde(default)]
686    pub viewport_height: Option<u32>,
687    /// Per-test budget override.
688    #[serde(default)]
689    pub budget: Option<BudgetDef>,
690    /// Endpoint to use for all steps in this test (can be overridden
691    /// per-step).
692    #[serde(default)]
693    pub endpoint: Option<String>,
694    /// Ordered steps to execute.
695    #[serde(default)]
696    pub steps: Vec<TestStep>,
697}
698
699/// A single step in a test. The `kind` field determines which variant is
700/// deserialized and which field constraints apply.
701#[derive(Debug, Deserialize, Clone)]
702#[serde(tag = "kind")]
703pub enum TestStep {
704    /// Navigate the browser to a URL.
705    #[serde(rename = "navigate")]
706    Navigate {
707        /// URL to navigate to (absolute, or relative to the test's
708        /// base URL).
709        url: String,
710        /// Milliseconds to wait after navigation completes.
711        #[serde(default)]
712        wait_after_ms: Option<u64>,
713    },
714
715    /// Click an element described in natural language.
716    #[serde(rename = "click")]
717    Click {
718        /// Natural language description of the element. The LLM resolves
719        /// this to a CSS selector at runtime.
720        target: String,
721        /// Explicit CSS selector override (bypasses LLM resolution).
722        #[serde(default)]
723        selector: Option<String>,
724        /// Milliseconds to wait after the click.
725        #[serde(default)]
726        wait_after_ms: Option<u64>,
727        /// Endpoint to use for LLM element targeting.
728        #[serde(default)]
729        endpoint: Option<String>,
730        /// Idempotent: when the target element is absent the step is
731        /// reported skipped instead of failed (the action was already
732        /// done / not applicable).
733        #[serde(default)]
734        idempotent: bool,
735    },
736
737    /// Type text into an input element.
738    #[serde(rename = "type")]
739    Type {
740        /// Natural language description of the target input element.
741        target: String,
742        /// Text to type into the element.
743        text: String,
744        /// Explicit CSS selector override (bypasses LLM resolution).
745        #[serde(default)]
746        selector: Option<String>,
747        /// Milliseconds to wait after typing.
748        #[serde(default)]
749        wait_after_ms: Option<u64>,
750        /// Endpoint to use for LLM element targeting.
751        #[serde(default)]
752        endpoint: Option<String>,
753        /// Idempotent: when the target element is absent the step is
754        /// reported skipped instead of failed (the action was already
755        /// done / not applicable).
756        #[serde(default)]
757        idempotent: bool,
758    },
759
760    /// Wait for an element to appear on the page.
761    #[serde(rename = "wait")]
762    Wait {
763        /// Natural language description of the element to wait for.
764        target: String,
765        /// Explicit CSS selector override (bypasses LLM resolution).
766        #[serde(default)]
767        selector: Option<String>,
768        /// Wait until the page's visible text contains this substring
769        /// (alternative to `selector`; either or both may be set — both are
770        /// required to hold when both are set).
771        #[serde(default)]
772        text: Option<String>,
773        /// Maximum milliseconds to wait (default: 10000).
774        #[serde(default)]
775        timeout_ms: Option<u64>,
776        /// Endpoint to use for LLM element targeting.
777        #[serde(default)]
778        endpoint: Option<String>,
779        /// Idempotent: when the condition never becomes true within the
780        /// timeout the step is reported skipped instead of failed (the
781        /// condition was not applicable, e.g. already-authenticated
782        /// pages in a viewport matrix).
783        #[serde(default)]
784        idempotent: bool,
785    },
786
787    /// Evaluate an assertion against the current page content.
788    #[serde(rename = "assert")]
789    Assert {
790        /// Reference to a named `[[definitions]]` entry.
791        #[serde(default)]
792        definition: Option<String>,
793        /// Inline predefined assertion preset (e.g. `no_error_on_page`).
794        #[serde(default)]
795        preset: Option<String>,
796        /// Inline custom LLM prompt for assertion evaluation.
797        #[serde(default)]
798        prompt: Option<String>,
799        /// Text that the `text_visible` preset checks for.
800        #[serde(default)]
801        assert_text: Option<String>,
802        /// Endpoint to use for this assertion's LLM call.
803        #[serde(default)]
804        endpoint: Option<String>,
805        /// Attach a screenshot of the current viewport to the assertion so
806        /// the LLM can evaluate visuals (overlaps, clipping, layout).
807        /// Requires the resolved endpoint to declare `vision = true`.
808        #[serde(default)]
809        screenshot: bool,
810    },
811
812    /// Take a screenshot of the current page.
813    #[serde(rename = "screenshot")]
814    Screenshot {
815        /// File path to save the screenshot (default: `screenshot.png`).
816        #[serde(default)]
817        path: Option<String>,
818    },
819
820    /// Call an A2A agent with a task.
821    #[serde(rename = "agent")]
822    Agent {
823        /// Name of the agent endpoint to call.
824        agent: String,
825        /// Task description / prompt for the agent.
826        task: String,
827        /// Optional definition name with a task template.
828        #[serde(default)]
829        definition: Option<String>,
830    },
831
832    /// Call an MCP server tool.
833    #[serde(rename = "mcp")]
834    Mcp {
835        /// Name of the MCP server endpoint.
836        server: String,
837        /// Tool name to invoke on the server.
838        tool: String,
839        /// Tool arguments as JSON.
840        #[serde(default)]
841        args: Option<serde_json::Value>,
842    },
843}
llm_browser_testkit/scenario.rs

llm_browser_testkit/
scenario.rs
