llm_browser_testkit/
scenario.rs

1//! Scenario types for human-readable browser test case definitions.
2//!
3//! Scenarios are written in [TOML](https://toml.io) and describe groups of
4//! browser interaction tests with reusable assertion definitions and
5//! configurable test-level overrides.
6//!
7//! # Structure
8//!
9//! ```toml
10//! [config]                         # Global defaults
11//! start_url = "/dashboard"
12//!
13//! [[definitions]]                  # Reusable assertion definitions
14//! name = "no_errors"
15//! preset = "no_error_on_page"
16//!
17//! [[test]]                         # Test group
18//! name = "Dashboard Smoke"
19//! start_url = "/dashboard"         # Override global start_url (optional)
20//!
21//! [[test.steps]]                   # Ordered steps — the `kind` field
22//! kind = "navigate"                # determines which other fields apply
23//! url = "/dashboard"
24//!
25//! [[test.steps]]
26//! kind = "click"
27//! target = "the Login button"      # Natural language — LLM resolves to selector
28//!
29//! [[test.steps]]
30//! kind = "assert"
31//! definition = "no_errors"
32//! ```
33//!
34//! ## Step Kinds
35//!
36//! | `kind`        | Required fields  | Optional fields                          |
37//! |---------------|-----------------|------------------------------------------|
38//! | `navigate`    | `url`           | `wait_after_ms`                          |
39//! | `click`       | `target`        | `selector`, `wait_after_ms`              |
40//! | `type`        | `target`, `text`| `selector`, `wait_after_ms`              |
41//! | `wait`        | `target`        | `selector`, `timeout_ms`                 |
42//! | `assert`      | *one of below*  | —                                        |
43//! | `screenshot`  | —               | `path`                                   |
44//! | `agent`       | `agent`, `task` | —                                        |
45//! | `mcp`         | `server`, `tool`| `args`                                   |
46//!
47//! Assert steps require one of: `definition` (references a named
48//! `[[definitions]]` entry), `preset` (built-in preset name), or `prompt`
49//! (custom LLM evaluation prompt).
50
51use std::collections::HashMap;
52
53use serde::Deserialize;
54use serde_json::Value;
55
56/// Top-level scenario file, deserialized from TOML.
57#[derive(Debug, Deserialize)]
58pub struct Scenario {
59    /// Global configuration (overridable per test).
60    #[serde(default)]
61    pub config: ScenarioConfig,
62
63    /// Reusable assertion definitions referenced by name in `assert` steps.
64    #[serde(default)]
65    pub definitions: Vec<AssertDefinition>,
66
67    /// Ordered test groups to execute.
68    #[serde(default)]
69    pub test: Vec<TestGroup>,
70}
71
72/// Global scenario configuration with per-test overridable fields.
73#[derive(Debug, Deserialize, Default, Clone)]
74pub struct ScenarioConfig {
75    /// Base URL for relative navigation.
76    #[serde(default)]
77    pub base_url: Option<String>,
78    /// LLM server base URL (deprecated; prefer `[config.endpoints]`).
79    #[serde(default)]
80    pub llm_url: Option<String>,
81    /// LLM model name (deprecated; prefer `[config.endpoints]`).
82    #[serde(default)]
83    pub llm_model: Option<String>,
84    /// LLM API key (Bearer token).
85    #[serde(default)]
86    pub llm_api_key: Option<String>,
87    /// Custom HTTP headers as JSON key-value pairs.
88    #[serde(default, deserialize_with = "deserialize_headers")]
89    pub llm_headers: HashMap<String, String>,
90    /// Run browser in headless mode.
91    #[serde(default)]
92    pub browser_headless: Option<bool>,
93    /// HTTP Basic Auth username for browser navigation.
94    #[serde(default)]
95    pub browser_basic_auth_user: Option<String>,
96    /// HTTP Basic Auth password for browser navigation.
97    #[serde(default)]
98    pub browser_basic_auth_password: Option<String>,
99    /// HTTP / browser action timeout in seconds.
100    #[serde(default)]
101    pub timeout_secs: Option<u64>,
102    /// Browser viewport width.
103    #[serde(default)]
104    pub viewport_width: Option<u32>,
105    /// Browser viewport height.
106    #[serde(default)]
107    pub viewport_height: Option<u32>,
108    /// Default URL every test auto-navigates to before running its steps.
109    #[serde(default)]
110    pub start_url: Option<String>,
111    /// Whether to auto-navigate to `start_url` before test steps.
112    ///
113    /// Continue running remaining steps after a step failure.
114    ///
115    /// Disable when a test starts with click-based navigation.
116    #[serde(default = "default_auto_navigate")]
117    pub auto_navigate: bool,
118    /// Re-run a failed test this many times before reporting it failed.
119    ///
120    /// A shared browser tab plus a contended runner makes some page
121    /// loads stall (a JS chunk or a GraphQL call hangs mid-flight);
122    /// the test then fails on a wait/assert that a fresh run passes.
123    /// The retry re-runs the WHOLE test (fresh per-test isolation and
124    /// auto-navigate), and both attempts' LLM spend stays in the
125    /// budget accounting; only the final attempt's result is reported.
126    #[serde(default)]
127    pub retry_failed_tests: Option<u32>,
128    /// LLM temperature (0.0–1.0). Lower = more deterministic.
129    #[serde(default = "default_temperature")]
130    pub temperature: f64,
131    /// Enable thinking/reasoning tokens. `None` means the provider default
132    /// is used (no `thinking` key is sent). Set to `true`/`false` to
133    /// explicitly enable or disable.
134    #[serde(default)]
135    pub thinking: Option<bool>,
136    /// Provider-specific model parameters merged into the chat completion
137    /// request body (e.g. `effort = "high"` for Anthropic).
138    #[serde(default, deserialize_with = "deserialize_model_params")]
139    pub model_params: HashMap<String, Value>,
140    /// Named endpoints (LLM, MCP, A2A agents) with pricing.
141    #[serde(default)]
142    pub endpoints: HashMap<String, EndpointConfig>,
143    /// Global and per-test budgets for cost/token/call limits.
144    #[serde(default)]
145    pub budgets: BudgetsConfig,
146    /// MCP server exposure configuration.
147    #[serde(default)]
148    pub mcp_server: Option<McpServerConfig>,
149    /// A2A agent server exposure configuration.
150    #[serde(default)]
151    pub a2a_server: Option<A2aServerConfig>,
152    /// Whether to continue running the remaining steps of a test after a
153    /// step fails. Default `false` = fail fast: the first failed step ends
154    /// the test and the rest are reported as skipped. Set to `true` to run
155    /// every step (more diagnostics, more LLM cost on broken apps).
156    #[serde(default)]
157    pub continue_on_failure: bool,
158    /// Longest edge (px) of screenshots attached to `screenshot = true`
159    /// assert steps, before they are JPEG-encoded and sent to the vision
160    /// endpoint. Downscaling keeps vision token cost/quality sane.
161    /// Default: 1400.
162    #[serde(default)]
163    pub screenshot_max_dimension: Option<u32>,
164    /// Height cap (px) of page coverage for screenshots attached to
165    /// `screenshot = true` assert steps. The full scrollable page is
166    /// captured, then split into viewport-tall tiles (each sent as its own
167    /// image part, ordered from the top) covering at most this many pixels
168    /// — below-the-fold content stays visible to the vision model at 1:1
169    /// detail while token cost stays bounded. Accepts an absolute pixel
170    /// count (`2880`) or a viewport multiple (`"20x"` = twenty times the
171    /// currently applied viewport height, which follows viewport-matrix
172    /// and per-test overrides). A value below the viewport height is
173    /// raised to it, so the visible viewport is always fully included
174    /// (`0` / `"0x"` therefore means "viewport only", the pre-full-page
175    /// behavior). Default: `"20x"`.
176    #[serde(default, deserialize_with = "deserialize_screenshot_max_height")]
177    pub screenshot_max_height: Option<ScreenshotHeight>,
178    /// Directory for failure artifacts (screenshots, page snapshots).
179    /// Defaults to `artifacts`.
180    #[serde(default)]
181    pub artifacts_dir: Option<String>,
182    /// Optional viewport matrix: when set, every test in the scenario is
183    /// expanded into one variant per viewport (e.g. mobile/tablet/desktop).
184    /// Each variant overrides the test's viewport and gets a ` — <name>`
185    /// suffix on the test name. Per-test budgets apply per variant.
186    #[serde(default)]
187    pub viewport_matrix: Option<ViewportMatrix>,
188    /// Class-name prefixes the `layout_no_issues` scan skips: elements
189    /// whose class matches any prefix are ignored by the fixed-element,
190    /// text-clipped, and overlap checks. Defaults cover the Angular CDK
191    /// screen-reader helpers (`cdk-visually-hidden`,
192    /// `cdk-describedby-message-container`, `cdk-overlay-container`),
193    /// which are intentionally 1x1 / off-screen.
194    #[serde(default = "default_layout_ignore_classes")]
195    pub layout_ignore_classes: Vec<String>,
196    /// Concurrency group for parallel runs across scenario files.
197    /// When several scenario files are run together (`--parallel > 1`),
198    /// files that declare the **same** `concurrency_group` are never
199    /// executed at the same time — use this for files that touch the same
200    /// shared backend state and would interfere if run concurrently. A file
201    /// with no group gets its own implicit group, so distinct files run in
202    /// parallel by default. Only honored when files are passed to the
203    /// runner as a batch; has no effect on the steps within a single file,
204    /// which always run sequentially.
205    #[serde(default)]
206    pub concurrency_group: Option<String>,
207    /// Default for per-endpoint `cache` across all endpoints (default
208    /// `true`). Set to `false` to disable provider-side prompt-cache
209    /// markers globally.
210    #[serde(default)]
211    pub cache: Option<bool>,
212    /// Default for per-endpoint `pricing.cache_pricing` across all
213    /// endpoints (default `true`). Set to `false` to bill all prompt
214    /// tokens at the flat input price instead of cache rates.
215    #[serde(default)]
216    pub cache_pricing: Option<bool>,
217}
218
219/// A list of named viewports a scenario is expanded across.
220#[derive(Debug, Deserialize, Clone, Default)]
221pub struct ViewportMatrix {
222    /// The viewport variants (`{name, width, height}`).
223    #[serde(default)]
224    pub viewports: Vec<ViewportDef>,
225}
226
227/// Height cap for screenshots attached to `screenshot = true` assert
228/// steps: either an absolute pixel count or a multiple of the currently
229/// applied viewport height.
230#[derive(Debug, Clone, PartialEq)]
231pub enum ScreenshotHeight {
232    /// Absolute height in pixels.
233    Pixels(u32),
234    /// Multiple of the active viewport height (e.g. `2x` = twice the
235    /// current viewport's pixel height).
236    ViewportTimes(f64),
237}
238
239/// Ceiling (px) applied when resolving screenshot height specs, so an
240/// absurdly large multiplier can never overflow or produce an unusable
241/// capture.
242const MAX_SCREENSHOT_HEIGHT: u32 = 4_000_000;
243
244impl ScreenshotHeight {
245    /// Resolves this height spec to a pixel value for the given viewport
246    /// height. Multipliers are rounded and clamped to
247    /// [`MAX_SCREENSHOT_HEIGHT`].
248    #[must_use]
249    pub fn to_px(&self, viewport_height: u32) -> u32 {
250        match self {
251            Self::Pixels(px) => *px,
252            Self::ViewportTimes(mult) => {
253                let scaled = f64::from(viewport_height) * mult;
254                #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
255                let px = scaled
256                    .round()
257                    .max(1.0)
258                    .min(f64::from(MAX_SCREENSHOT_HEIGHT)) as u32;
259                px
260            }
261        }
262    }
263}
264
265/// One named viewport size in a matrix.
266#[derive(Debug, Deserialize, Clone)]
267pub struct ViewportDef {
268    /// Human-readable variant name (appended to test names, e.g.
269    /// `— mobile`).
270    pub name: String,
271    /// Browser viewport width in pixels.
272    pub width: u32,
273    /// Browser viewport height in pixels.
274    pub height: u32,
275}
276
277/// A named endpoint definition with pricing.
278#[derive(Debug, Deserialize, Clone, Default)]
279pub struct EndpointConfig {
280    /// Endpoint type: `llm`, `mcp`, or `a2a`.
281    #[serde(rename = "type")]
282    pub endpoint_type: EndpointType,
283    /// Base URL for the endpoint.
284    #[serde(default)]
285    pub url: Option<String>,
286    /// Model name (LLM endpoints only).
287    #[serde(default)]
288    pub model: Option<String>,
289    /// API key / bearer token.
290    #[serde(default)]
291    pub api_key: Option<String>,
292    /// Custom HTTP headers as JSON key-value pairs.
293    #[serde(default, deserialize_with = "deserialize_headers")]
294    pub headers: HashMap<String, String>,
295    /// Pricing configuration.
296    #[serde(default)]
297    pub pricing: Option<PricingConfig>,
298    /// Automatically fetch exact per-token pricing for this endpoint at
299    /// startup from a provider's public pricing API, filling in any pricing
300    /// fields left unset (explicit `pricing` values win). Defaults to
301    /// `"auto"`: Bedrock endpoints use the AWS Price List and `openrouter.ai`
302    /// URLs use the `OpenRouter` models API; other providers are a no-op.
303    /// Force a source with `"bedrock"` / `"openrouter"`, or disable the
304    /// lookup with `"off"` / `"none"` / `"disabled"`.
305    #[serde(default)]
306    pub pricing_source: Option<String>,
307    /// Task types this endpoint serves by default
308    /// (e.g. `["targeting", "assertion"]`).
309    #[serde(default)]
310    pub default_for: Vec<String>,
311    /// Command to launch an MCP server subprocess (stdio transport).
312    #[serde(default)]
313    pub command: Option<String>,
314    /// Arguments for the MCP server command.
315    #[serde(default)]
316    pub args: Vec<String>,
317    /// Whether this LLM endpoint accepts image parts (vision) in addition
318    /// to text. `assert` steps with `screenshot = true` require a vision
319    /// endpoint.
320    #[serde(default)]
321    pub vision: bool,
322    /// How often a single chat completion against this endpoint is retried
323    /// on transient failures (HTTP 429/5xx, empty 200 bodies, network
324    /// errors) before the fallback chain is tried. Default: 3 (override
325    /// globally with `HARNESS_LLM_CALL_ATTEMPTS`).
326    #[serde(default)]
327    pub max_attempts: Option<u32>,
328    /// Ordered names of other endpoints to try when this endpoint exhausts
329    /// its attempts. Only LLM endpoints are eligible. Useful for pairing a
330    /// cheap primary model with a more powerful/expensive fallback.
331    #[serde(default)]
332    pub fallbacks: Vec<String>,
333    /// LLM provider protocol: `openai` (default, OpenAI-compatible chat
334    /// completions), `azure` (`Azure` `OpenAI`), or `bedrock` (AWS Bedrock
335    /// Converse API; requires the `aws` cargo feature).
336    #[serde(default)]
337    pub provider: Provider,
338    /// `Azure` `OpenAI` deployment name (`provider = "azure"`). Defaults to
339    /// `model` when unset.
340    #[serde(default)]
341    pub deployment: Option<String>,
342    /// `Azure` `OpenAI` API version (`provider = "azure"`). Defaults to
343    /// `2024-10-21`.
344    #[serde(default)]
345    pub api_version: Option<String>,
346    /// Authentication configuration for LLM endpoints (API key, token
347    /// command, Entra ID client credentials / managed identity).
348    #[serde(default)]
349    pub auth: AuthConfig,
350    /// Extra HTTP headers produced by running a command per call, keyed by
351    /// header name. The command's stdout (first line) becomes the header
352    /// value. Provider-agnostic — applies to every LLM provider.
353    #[serde(default, deserialize_with = "deserialize_headers")]
354    pub header_commands: HashMap<String, String>,
355    /// AWS credential settings (`provider = "bedrock"`).
356    #[serde(default)]
357    pub aws: AwsConfig,
358    /// Send provider-side prompt-cache markers from this endpoint
359    /// (default `true`). Only providers that require explicit markers are
360    /// affected: AWS Bedrock gets a `cachePoint` block, and Anthropic-style
361    /// OpenAI-compatible models (model name contains `claude`/`anthropic`,
362    /// e.g. via `OpenRouter`) get a `cache_control: ephemeral` block on the
363    /// system message. `OpenAI`, `Azure`, Groq, xAI and `DeepSeek` cache
364    /// automatically and need no markers. Set to `false` to disable.
365    #[serde(default)]
366    pub cache: Option<bool>,
367}
368
369/// Type discriminator for endpoint configuration.
370#[derive(Debug, Deserialize, Clone, PartialEq, Eq, Default)]
371#[serde(rename_all = "lowercase")]
372pub enum EndpointType {
373    /// OpenAI-compatible LLM API.
374    #[default]
375    Llm,
376    /// Model Context Protocol server.
377    Mcp,
378    /// Agent-to-Agent protocol agent.
379    A2a,
380}
381
382/// LLM provider protocol used by an LLM endpoint.
383#[derive(Debug, Deserialize, Clone, Copy, PartialEq, Eq, Default)]
384#[serde(rename_all = "lowercase")]
385pub enum Provider {
386    /// OpenAI-compatible Chat Completions API
387    /// (`POST <url>/v1/chat/completions`).
388    #[default]
389    Openai,
390    /// `Azure` `OpenAI`
391    /// (`POST <url>/openai/deployments/<deployment>/chat/completions`).
392    Azure,
393    /// AWS Bedrock Converse API (`POST https://bedrock-runtime.<region>
394    /// .amazonaws.com/model/<model>/converse`). Requires the `aws` cargo
395    /// feature; uses the standard AWS credential chain unless overridden.
396    Bedrock,
397}
398
399/// Authentication mode for an LLM endpoint.
400#[derive(Debug, Deserialize, Clone, Copy, PartialEq, Eq, Default)]
401#[serde(rename_all = "kebab-case")]
402pub enum AuthMode {
403    /// Static API key. Sent as `Authorization: Bearer <key>` by default;
404    /// set [`AuthConfig::api_key_header`] (or use the `azure` provider)
405    /// to send it in a different header.
406    #[default]
407    ApiKey,
408    /// Execute a command and use its stdout (first line) as the bearer
409    /// token. Uses the `token_command` field. Provider-agnostic escape
410    /// hatch — e.g. `az account get-access-token …`.
411    TokenCommand,
412    /// Entra ID (`Azure` AD) OAuth 2.0 client-credentials grant: exchanges
413    /// `client_id` + `client_secret` in `tenant_id` for a bearer token.
414    EntraClientCredentials,
415    /// Entra ID managed identity: fetches a bearer token from the IMDS
416    /// endpoint (works on `Azure` VMs / App Service / ACI with a
417    /// system-assigned identity; zero credentials in the config).
418    EntraManagedIdentity,
419}
420
421/// Authentication settings for an LLM endpoint.
422#[derive(Debug, Deserialize, Clone, Default)]
423pub struct AuthConfig {
424    /// Which authentication mode to use. Default: `api-key`.
425    #[serde(default)]
426    pub mode: AuthMode,
427    /// Header name that receives the API key instead of
428    /// `Authorization: Bearer` (`mode = "api-key"`). For example the `Azure`
429    /// `OpenAI` `api-key` header.
430    #[serde(default)]
431    pub api_key_header: Option<String>,
432    /// Command whose stdout (first line) is used as the bearer token
433    /// (`mode = "token-command"`).
434    #[serde(default)]
435    pub token_command: Option<String>,
436    /// Entra tenant id (`mode = "entra-client-credentials"`,
437    /// `"entra-managed-identity"`).
438    #[serde(default)]
439    pub tenant_id: Option<String>,
440    /// Entra client id (`mode = "entra-client-credentials"`).
441    #[serde(default)]
442    pub client_id: Option<String>,
443    /// Entra client secret (`mode = "entra-client-credentials"`).
444    #[serde(default)]
445    pub client_secret: Option<String>,
446    /// Entra token scope. Defaults to
447    /// `https://cognitiveservices.azure.com/.default`.
448    #[serde(default)]
449    pub scope: Option<String>,
450    /// Override for the Entra token endpoint
451    /// (`mode = "entra-client-credentials"`), default
452    /// `https://login.microsoftonline.com`. Also used as the IMDS identity
453    /// endpoint base for `"entra-managed-identity"` (default
454    /// `http://169.254.169.254`). Mainly useful for tests and proxies.
455    #[serde(default)]
456    pub token_url: Option<String>,
457    /// Seconds to reuse a fetched (non-API-key) token before refetching.
458    /// For Entra modes the server-issued expiry is used when available;
459    /// this caps the reuse window. Default 300.
460    #[serde(default)]
461    pub cache_ttl_secs: Option<u64>,
462}
463
464/// AWS credential and region settings for a `bedrock` provider endpoint.
465///
466/// When no explicit access key is given, the standard AWS credential chain
467/// is used (env vars, shared config `~/.aws/config` + `~/.aws/credentials`,
468/// SSO, ECS/IMDS) — the same behavior as the AWS CLI. `profile` selects a
469/// named profile from the shared config, and `region` overrides the chain
470/// default.
471#[derive(Debug, Deserialize, Clone, Default)]
472pub struct AwsConfig {
473    /// Named profile from `~/.aws/config` / `~/.aws/credentials` to use
474    /// (default: the AWS CLI default selection via `AWS_PROFILE` or
475    /// `default`).
476    #[serde(default)]
477    pub profile: Option<String>,
478    /// AWS region (default: `AWS_REGION` env or the profile's region;
479    /// required if neither is set).
480    #[serde(default)]
481    pub region: Option<String>,
482    /// Explicit access key id (bypasses the credential chain).
483    #[serde(default)]
484    pub access_key_id: Option<String>,
485    /// Explicit secret access key (with `access_key_id`).
486    #[serde(default)]
487    pub secret_access_key: Option<String>,
488    /// Optional session token for explicit temporary credentials.
489    #[serde(default)]
490    pub session_token: Option<String>,
491}
492
493/// Pricing configuration for an endpoint.
494#[derive(Debug, Deserialize, Clone, Default)]
495pub struct PricingConfig {
496    /// Cost per 1M input tokens (USD).
497    #[serde(default)]
498    pub input_per_1m_tokens: f64,
499    /// Cost per 1M output tokens (USD).
500    #[serde(default)]
501    pub output_per_1m_tokens: f64,
502    /// Flat cost per call (USD), used for MCP/agent endpoints.
503    #[serde(default)]
504    pub per_call: f64,
505    /// Cost per 1M cached-input (prompt cache read) tokens (USD). When
506    /// unset, `input_per_1m_tokens * cache_read_multiplier` is used.
507    #[serde(default)]
508    pub cached_input_per_1m_tokens: Option<f64>,
509    /// Cost per 1M cache-write (cache creation) input tokens (USD). When
510    /// unset, `input_per_1m_tokens * cache_write_multiplier` is used.
511    #[serde(default)]
512    pub cache_write_per_1m_tokens: Option<f64>,
513    /// Multiplier applied to `input_per_1m_tokens` for cache reads when
514    /// `cached_input_per_1m_tokens` is unset. Default: 0.1.
515    #[serde(default)]
516    pub cache_read_multiplier: Option<f64>,
517    /// Multiplier applied to `input_per_1m_tokens` for cache writes when
518    /// `cache_write_per_1m_tokens` is unset. Default: 1.25.
519    #[serde(default)]
520    pub cache_write_multiplier: Option<f64>,
521    /// Bill cache reads/writes at their cache rates (default `true`). Set
522    /// to `false` to bill every prompt token at the flat input price.
523    #[serde(default)]
524    pub cache_pricing: Option<bool>,
525}
526
527/// Budget limits for test execution.
528#[derive(Debug, Deserialize, Clone, Default)]
529pub struct BudgetsConfig {
530    /// Global budget across all tests in the scenario.
531    #[serde(default)]
532    pub global: Option<BudgetDef>,
533    /// Default per-test budget. Individual tests can override.
534    #[serde(default)]
535    pub per_test_default: Option<BudgetDef>,
536}
537
538/// A budget definition with limits and enforcement mode.
539#[derive(Debug, Deserialize, Clone)]
540pub struct BudgetDef {
541    /// Maximum cost in USD.
542    #[serde(default)]
543    pub max_cost: Option<f64>,
544    /// Maximum total tokens (input + output).
545    #[serde(default)]
546    pub max_tokens: Option<u64>,
547    /// Maximum number of calls (LLM, MCP, agent combined).
548    #[serde(default)]
549    pub max_calls: Option<u64>,
550    /// Enforcement mode: `hard` (abort) or `soft` (warn and continue).
551    #[serde(default)]
552    pub enforcement: Option<BudgetEnforcement>,
553}
554
555/// Budget enforcement strategy.
556#[derive(Debug, Deserialize, Clone, PartialEq, Eq)]
557#[serde(rename_all = "lowercase")]
558pub enum BudgetEnforcement {
559    /// Abort the test or run when budget is exceeded.
560    Hard,
561    /// Log a warning but continue execution.
562    Soft,
563}
564
565/// MCP server exposure configuration.
566#[derive(Debug, Deserialize, Clone)]
567pub struct McpServerConfig {
568    /// Whether to enable the embedded MCP server.
569    #[serde(default)]
570    pub enabled: bool,
571    /// Port to listen on.
572    #[serde(default = "default_mcp_port")]
573    pub port: u16,
574}
575
576const fn default_mcp_port() -> u16 {
577    3000
578}
579
580/// A2A agent server exposure configuration.
581#[derive(Debug, Deserialize, Clone)]
582pub struct A2aServerConfig {
583    /// Whether to enable the embedded A2A agent server.
584    #[serde(default)]
585    pub enabled: bool,
586    /// Port to listen on.
587    #[serde(default = "default_a2a_port")]
588    pub port: u16,
589}
590
591const fn default_a2a_port() -> u16 {
592    3100
593}
594
595fn deserialize_headers<'de, D>(deserializer: D) -> Result<HashMap<String, String>, D::Error>
596where
597    D: serde::Deserializer<'de>,
598{
599    let raw: Option<serde_json::Value> = Option::deserialize(deserializer)?;
600    let Some(json) = raw else {
601        return Ok(HashMap::new());
602    };
603    let serde_json::Value::Object(obj) = json else {
604        return Ok(HashMap::new());
605    };
606    Ok(obj
607        .into_iter()
608        .filter_map(|(k, v)| v.as_str().map(|s| (k, s.to_owned())))
609        .collect())
610}
611
612const fn default_auto_navigate() -> bool {
613    true
614}
615
616fn default_layout_ignore_classes() -> Vec<String> {
617    vec![
618        "cdk-visually-hidden".to_owned(),
619        "cdk-describedby-message-container".to_owned(),
620        "cdk-overlay-container".to_owned(),
621    ]
622}
623
624const fn default_temperature() -> f64 {
625    0.0
626}
627
628fn deserialize_model_params<'de, D>(deserializer: D) -> Result<HashMap<String, Value>, D::Error>
629where
630    D: serde::Deserializer<'de>,
631{
632    #[derive(Deserialize)]
633    #[serde(untagged)]
634    enum Raw {
635        Map(HashMap<String, Value>),
636        Table(HashMap<String, Value>),
637    }
638    let raw: Option<Raw> = Option::deserialize(deserializer)?;
639    Ok(match raw {
640        Some(Raw::Map(m) | Raw::Table(m)) => m,
641        None => HashMap::new(),
642    })
643}
644
645fn deserialize_screenshot_max_height<'de, D>(
646    deserializer: D,
647) -> Result<Option<ScreenshotHeight>, D::Error>
648where
649    D: serde::Deserializer<'de>,
650{
651    let raw: Option<serde_json::Value> = Option::deserialize(deserializer)?;
652    match raw {
653        None => Ok(None),
654        Some(value) => match value.as_u64() {
655            // Integer: absolute pixel count.
656            Some(px) if px <= u64::from(MAX_SCREENSHOT_HEIGHT) => {
657                #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
658                Ok(Some(ScreenshotHeight::Pixels(px as u32)))
659            }
660            // String: viewport multiple like "2x" or "1.5x".
661            None => match value.as_str() {
662                Some(s) => {
663                    let lower = s.trim().to_lowercase();
664                    if lower.ends_with('x') {
665                        let num = lower.strip_suffix("x");
666                        if let Some(num) = num {
667                            if let Ok(m) = num.parse::<f64>() {
668                                if m > 0.0 {
669                                    return Ok(Some(ScreenshotHeight::ViewportTimes(m)));
670                                }
671                            }
672                        }
673                    }
674                    Err(serde::de::Error::custom(format_args!(
675                        "screenshot_max_height must be a pixel count (e.g. 2880) or a viewport multiple like \"2x\", found {s:?}"
676                    )))
677                }
678                None => Err(serde::de::Error::custom(format_args!(
679                    "screenshot_max_height must be a pixel count (e.g. 2880) or a viewport multiple like \"2x\", found {value:?}"
680                ))),
681            },
682            Some(_) => Err(serde::de::Error::custom(format_args!(
683                "screenshot_max_height value too large (max {MAX_SCREENSHOT_HEIGHT} px)"
684            ))),
685        },
686    }
687}
688
689/// Reusable assertion definition referenced by name from `assert` steps.
690///
691/// Definitions can either reference a built-in preset via `preset`, supply a
692/// custom LLM `prompt`, or define a **custom preset** by providing both
693/// `system` and `user_template`. Custom presets support the same template
694/// variables as built-in presets: `{url}`, `{title}`, `{content}`,
695/// `{expected_text}`, `{description}`.
696#[derive(Debug, Deserialize, Clone)]
697pub struct AssertDefinition {
698    /// Unique name used to reference this definition from steps.
699    pub name: String,
700    /// Predefined assertion preset name
701    /// (e.g. `no_error_on_page`, `text_visible`).
702    #[serde(default)]
703    pub preset: Option<String>,
704    /// Custom LLM prompt for assertion evaluation.
705    #[serde(default)]
706    pub prompt: Option<String>,
707    /// System prompt for a custom preset.
708    #[serde(default)]
709    pub system: Option<String>,
710    /// User template (with `{placeholders}`) for a custom preset.
711    #[serde(default)]
712    pub user_template: Option<String>,
713    /// Text that the `text_visible` preset checks for, or the
714    /// `{expected_text}` placeholder value for custom presets.
715    #[serde(default)]
716    pub assert_text: Option<String>,
717    /// Agent endpoint to call for this assertion.
718    #[serde(default)]
719    pub agent: Option<String>,
720    /// Agent task template for this assertion.
721    #[serde(default)]
722    pub task_template: Option<String>,
723}
724
725/// A group of steps that form a single test scenario.
726#[derive(Debug, Deserialize, Clone)]
727pub struct TestGroup {
728    /// Human-readable test name.
729    pub name: String,
730    /// Override the global `start_url` for this test.
731    #[serde(default)]
732    pub start_url: Option<String>,
733    /// Override the global `auto_navigate` for this test.
734    #[serde(default)]
735    pub auto_navigate: Option<bool>,
736    /// Override the global `base_url` for this test.
737    #[serde(default)]
738    pub base_url: Option<String>,
739    /// Override the global `timeout_secs` for this test.
740    #[serde(default)]
741    pub timeout_secs: Option<u64>,
742    /// Override the global `browser_headless` for this test.
743    #[serde(default)]
744    pub browser_headless: Option<bool>,
745    /// Override the global viewport width for this test (applied via CDP
746    /// `Emulation.setDeviceMetricsOverride` before the test runs).
747    #[serde(default)]
748    pub viewport_width: Option<u32>,
749    /// Override the global viewport height for this test.
750    #[serde(default)]
751    pub viewport_height: Option<u32>,
752    /// Per-test budget override.
753    #[serde(default)]
754    pub budget: Option<BudgetDef>,
755    /// Endpoint to use for all steps in this test (can be overridden
756    /// per-step).
757    #[serde(default)]
758    pub endpoint: Option<String>,
759    /// Ordered steps to execute.
760    #[serde(default)]
761    pub steps: Vec<TestStep>,
762}
763
764/// A single step in a test. The `kind` field determines which variant is
765/// deserialized and which field constraints apply.
766#[derive(Debug, Deserialize, Clone)]
767#[serde(tag = "kind")]
768pub enum TestStep {
769    /// Navigate the browser to a URL.
770    #[serde(rename = "navigate")]
771    Navigate {
772        /// URL to navigate to (absolute, or relative to the test's
773        /// base URL).
774        url: String,
775        /// Milliseconds to wait after navigation completes.
776        #[serde(default)]
777        wait_after_ms: Option<u64>,
778    },
779
780    /// Click an element described in natural language.
781    #[serde(rename = "click")]
782    Click {
783        /// Natural language description of the element. The LLM resolves
784        /// this to a CSS selector at runtime.
785        target: String,
786        /// Explicit CSS selector override (bypasses LLM resolution).
787        #[serde(default)]
788        selector: Option<String>,
789        /// Milliseconds to wait after the click.
790        #[serde(default)]
791        wait_after_ms: Option<u64>,
792        /// Endpoint to use for LLM element targeting.
793        #[serde(default)]
794        endpoint: Option<String>,
795        /// Idempotent: when the target element is absent the step is
796        /// reported skipped instead of failed (the action was already
797        /// done / not applicable).
798        #[serde(default)]
799        idempotent: bool,
800    },
801
802    /// Type text into an input element.
803    #[serde(rename = "type")]
804    Type {
805        /// Natural language description of the target input element.
806        target: String,
807        /// Text to type into the element.
808        text: String,
809        /// Explicit CSS selector override (bypasses LLM resolution).
810        #[serde(default)]
811        selector: Option<String>,
812        /// Milliseconds to wait after typing.
813        #[serde(default)]
814        wait_after_ms: Option<u64>,
815        /// Endpoint to use for LLM element targeting.
816        #[serde(default)]
817        endpoint: Option<String>,
818        /// Idempotent: when the target element is absent the step is
819        /// reported skipped instead of failed (the action was already
820        /// done / not applicable).
821        #[serde(default)]
822        idempotent: bool,
823    },
824
825    /// Wait for an element to appear on the page.
826    #[serde(rename = "wait")]
827    Wait {
828        /// Natural language description of the element to wait for.
829        target: String,
830        /// Explicit CSS selector override (bypasses LLM resolution).
831        #[serde(default)]
832        selector: Option<String>,
833        /// Wait until the page's visible text contains this substring
834        /// (alternative to `selector`; either or both may be set — both are
835        /// required to hold when both are set).
836        #[serde(default)]
837        text: Option<String>,
838        /// Maximum milliseconds to wait (default: 10000).
839        #[serde(default)]
840        timeout_ms: Option<u64>,
841        /// Endpoint to use for LLM element targeting.
842        #[serde(default)]
843        endpoint: Option<String>,
844        /// Idempotent: when the condition never becomes true within the
845        /// timeout the step is reported skipped instead of failed (the
846        /// condition was not applicable, e.g. already-authenticated
847        /// pages in a viewport matrix).
848        #[serde(default)]
849        idempotent: bool,
850    },
851
852    /// Evaluate an assertion against the current page content.
853    #[serde(rename = "assert")]
854    Assert {
855        /// Reference to a named `[[definitions]]` entry.
856        #[serde(default)]
857        definition: Option<String>,
858        /// Inline predefined assertion preset (e.g. `no_error_on_page`).
859        #[serde(default)]
860        preset: Option<String>,
861        /// Inline custom LLM prompt for assertion evaluation.
862        #[serde(default)]
863        prompt: Option<String>,
864        /// Text that the `text_visible` preset checks for.
865        #[serde(default)]
866        assert_text: Option<String>,
867        /// Endpoint to use for this assertion's LLM call.
868        #[serde(default)]
869        endpoint: Option<String>,
870        /// Attach a screenshot of the current viewport to the assertion so
871        /// the LLM can evaluate visuals (overlaps, clipping, layout).
872        /// Requires the resolved endpoint to declare `vision = true`.
873        #[serde(default)]
874        screenshot: bool,
875    },
876
877    /// Take a screenshot of the current page.
878    #[serde(rename = "screenshot")]
879    Screenshot {
880        /// File path to save the screenshot (default: `screenshot.png`).
881        #[serde(default)]
882        path: Option<String>,
883    },
884
885    /// Call an A2A agent with a task.
886    #[serde(rename = "agent")]
887    Agent {
888        /// Name of the agent endpoint to call.
889        agent: String,
890        /// Task description / prompt for the agent.
891        task: String,
892        /// Optional definition name with a task template.
893        #[serde(default)]
894        definition: Option<String>,
895    },
896
897    /// Call an MCP server tool.
898    #[serde(rename = "mcp")]
899    Mcp {
900        /// Name of the MCP server endpoint.
901        server: String,
902        /// Tool name to invoke on the server.
903        tool: String,
904        /// Tool arguments as JSON.
905        #[serde(default)]
906        args: Option<serde_json::Value>,
907    },
908}
llm_browser_testkit/scenario.rs

llm_browser_testkit/
scenario.rs
