use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::{Duration, Instant};
use anyhow::Context;
use headless_chrome::{Browser, LaunchOptions, Tab};
use crate::a2a::A2aClient;
use crate::budgets::{BudgetStatus, BudgetTracker};
use crate::costs::{calculate_llm_cost, UsageTracker};
use crate::diagnostics;
use crate::endpoints::{EndpointRegistry, ResolvedEndpoint, TaskType};
use crate::events::TestEvent;
use crate::llm_chat_vision_with_usage_chain;
use crate::llm_chat_with_usage_chain;
use crate::mcp_client::McpClient;
use crate::reporting::Reporter;
use crate::scenario::{AssertDefinition, ScenarioConfig, TestGroup, TestStep};
use crate::selectors::{sanitize_selector, selector_is_useless, validate_selector};
use crate::truncate;
use crate::LlmConfig;
use crate::DOM_EXTRACT_JS;
/// One detected layout defect (`layout_no_issues` preset).
#[derive(Debug, serde::Deserialize)]
struct LayoutIssue {
#[serde(rename = "type")]
issue_type: String,
element: String,
detail: String,
}
/// In-browser DOM layout scan for `layout_no_issues`.
///
/// Geometry-only checks (no LLM, no pixels):
/// 1. page horizontal overflow (`scrollWidth` > viewport width);
/// 2. elements outside the viewport that scrolling cannot reveal
/// (fixed elements off-screen, left/negative overflow, right-edge
/// overflow beyond the horizontally scrollable area, and bottom
/// overflow on a page that cannot scroll down) — below-the-fold
/// content on a tall scrollable page is normal flow, NOT a defect;
/// 3. text clipped by `overflow: hidden` containers whose content
/// is measurably larger than the box;
/// 4. interactive elements (buttons/links/inputs) whose center point
/// is covered by a different element that would intercept the click.
///
/// Elements whose class matches a configured ignore prefix (default:
/// the Angular CDK screen-reader helpers `cdk-visually-hidden`,
/// `cdk-describedby-message-container`, `cdk-overlay-container`) are
/// skipped — they are intentionally 1x1 / off-screen. The prefix list
/// is injected at `__IGNORE_CLASSES__` from
/// [`ScenarioConfig::layout_ignore_classes`].
///
/// Intentional stacking (off-canvas drawers, dropdown menus, badges,
/// sticky headers) is excluded by the position/relation filters.
const LAYOUT_SCAN_JS: &str = r#"
(() => {
const issues = [];
const push = (type, el, detail) => {
if (issues.length >= 30) return;
let element = el.tagName.toLowerCase();
if (el.id) element += '#' + el.id;
else if (typeof el.className === 'string' && el.className.trim())
element += '.' + el.className.trim().split(/\s+/).join('.');
issues.push({ type, element, detail: String(detail).slice(0, 220) });
};
const vw = document.documentElement.clientWidth || window.innerWidth;
const vh = document.documentElement.clientHeight || window.innerHeight;
if (!vw || !vh) return JSON.stringify(issues);
const de = document.documentElement;
// 1. Page-level horizontal overflow.
if (de.scrollWidth > vw + 2)
push('page-overflow-x', de,
'page scrollWidth ' + de.scrollWidth + ' exceeds viewport width ' + vw);
// Class-name prefixes to skip (injected; default = Angular CDK
// screen-reader helpers, which are intentionally 1x1 / off-screen).
const ignorePrefixes = __IGNORE_CLASSES__;
const isIgnored = (el) => {
if (typeof el.className !== 'string' || !el.className.trim()) return false;
const classes = el.className.trim().split(/\s+/);
return classes.some((c) => ignorePrefixes.some((p) => c.startsWith(p)));
};
const vScrollable = de.scrollHeight > vh + 2;
const hScrollable = de.scrollWidth > vw + 2;
const all = Array.prototype.slice.call(document.querySelectorAll('body *'));
const visible = (cs) => cs.display !== 'none' && cs.visibility !== 'hidden' && parseFloat(cs.opacity || '1') !== 0;
const hasContent = (el) =>
((el.textContent || '').trim().length > 0) ||
!!el.querySelector('img,svg,video,canvas,iframe,button,input,textarea,select');
// 2. Elements outside the viewport that scrolling cannot reveal.
for (const el of all) {
const cs = getComputedStyle(el);
if (!visible(cs)) continue;
if (isIgnored(el)) continue;
const r = el.getBoundingClientRect();
if (r.width < 2 || r.height < 2) continue;
if (!hasContent(el) && el.children.length === 0) continue;
if (cs.position === 'fixed') {
// Fixed elements never move with the scroll: any edge outside the
// viewport is unreachable content and therefore a defect.
const overTop = -r.top;
const overLeft = -r.left;
const overRight = r.right - vw;
const overBottom = r.bottom - vh;
if (overTop > 2 || overLeft > 2 || overRight > 2 || overBottom > 2) {
let where = '';
if (overTop > 2 && overLeft > 2) where = 'top+left edges';
else if (overTop > 2) where = 'top edge (' + Math.round(r.top) + ' < 0)';
else if (overLeft > 2) where = 'left edge (' + Math.round(r.left) + ' < 0)';
else if (overRight > 2 && overBottom > 2) where = 'right+bottom edges';
else if (overRight > 2) where = 'right edge (' + Math.round(r.right) + ' > ' + vw + ')';
else where = 'bottom edge (' + Math.round(r.bottom) + ' > ' + vh + ')';
push('element-out-of-viewport', el,
'fixed element lies ' + Math.max(overTop, overLeft, overRight, overBottom).toFixed(0) + 'px past the ' + where);
}
continue;
}
if (cs.position === 'sticky') continue;
// Negative left overflow cannot be reached by scrolling (scrollLeft
// never goes below 0).
if (r.left < -2) {
push('element-out-of-viewport', el,
'extends ' + Math.round(-r.left).toFixed(0) + 'px past the left edge (' + Math.round(r.left) + ' < 0)');
continue;
}
// Negative top with the page at the top means the element sits above
// the document origin — also unreachable.
if (r.top < -2 && de.scrollTop <= 2) {
push('element-out-of-viewport', el,
'extends ' + Math.round(-r.top).toFixed(0) + 'px past the top edge (' + Math.round(r.top) + ' < 0)');
continue;
}
const overRight = r.right - vw;
// Right-edge overflow is only a defect when the page cannot scroll
// horizontally to reveal it (or the element sticks out past the
// scrollable content width itself).
if (overRight > 2 && (!hScrollable || r.right > de.scrollWidth + 2)) {
push('element-out-of-viewport', el,
'extends ' + overRight.toFixed(0) + 'px past the right edge (' + Math.round(r.right) + ' > ' + vw + ')');
continue;
}
// Below-the-fold content on a scrollable page is normal (tall landing
// pages); only flag bottom overflow the user can never scroll to.
const overBottom = r.bottom - vh;
if (overBottom > 2 && (!vScrollable || r.bottom > de.scrollHeight + 2)) {
push('element-out-of-viewport', el,
'extends ' + overBottom.toFixed(0) + 'px past the bottom edge (' + Math.round(r.bottom) + ' > ' + vh + ')');
}
}
// 3. Text clipped by overflow:hidden containers.
for (const el of all) {
if (isIgnored(el)) continue;
const cs = getComputedStyle(el);
if (cs.overflowX !== 'hidden' && cs.overflowY !== 'hidden') continue;
if (el.scrollWidth <= el.clientWidth + 2 && el.scrollHeight <= el.clientHeight + 2) continue;
if (!(el.textContent || '').trim()) continue;
push('text-clipped', el,
'content ' + el.scrollWidth + 'x' + el.scrollHeight +
' clipped to ' + el.clientWidth + 'x' + el.clientHeight);
}
// 4. Interactive elements covered by a different element.
const interactive =
'button, a[href], input, textarea, select, [role="button"], [role="link"], [role="menuitem"], label, .mdc-button, .mat-mdc-button, .mat-mdc-icon-button, .mdc-fab';
const targets = document.querySelectorAll(interactive);
for (const el of targets) {
if (isIgnored(el)) continue;
const r = el.getBoundingClientRect();
if (r.width < 6 || r.height < 6) continue;
const cs = getComputedStyle(el);
if (!visible(cs)) continue;
const cx = r.left + r.width / 2;
const cy = r.top + r.height / 2;
if (cx < 0 || cy < 0 || cx > vw || cy > vh) continue;
const top = document.elementFromPoint(cx, cy);
if (!top || top === el || el.contains(top) || top.contains(el)) continue;
if (isIgnored(top)) continue;
const tcs = getComputedStyle(top);
if (!visible(tcs)) continue;
if (tcs.pointerEvents === 'none') continue;
const tr = top.getBoundingClientRect();
if (tr.width * tr.height < r.width * r.height * 0.25) continue;
const tname = top.tagName.toLowerCase() + (top.id ? '#' + top.id : '') +
(typeof top.className === 'string' && top.className.trim() ? '.' + top.className.trim().split(/\s+/).join('.') : '');
push('element-overlap', el,
'center point at ' + Math.round(cx) + ',' + Math.round(cy) +
' is covered by <' + tname + '>');
}
return JSON.stringify(issues);
})()
"#;
/// How long the CDP connection stays open after the browser goes quiet.
///
/// `headless_chrome` ships a 30s default and tears down the entire connection
/// when no traffic arrives for that long; a run must own its connection for
/// its full duration instead.
const BROWSER_IDLE_TIMEOUT: Duration = Duration::from_secs(6 * 60 * 60);
/// Executes a [`Scenario`] against a real browser with optional LLM
/// assistance for element targeting and assertions.
pub struct ScenarioRunner {
config: ScenarioConfig,
definitions: HashMap<String, AssertDefinition>,
llm: LlmConfig,
timeout: Duration,
viewport_width: u32,
viewport_height: u32,
/// The viewport currently applied in the browser (CDP emulation).
/// Per-test overrides switch it mid-run; `(0, 0)` = not yet applied.
applied_viewport: std::cell::Cell<(u32, u32)>,
endpoints: EndpointRegistry,
usage: Arc<UsageTracker>,
budgets: BudgetTracker,
/// Directory for failure artifacts (screenshots).
artifacts_dir: PathBuf,
/// Event sink for test reporting (console, `JSONL`, `JUnit`, trace).
reporter: Arc<Reporter>,
/// The test + step index currently executing, for LLM-call events.
current_step: std::cell::RefCell<Option<(String, u32)>>,
}
/// Aggregated results from a scenario run.
#[derive(Debug, Default)]
pub struct RunReport {
/// Number of tests that passed.
pub tests_passed: u32,
/// Number of tests that failed.
pub tests_failed: u32,
/// Number of steps that passed.
pub passed: u32,
/// Number of steps that failed.
pub failed: u32,
/// Number of steps that were skipped.
pub skipped: u32,
/// Per-step details.
pub details: Vec<StepResult>,
}
/// Result of a single step execution.
#[derive(Debug)]
pub struct StepResult {
/// The step name.
pub name: String,
/// Whether the step passed, failed, or was skipped.
pub status: StepStatus,
/// Human-readable result message.
pub message: String,
}
/// Outcome for a single step.
pub use crate::events::StepStatus;
/// Predefined assertion preset definition.
struct AssertPreset {
name: &'static str,
system: &'static str,
user_template: &'static str,
}
/// Built-in assertion presets.
#[allow(clippy::literal_string_with_formatting_args)]
const ASSERTION_PRESETS: &[AssertPreset] = &[
AssertPreset {
name: "no_error_on_page",
system: "You are a QA tester. Evaluate if a web page contains error messages, stack traces, exception text, HTTP error codes, 'undefined' errors, or any indication of a malfunction. Be strict — even minor rendering glitches count as errors.",
user_template: "Check if the following page content contains ANY errors or malfunctions:\n\nURL: {url}\nTitle: {title}\n\nPage Content:\n{content}\n\nRespond with exactly \"PASS\" if there are NO errors, or \"FAIL: <reason>\" if there are errors. Only respond with PASS or FAIL.",
},
AssertPreset {
name: "text_visible",
system: "You are a QA tester. Your task is to check if specific text is visible in the page content.",
user_template: "Check if the following text appears in the page content:\n\nTEXT TO FIND: \"{expected_text}\"\n\nURL: {url}\n\nPage Content:\n{content}\n\nRespond with exactly \"PASS\" if the text is present (even partial match is OK), or \"FAIL: text not found\" if it is not.",
},
AssertPreset {
name: "element_exists",
system: "You are a QA tester. Check if a described UI element exists on a web page.",
user_template: "Check if the following element exists on the page:\n\nELEMENT: \"{description}\"\n\nURL: {url}\n\nPage Content:\n{content}\n\nRespond with exactly \"PASS\" if the element exists, or \"FAIL: <reason>\" if it does not.",
},
AssertPreset {
name: "visual_no_issues",
system: "You are a visual QA engineer inspecting a website screenshot. Detect clearly visible layout and rendering defects: overlapping elements that hide content or controls, clipped or truncated text, content cut off at the viewport edges, misaligned or broken UI, blank/empty panels where content is expected, broken or missing images, duplicated elements, or rendering glitches. Ignore subjective aesthetics, intentional stacking (dropdowns, tooltips, layered design), and content that is simply not loaded (empty states). Only report defects that a user would actually see or be blocked by.",
user_template: "Inspect the attached screenshot(s) (ordered from the top of the page down) and the page text below.\n\nURL: {url}\nTitle: {title}\n\nPage Content:\n{content}\n\nAre there any clearly visible layout or rendering defects (overlaps, clipping, cut-off content, broken images, blank panels)? Respond with exactly \"PASS\" if the page renders cleanly, or \"FAIL: <describe each defect and where it appears>\" otherwise.",
},
AssertPreset {
name: "visual_no_overlaps",
system: "You are a visual QA engineer inspecting a website screenshot for OVERLAPPING elements that harm usability: one element covering another element's text, buttons, links, or input fields (cookie banners, modals, popovers, chat widgets, sticky headers, or mispositioned layers that hide content or intercept clicks). Ignore intentional, non-harmful stacking (dropdowns, tooltips, badges over avatars, layered design where nothing is hidden or unclickable). Only fail on overlaps that visibly hide content or would block a click.",
user_template: "Inspect the attached screenshot(s) (ordered from the top of the page down) and the page text below.\n\nURL: {url}\nTitle: {title}\n\nPage Content:\n{content}\n\nAre any elements overlapping in a way that hides page content, text, or interactive controls, or that would block clicks? Respond with exactly \"PASS\" if there are no such overlaps, or \"FAIL: <describe the overlapping elements and what they hide>\" otherwise.",
},
AssertPreset {
name: "visual_text_visible",
system: "You are a visual QA engineer inspecting a website screenshot. Determine whether a specific text is FULLY visible and readable: present in the viewport, not clipped, not cut off, not covered by another element, and not obscured by overlays or low contrast stacking. A partial word or a covered text counts as FAIL.",
user_template: "Inspect the attached screenshot(s) (ordered from the top of the page down) and the page text below.\n\nTEXT TO CHECK: \"{expected_text}\"\n\nURL: {url}\nTitle: {title}\n\nPage Content:\n{content}\n\nIs the text fully visible and readable in the screenshot (not clipped, covered, or hidden)? Respond with exactly \"PASS\" if it is fully visible, or \"FAIL: <explain what hides or clips it>\" otherwise.",
},
AssertPreset {
name: "layout_no_issues",
system: "DOM layout scanner: reports horizontal page overflow, elements sticking out of the viewport, clipped text in overflow:hidden containers, and interactive elements covered by other elements. Deterministic in-browser checks — no LLM call.",
user_template: "Runs a deterministic DOM layout scan in the browser (no LLM call). Fails with the list of detected issues.",
},
];
impl ScenarioRunner {
/// Creates a new runner with the given scenario configuration and
/// assertion definitions.
#[must_use]
#[allow(clippy::needless_pass_by_value)]
pub fn new(scenario_config: ScenarioConfig, definitions: Vec<AssertDefinition>) -> Self {
Self::with_reporter(scenario_config, definitions, Arc::new(Reporter::default()))
}
/// Creates a runner that reports run events through the given reporter.
#[must_use]
#[allow(clippy::needless_pass_by_value)]
pub fn with_reporter(
scenario_config: ScenarioConfig,
definitions: Vec<AssertDefinition>,
reporter: Arc<Reporter>,
) -> Self {
reporter.add_redaction_secrets(crate::redact::collect_secrets_from_scenario_config(
&scenario_config,
));
let llm = LlmConfig {
url: scenario_config
.llm_url
.clone()
.unwrap_or_else(crate::llm_base_url),
model: scenario_config
.llm_model
.clone()
.unwrap_or_else(crate::llm_model),
api_key: scenario_config
.llm_api_key
.clone()
.or_else(|| std::env::var("HARNESS_LLM_API_KEY").ok()),
headers: if scenario_config.llm_headers.is_empty() {
crate::parse_headers_env()
} else {
scenario_config.llm_headers.clone()
},
timeout: Duration::from_secs(scenario_config.timeout_secs.unwrap_or(60)),
temperature: scenario_config.temperature,
thinking: scenario_config.thinking,
model_params: scenario_config.model_params.clone(),
max_attempts: crate::default_llm_attempts(),
provider: crate::scenario::Provider::Openai,
deployment: None,
api_version: None,
auth: crate::scenario::AuthConfig::default(),
header_commands: std::collections::HashMap::new(),
aws: crate::scenario::AwsConfig::default(),
};
let endpoints = EndpointRegistry::from_config(&scenario_config.endpoints, Some(&llm));
let budgets = BudgetTracker::from_config(&scenario_config.budgets);
let defs_map: HashMap<String, AssertDefinition> = definitions
.into_iter()
.map(|d| (d.name.clone(), d))
.collect();
Self {
timeout: Duration::from_secs(scenario_config.timeout_secs.unwrap_or(60)),
viewport_width: scenario_config.viewport_width.unwrap_or(1280),
viewport_height: scenario_config.viewport_height.unwrap_or(720),
applied_viewport: std::cell::Cell::new((0, 0)),
config: scenario_config.clone(),
definitions: defs_map,
llm,
endpoints,
usage: Arc::new(UsageTracker::new()),
budgets,
artifacts_dir: PathBuf::from(
scenario_config
.artifacts_dir
.unwrap_or_else(|| "artifacts".to_owned()),
),
reporter,
current_step: std::cell::RefCell::new(None),
}
}
/// Emits an event; a sink failure degrades to a console warning so a
/// broken log file can never mask the run itself.
fn emit_event(&self, event: &TestEvent) {
if let Err(err) = self.reporter.emit(event) {
use std::io::Write as _;
let mut out = std::io::stderr().lock();
let _ = out.write_fmt(format_args!(" ! failed to record event: {err}\n"));
}
}
/// Returns a clone of the [`UsageTracker`] for reporting.
#[must_use]
pub fn usage_tracker(&self) -> Arc<UsageTracker> {
Arc::clone(&self.usage)
}
/// Returns a reference to the [`BudgetTracker`].
#[must_use]
pub const fn budget_tracker(&self) -> &BudgetTracker {
&self.budgets
}
/// Executes all test groups in the scenario and returns a report.
///
/// # Errors
///
/// Returns an error if the browser fails to launch.
#[allow(clippy::too_many_lines, clippy::cast_possible_truncation)]
pub fn run(&self, tests: &[TestGroup]) -> anyhow::Result<RunReport> {
let mut report = RunReport::default();
if tests.is_empty() {
self.reporter.warn("No tests defined in scenario.");
self.emit_event(&TestEvent::RunFinished {
tests_passed: 0,
tests_failed: 0,
steps_passed: 0,
steps_failed: 0,
steps_skipped: 0,
total_cost: 0.0,
total_tokens: 0,
total_calls: 0,
});
return Ok(report);
}
self.emit_event(&TestEvent::RunStarted {
total_tests: tests.len() as u32,
});
let browser_headless = self.config.browser_headless.unwrap_or(true);
let launch_opts = LaunchOptions {
headless: browser_headless,
window_size: Some((self.viewport_width, self.viewport_height)),
sandbox: false,
// headless_chrome defaults this to 30s and shuts down the whole CDP
// connection when no messages arrive for that long. A scenario can
// easily exceed 30s of browser silence (slow LLM targeting/assertion
// calls, page waits, budget checks between steps), after which every
// remaining step fails with "Unable to make method calls because
// underlying connection is closed" — one quiet gap kills the run.
// Open-ended scenarios must own the connection for their full
// duration, so keep it alive for 6 hours.
idle_browser_timeout: BROWSER_IDLE_TIMEOUT,
..LaunchOptions::default()
};
let browser = Browser::new(launch_opts).context("failed to launch browser")?;
let tab = browser.new_tab().context("failed to open browser tab")?;
let _ = tab.set_default_timeout(self.timeout);
// Start MCP server if configured
#[cfg(feature = "mcp-server")]
if let Some(ref mcp_cfg) = self.config.mcp_server {
if mcp_cfg.enabled {
let port = mcp_cfg.port;
std::thread::spawn(move || {
let _ = crate::mcp_server::start_mcp_server(port);
});
}
}
#[cfg(not(feature = "mcp-server"))]
if let Some(mcp_cfg) = &self.config.mcp_server {
if mcp_cfg.enabled {
self.reporter
.warn("MCP server configured but 'mcp-server' feature not enabled");
}
}
// Start A2A agent server if configured
#[cfg(feature = "a2a-server")]
if let Some(ref a2a_cfg) = self.config.a2a_server {
if a2a_cfg.enabled {
let port = a2a_cfg.port;
tokio::spawn(crate::a2a_server::start_a2a_server(port));
}
}
#[cfg(not(feature = "a2a-server"))]
if let Some(a2a_cfg) = &self.config.a2a_server {
if a2a_cfg.enabled {
self.reporter
.warn("A2A server configured but 'a2a-server' feature not enabled");
}
}
for test in tests {
self.emit_event(&TestEvent::TestStarted {
test: test.name.clone(),
});
self.usage.reset_per_test();
let test_started = Instant::now();
let test_result = self.run_test(test, &tab);
let duration_ms = test_started.elapsed().as_millis() as u64;
let usage = self.usage.current_test_snapshot();
self.usage.commit_test(&test.name);
self.emit_event(&TestEvent::TestFinished {
test: test.name.clone(),
passed: test_result.passed,
failed: test_result.failed,
skipped: test_result.skipped,
duration_ms,
cost: usage.total_cost,
tokens: usage.total_tokens,
calls: usage.total_calls,
});
if test_result.failed == 0 && test_result.total > 0 {
report.tests_passed += 1;
} else if test_result.total > 0 {
report.tests_failed += 1;
}
report.passed += test_result.passed;
report.failed += test_result.failed;
report.skipped += test_result.skipped;
report.details.extend(test_result.details);
}
let global = self.usage.global_snapshot();
self.emit_event(&TestEvent::RunFinished {
tests_passed: report.tests_passed,
tests_failed: report.tests_failed,
steps_passed: report.passed,
steps_failed: report.failed,
steps_skipped: report.skipped,
total_cost: global.total_cost,
total_tokens: global.total_tokens,
total_calls: global.total_calls,
});
Ok(report)
}
#[allow(clippy::too_many_lines, clippy::cast_possible_truncation)]
fn run_test(&self, test: &TestGroup, tab: &Tab) -> TestRunResult {
let base_url = test
.base_url
.clone()
.or_else(|| self.config.base_url.clone())
.unwrap_or_else(crate::base_url);
// Per-test viewport override: switch the browser via CDP
// device-metrics emulation before this test runs.
let vw = test.viewport_width.unwrap_or(self.viewport_width);
let vh = test.viewport_height.unwrap_or(self.viewport_height);
if self.applied_viewport.get() != (vw, vh) {
self.apply_viewport(tab, vw, vh);
self.applied_viewport.set((vw, vh));
}
// Per-test isolation: every test starts from its own start_url
// (unless auto_navigate is disabled), so a test never inherits the
// previous test's page state.
let auto_navigate = test.auto_navigate.unwrap_or(self.config.auto_navigate);
let start_url = test
.start_url
.clone()
.or_else(|| self.config.start_url.clone())
.unwrap_or_else(|| "/dashboard".to_owned());
if auto_navigate {
let full_url = resolve_url(&start_url, &base_url);
self.reporter.debug(format!("auto-navigate: {full_url}"));
let _ = tab.navigate_to(&full_url);
let _ = tab.wait_until_navigated();
std::thread::sleep(Duration::from_secs(4));
}
let mut result = TestRunResult::default();
for (step_index, step) in test.steps.iter().enumerate() {
result.total += 1;
let wait_ms = match step {
TestStep::Navigate { wait_after_ms, .. }
| TestStep::Click { wait_after_ms, .. }
| TestStep::Type { wait_after_ms, .. } => *wait_after_ms,
_ => None,
};
self.current_step
.replace(Some((test.name.clone(), step_index as u32)));
self.emit_event(&TestEvent::StepStarted {
test: test.name.clone(),
index: step_index as u32,
label: step_label(step),
});
let step_started = Instant::now();
let mut step_result = match step {
TestStep::Navigate { url, .. } => {
let full_url = resolve_url(url, &base_url);
run_navigate_step(&full_url, tab)
}
TestStep::Click {
target,
selector,
endpoint,
idempotent,
..
} => self.run_click(
target,
selector.as_deref(),
endpoint.as_deref(),
test.endpoint.as_deref(),
*idempotent,
tab,
),
TestStep::Type {
target,
text,
selector,
endpoint,
idempotent,
..
} => self.run_type(
target,
text,
selector.as_deref(),
endpoint.as_deref(),
test.endpoint.as_deref(),
*idempotent,
tab,
),
TestStep::Wait {
target,
selector,
text,
timeout_ms,
endpoint,
idempotent,
} => self.run_wait(
target,
selector.as_deref(),
text.as_deref(),
*timeout_ms,
endpoint.as_deref(),
test.endpoint.as_deref(),
*idempotent,
tab,
),
TestStep::Assert {
definition,
preset,
prompt,
assert_text,
endpoint,
screenshot,
} => self.run_assert(
definition.as_deref(),
preset.as_deref(),
prompt.as_deref(),
assert_text.as_deref(),
*screenshot,
endpoint.as_deref(),
test.endpoint.as_deref(),
tab,
),
TestStep::Screenshot { path } => Self::run_screenshot(path.as_deref(), tab),
TestStep::Agent {
agent,
task,
definition,
} => self.run_agent(agent, task, definition.as_deref(), test.endpoint.as_deref()),
TestStep::Mcp { server, tool, args } => self.run_mcp(server, tool, args.as_ref()),
};
// Failure diagnostics: capture the page state and a screenshot so
// CI logs say WHAT the page looked like when the step failed,
// instead of a bare "timed out: The event waited for never came".
let (diagnostics_block, screenshot_path) = if step_result.status == StepStatus::Failed {
let state = diagnostics::capture(tab);
let screenshot = diagnostics::save_screenshot(
tab,
&self.artifacts_dir,
&test.name,
&test.name,
step_index,
step_kind_label(step),
);
step_result.message = format!(
"{base} — {excerpt}",
base = step_result.message,
excerpt = diagnostics::inline_excerpt(&state),
);
(Some(diagnostics::full_context(&state)), screenshot)
} else {
(None, None)
};
let duration_ms = step_started.elapsed().as_millis() as u64;
self.emit_event(&TestEvent::StepFinished {
test: test.name.clone(),
index: step_index as u32,
label: step_result.name.clone(),
status: step_result.status,
duration_ms,
message: step_result.message.clone(),
diagnostics: diagnostics_block,
screenshot: screenshot_path,
});
self.current_step.replace(None);
match step_result.status {
StepStatus::Passed => result.passed += 1,
StepStatus::Failed => result.failed += 1,
StepStatus::Skipped => result.skipped += 1,
}
// Fail fast: the first failed step ends the test and the
// remaining steps are reported as skipped (no LLM budget is
// burned asserting against a page that is already known broken).
if step_result.status == StepStatus::Failed
&& !self.config.continue_on_failure
&& step_index + 1 < test.steps.len()
{
self.emit_event(&TestEvent::Warning {
message: format!(
"failing fast: {} remaining step(s) skipped (set continue_on_failure = true in [config] to disable)",
test.steps.len() - step_index - 1
),
});
for (offset, skipped) in test.steps[step_index + 1..].iter().enumerate() {
let skipped_index = step_index + 1 + offset;
let label = step_label(skipped);
result.total += 1;
result.skipped += 1;
self.emit_event(&TestEvent::StepStarted {
test: test.name.clone(),
index: skipped_index as u32,
label: label.clone(),
});
self.emit_event(&TestEvent::StepFinished {
test: test.name.clone(),
index: skipped_index as u32,
label,
status: StepStatus::Skipped,
duration_ms: 0,
message: "skipped: previous step failed".into(),
diagnostics: None,
screenshot: None,
});
result.details.push(StepResult {
name: step_label(skipped),
status: StepStatus::Skipped,
message: "skipped: previous step failed".into(),
});
}
result.details.push(step_result);
return result;
}
// Check per-test budget after each step
let test_usage = self.usage.current_test_snapshot();
let global_usage = self.usage.global_snapshot();
let budget_status = self.budgets.check_all(
&test.name,
&test_usage,
&global_usage,
test.budget.as_ref(),
);
match budget_status {
BudgetStatus::HardExceeded { message, .. } => {
self.emit_event(&TestEvent::Warning {
message: format!("budget exceeded: {message}"),
});
result.details.push(StepResult {
name: "[budget]".into(),
status: StepStatus::Failed,
message,
});
result.failed += 1;
return result;
}
BudgetStatus::SoftExceeded { message, .. } => {
self.emit_event(&TestEvent::Warning {
message: format!("budget warning: {message}"),
});
}
BudgetStatus::Ok => {}
}
if let Some(ms) = wait_ms {
std::thread::sleep(Duration::from_millis(ms));
}
result.details.push(step_result);
}
result
}
/// Applies a viewport size to the current tab via CDP
/// `Emulation.setDeviceMetricsOverride`. Used by per-test viewport
/// overrides and the viewport matrix. The initial window size set at
/// browser launch is replaced by emulation; failures are logged but
/// do not fail the test (a mismatched viewport only weakens coverage).
fn apply_viewport(&self, tab: &Tab, width: u32, height: u32) {
use headless_chrome::protocol::cdp::Emulation;
let _ = self;
let params = Emulation::SetDeviceMetricsOverride {
width,
height,
device_scale_factor: 1.0,
mobile: false,
scale: None,
screen_width: Some(width),
screen_height: Some(height),
position_x: None,
position_y: None,
dont_set_visible_size: None,
screen_orientation: None,
viewport: None,
display_feature: None,
device_posture: None,
};
self.reporter.debug(format!("viewport: {width}x{height}"));
if let Err(e) = tab.call_method(params) {
self.reporter
.warn(format!("viewport switch to {width}x{height} failed: {e}"));
}
}
/// Height (px) covered by assert-step screenshots: the configured
/// `screenshot_max_height` (absolute px or viewport multiple, default
/// `"20x"`) resolved against the currently applied viewport, raised to
/// at least the viewport height so the visible screen is always fully
/// included. The capture is split into viewport-tall tiles, so this
/// value bounds total coverage (and hence the number of image parts).
#[must_use]
fn screenshot_height_cap(&self) -> u32 {
let viewport_height = self.current_viewport_height();
let cap = self
.config
.screenshot_max_height
.as_ref()
.map_or(viewport_height * 20, |h| h.to_px(viewport_height));
cap.max(viewport_height)
}
/// Height of the viewport currently emulated in the browser (falling
/// back to the configured default before any emulation was applied).
#[must_use]
const fn current_viewport_height(&self) -> u32 {
let (_, height) = self.applied_viewport.get();
if height > 0 {
height
} else {
self.viewport_height
}
}
// ── step handlers ───────────────────────────────────────────────────
#[allow(clippy::too_many_lines)]
fn run_click(
&self,
target: &str,
selector_override: Option<&str>,
step_endpoint: Option<&str>,
test_endpoint: Option<&str>,
idempotent: bool,
tab: &Tab,
) -> StepResult {
let name = format!("[click] {target}");
let selector = match self.resolve_selector(
selector_override,
target,
step_endpoint,
test_endpoint,
tab,
) {
Ok(s) => s,
Err(msg) => {
if idempotent {
return StepResult {
name,
status: StepStatus::Skipped,
message: format!("skipped (idempotent): no target found — {msg}"),
};
}
return StepResult {
name,
status: StepStatus::Failed,
message: msg,
};
}
};
// Idempotent steps probe briefly: a missing target means the
// action was already done / not applicable (e.g. an
// already-authenticated session), and skipping is the success
// path, not a failure.
let probe_secs = if idempotent { 5 } else { 10 };
match tab.wait_for_element_with_custom_timeout(&selector, Duration::from_secs(probe_secs)) {
Ok(element) => match element.click() {
Ok(_) => StepResult {
name,
status: StepStatus::Passed,
message: format!("clicked {selector}"),
},
Err(e) => StepResult {
name,
status: StepStatus::Failed,
message: format!("click failed on {selector}: {e}"),
},
},
Err(e) if idempotent => StepResult {
name,
status: StepStatus::Skipped,
message: format!("skipped (idempotent): element {selector} not present — {e}"),
},
Err(e) => StepResult {
name,
status: StepStatus::Failed,
message: format!("element {selector} not found: {e}"),
},
}
}
#[allow(clippy::too_many_arguments)]
fn run_type(
&self,
target: &str,
text: &str,
selector_override: Option<&str>,
step_endpoint: Option<&str>,
test_endpoint: Option<&str>,
idempotent: bool,
tab: &Tab,
) -> StepResult {
let name = format!("[type] {target}");
let selector = match self.resolve_selector(
selector_override,
target,
step_endpoint,
test_endpoint,
tab,
) {
Ok(s) => s,
Err(msg) => {
if idempotent {
return StepResult {
name,
status: StepStatus::Skipped,
message: format!("skipped (idempotent): no target found — {msg}"),
};
}
return StepResult {
name,
status: StepStatus::Failed,
message: msg,
};
}
};
let probe_secs = if idempotent { 5 } else { 10 };
match tab.wait_for_element_with_custom_timeout(&selector, Duration::from_secs(probe_secs)) {
Ok(element) => {
if let Err(e) = element.click() {
return StepResult {
name,
status: StepStatus::Failed,
message: format!("click to focus {selector} failed: {e}"),
};
}
let js = format!(
"document.querySelector('{}').value = '';",
selector.replace('\'', "\\'")
);
let _ = tab.evaluate(&js, false);
match element.type_into(text) {
Ok(_) => StepResult {
name,
status: StepStatus::Passed,
message: format!("typed {text:?} into {selector}"),
},
Err(e) => StepResult {
name,
status: StepStatus::Failed,
message: format!("type into {selector} failed: {e}"),
},
}
}
Err(e) if idempotent => StepResult {
name,
status: StepStatus::Skipped,
message: format!("skipped (idempotent): element {selector} not present — {e}"),
},
Err(e) => StepResult {
name,
status: StepStatus::Failed,
message: format!("element {selector} not found: {e}"),
},
}
}
#[allow(clippy::too_many_arguments)]
#[allow(clippy::too_many_lines)]
fn run_wait(
&self,
target: &str,
selector_override: Option<&str>,
text: Option<&str>,
timeout_ms: Option<u64>,
step_endpoint: Option<&str>,
test_endpoint: Option<&str>,
idempotent: bool,
tab: &Tab,
) -> StepResult {
let timeout = Duration::from_millis(timeout_ms.unwrap_or(10_000));
let step_name = format!("[wait] {target}");
// Resolve an explicit selector only (text-only waits are LLM-free).
let selector = match selector_override {
Some(s) => Some(s.to_owned()),
None if text.is_some() => None,
None => match self.resolve_selector(None, target, step_endpoint, test_endpoint, tab) {
Ok(s) => Some(s),
Err(msg) => {
if idempotent {
return StepResult {
name: step_name,
status: StepStatus::Skipped,
message: format!("skipped (idempotent): no target found — {msg}"),
};
}
return StepResult {
name: step_name,
status: StepStatus::Failed,
message: msg,
};
}
},
};
if text.is_some() {
let sel_js = selector
.as_deref()
.map(crate::selectors::selector_matches_js);
let text_js = text.map(|t| {
let escaped = t.replace('\\', "\\\\").replace('\'', "\\'");
format!("document.body ? document.body.innerText.includes('{escaped}') : false")
});
let deadline = Instant::now() + timeout;
loop {
let sel_ok = sel_js
.as_ref()
.is_none_or(|js| eval_bool(tab, js).unwrap_or(false));
let text_ok = text_js
.as_ref()
.is_none_or(|js| eval_bool(tab, js).unwrap_or(false));
if sel_ok && text_ok {
let mut what = Vec::new();
if let Some(sel) = &selector {
what.push(format!("found {sel}"));
}
if let Some(t) = text {
what.push(format!("text {t:?} visible"));
}
return StepResult {
name: step_name,
status: StepStatus::Passed,
message: what.join(" and "),
};
}
if Instant::now() >= deadline {
let mut what = Vec::new();
if let Some(sel) = &selector {
what.push(sel.clone());
}
if let Some(t) = text {
what.push(format!("text {t:?}"));
}
let message = format!(
"wait for {} timed out after {}ms: the event waited for never came",
what.join(" / "),
timeout.as_millis(),
);
if idempotent {
return StepResult {
name: step_name,
status: StepStatus::Skipped,
message: format!("skipped (idempotent): {message}"),
};
}
return StepResult {
name: step_name,
status: StepStatus::Failed,
message,
};
}
std::thread::sleep(Duration::from_millis(250));
}
}
match selector.as_deref() {
Some(sel) => match tab.wait_for_element_with_custom_timeout(sel, timeout) {
Ok(_) => StepResult {
name: step_name,
status: StepStatus::Passed,
message: format!("found {sel}"),
},
Err(e) if idempotent => StepResult {
name: step_name,
status: StepStatus::Skipped,
message: format!(
"skipped (idempotent): wait for {sel} timed out after {}ms: {e}",
timeout.as_millis()
),
},
Err(e) => StepResult {
name: step_name,
status: StepStatus::Failed,
message: format!(
"wait for {sel} timed out after {}ms: {e}",
timeout.as_millis()
),
},
},
None => StepResult {
name: step_name,
status: StepStatus::Failed,
message: "wait step has neither selector nor text".into(),
},
}
}
#[allow(clippy::too_many_arguments)]
fn run_assert(
&self,
definition: Option<&str>,
preset: Option<&str>,
prompt: Option<&str>,
assert_text: Option<&str>,
screenshot: bool,
step_endpoint: Option<&str>,
test_endpoint: Option<&str>,
tab: &Tab,
) -> StepResult {
std::thread::sleep(Duration::from_millis(500));
let page_content = get_page_text(tab);
// Vision attach: capture the full page once per assert step and
// split it into viewport-tall tiles (the total coverage is bounded
// by the configured height cap so vision tokens stay sane). All
// tile data URLs are handed to the preset/prompt evaluation below.
let image: Option<Vec<String>> = if screenshot {
let endpoint = self
.endpoints
.resolve(step_endpoint.or(test_endpoint), TaskType::Assertion);
if !endpoint.vision {
return StepResult {
name: "[assert]".into(),
status: StepStatus::Failed,
message: format!(
"screenshot requested but endpoint '{name}' does not declare vision = true (add vision = true to its [config.endpoints] entry)",
name = endpoint.name
),
};
}
match crate::vision::capture_screenshot_data_urls(
tab,
self.config
.screenshot_max_dimension
.unwrap_or(crate::vision::DEFAULT_MAX_DIMENSION),
self.screenshot_height_cap(),
self.current_viewport_height(),
) {
Ok(urls) => Some(urls),
Err(e) => {
return StepResult {
name: "[assert]".into(),
status: StepStatus::Failed,
message: format!("screenshot capture failed: {e}"),
};
}
}
} else {
None
};
if let Some(def_name) = definition {
if let Some(def) = self.definitions.get(def_name) {
return self.run_assert_def(
def,
&page_content,
image.as_deref(),
step_endpoint,
test_endpoint,
tab,
);
}
return StepResult {
name: format!("[assert] {def_name}"),
status: StepStatus::Failed,
message: format!("definition '{def_name}' not found"),
};
}
if let Some(preset_name) = preset {
// Deterministic DOM layout scan — runs JS in the browser and
// never calls the LLM (free, fast, no pixel budget).
if preset_name == "layout_no_issues" {
return self.run_layout_preset(tab);
}
return self.run_preset(
preset_name,
assert_text,
&page_content,
image.as_deref(),
step_endpoint,
test_endpoint,
);
}
if let Some(prompt_text) = prompt {
return self.run_custom(
prompt_text,
&page_content,
image.as_deref(),
step_endpoint,
test_endpoint,
);
}
StepResult {
name: "[assert]".into(),
status: StepStatus::Skipped,
message: "no definition, preset, or prompt specified".into(),
}
}
fn run_assert_def(
&self,
def: &AssertDefinition,
page_content: &PageContent,
image: Option<&[String]>,
step_endpoint: Option<&str>,
test_endpoint: Option<&str>,
tab: &Tab,
) -> StepResult {
// Agent-based definition: delegate to an A2A agent
if let Some(ref agent) = def.agent {
if image.is_some() {
return StepResult {
name: format!("[assert] {}", def.name),
status: StepStatus::Failed,
message: "agent-backed assertions do not support screenshots".into(),
};
}
let task = def
.task_template
.as_deref()
.unwrap_or("Evaluate the assertion")
.replace("{url}", &page_content.url)
.replace("{title}", &page_content.title)
.replace("{content}", &page_content.body_text)
.replace("{expected_text}", def.assert_text.as_deref().unwrap_or(""));
return self.run_agent_step(agent, &task, &def.name);
}
// Custom preset: system + user_template provided in the definition
if let (Some(system), Some(template)) = (&def.system, &def.user_template) {
return self.run_custom_preset(
&def.name,
system,
template,
def.assert_text.as_deref(),
page_content,
image,
step_endpoint,
test_endpoint,
);
}
def.preset.as_ref().map_or_else(
|| {
def.prompt.as_ref().map_or_else(
|| StepResult {
name: format!("[assert] {}", def.name),
status: StepStatus::Failed,
message: "definition has no preset, prompt, or system+user_template".into(),
},
|prompt| {
self.run_custom(prompt, page_content, image, step_endpoint, test_endpoint)
},
)
},
|preset_name| {
if preset_name == "layout_no_issues" {
return self.run_layout_preset(tab);
}
self.run_preset(
preset_name,
def.assert_text.as_deref(),
page_content,
image,
step_endpoint,
test_endpoint,
)
},
)
}
#[allow(clippy::too_many_arguments)]
fn run_custom_preset(
&self,
name: &str,
system: &str,
template: &str,
assert_text: Option<&str>,
page_content: &PageContent,
image: Option<&[String]>,
step_endpoint: Option<&str>,
test_endpoint: Option<&str>,
) -> StepResult {
let user_prompt = template
.replace("{url}", &page_content.url)
.replace("{title}", &page_content.title)
.replace("{content}", &page_content.body_text)
.replace("{expected_text}", assert_text.unwrap_or(""))
.replace("{description}", "");
// Custom preset definitions frequently forget the {content}
// placeholder — without it the LLM has no page to evaluate and
// answers "I can't determine that without seeing the page". Always
// append the page context unless the template already references it.
let user_prompt = if template.contains("{content}") {
user_prompt
} else {
format!(
"{user_prompt}\n\nPage URL: {url}\nPage Title: {title}\n\nPage Content:\n{content}",
url = page_content.url,
title = page_content.title,
content = page_content.body_text,
)
};
self.reporter
.debug(format!("assert: {name} (custom preset)"));
let chain = self
.endpoints
.resolve_chain(step_endpoint.or(test_endpoint), TaskType::Assertion);
let sys = system.to_owned();
let response = self.llm_call_chain(&chain, &sys, &user_prompt, image, "assertion");
response.map_or_else(
|e| StepResult {
name: format!("[assert] {name}"),
status: StepStatus::Failed,
message: format!("LLM assertion call failed: {e}"),
},
|(lr, idx)| {
self.usage.record_llm_call(
&chain[idx].name,
chain[idx],
lr.usage.prompt_tokens,
lr.usage.completion_tokens,
);
let content_lower = lr.content.to_lowercase().trim().to_owned();
if content_lower.starts_with("pass") {
StepResult {
name: format!("[assert] {name}"),
status: StepStatus::Passed,
message: "PASS".into(),
}
} else {
StepResult {
name: format!("[assert] {name}"),
status: StepStatus::Failed,
message: lr.content,
}
}
},
)
}
fn run_preset(
&self,
preset_name: &str,
assert_text: Option<&str>,
page_content: &PageContent,
image: Option<&[String]>,
step_endpoint: Option<&str>,
test_endpoint: Option<&str>,
) -> StepResult {
let Some(preset) = ASSERTION_PRESETS.iter().find(|p| p.name == preset_name) else {
return StepResult {
name: format!("[assert] {preset_name}"),
status: StepStatus::Failed,
message: format!("unknown assertion preset: {preset_name}"),
};
};
if preset_name.starts_with("visual_") && image.is_none() {
return StepResult {
name: format!("[assert] {preset_name}"),
status: StepStatus::Failed,
message: format!(
"preset '{preset_name}' evaluates the page screenshot — set screenshot = true on the assert step and point it at a vision endpoint (vision = true)"
),
};
}
let user_prompt = preset
.user_template
.replace("{url}", &page_content.url)
.replace("{title}", &page_content.title)
.replace("{content}", &page_content.body_text)
.replace("{expected_text}", assert_text.unwrap_or(""))
.replace("{description}", "");
// Same safety net as custom presets: never let the LLM answer with
// no page context at all.
let user_prompt = if preset.user_template.contains("{content}") {
user_prompt
} else {
format!(
"{user_prompt}\n\nPage URL: {url}\nPage Title: {title}\n\nPage Content:\n{content}",
url = page_content.url,
title = page_content.title,
content = page_content.body_text,
)
};
self.reporter.debug(format!("assert: {preset_name}"));
let chain = self
.endpoints
.resolve_chain(step_endpoint.or(test_endpoint), TaskType::Assertion);
let sys = preset.system.to_owned();
let response = self.llm_call_chain(&chain, &sys, &user_prompt, image, "assertion");
response.map_or_else(
|e| StepResult {
name: format!("[assert] {preset_name}"),
status: StepStatus::Failed,
message: format!("LLM assertion call failed: {e}"),
},
|(lr, idx)| {
self.usage.record_llm_call(
&chain[idx].name,
chain[idx],
lr.usage.prompt_tokens,
lr.usage.completion_tokens,
);
let content_lower = lr.content.to_lowercase().trim().to_owned();
if content_lower.starts_with("pass") {
StepResult {
name: format!("[assert] {preset_name}"),
status: StepStatus::Passed,
message: "PASS".into(),
}
} else {
StepResult {
name: format!("[assert] {preset_name}"),
status: StepStatus::Failed,
message: lr.content,
}
}
},
)
}
/// Runs the deterministic DOM layout scan (`layout_no_issues`).
///
/// Evaluates the layout-scan JS in the page and fails with the list of
/// detected issues: horizontal page overflow, visible elements sticking
/// out of the viewport, text clipped by `overflow: hidden` containers,
/// and interactive elements covered by other elements. No LLM call —
/// checks are geometry-based so the check is free, deterministic, and
/// safe to run on every page × viewport variant.
fn run_layout_preset(&self, tab: &Tab) -> StepResult {
let name = "[assert] layout_no_issues".to_owned();
self.reporter
.debug("assert: layout_no_issues (DOM layout scan)");
let js = LAYOUT_SCAN_JS.replace(
"__IGNORE_CLASSES__",
&serde_json::to_string(&self.config.layout_ignore_classes)
.unwrap_or_else(|_| "[]".to_owned()),
);
let result = tab.evaluate(&js, false);
let json_str = match result {
Ok(r) => r
.value
.as_ref()
.and_then(|v| v.as_str().map(String::from))
.unwrap_or_else(|| "[]".to_owned()),
Err(e) => {
return StepResult {
name,
status: StepStatus::Failed,
message: format!("layout scan JS failed: {e}"),
};
}
};
let issues: Vec<LayoutIssue> = serde_json::from_str(&json_str).unwrap_or_default();
if issues.is_empty() {
return StepResult {
name,
status: StepStatus::Passed,
message: "PASS — no layout defects detected".into(),
};
}
let mut lines: Vec<String> = issues
.iter()
.take(10)
.map(|i| {
format!(
"- [{type_}] {element}: {detail}",
type_ = i.issue_type,
element = i.element,
detail = i.detail
)
})
.collect();
if issues.len() > 10 {
lines.push(format!("- … and {} more", issues.len() - 10));
}
StepResult {
name,
status: StepStatus::Failed,
message: format!(
"FAIL — {} layout defect(s) detected:\n{}",
issues.len(),
lines.join("\n")
),
}
}
fn run_custom(
&self,
prompt: &str,
page_content: &PageContent,
image: Option<&[String]>,
step_endpoint: Option<&str>,
test_endpoint: Option<&str>,
) -> StepResult {
let system = "You are a QA tester evaluating a web page. Respond with exactly \"PASS\" if the assertion holds, or \"FAIL: <reason>\" if it does not.";
let mut user = format!(
"Page URL: {url}\nPage Title: {title}\n\nPage Content:\n{content}\n\nAssertion: {prompt}",
url = page_content.url,
title = page_content.title,
content = page_content.body_text,
);
if image.is_some() {
user.push_str(
"\n\nA screenshot of the page is attached — inspect it for visual evidence when answering.",
);
}
self.reporter.debug("custom assert");
let chain = self
.endpoints
.resolve_chain(step_endpoint.or(test_endpoint), TaskType::Assertion);
let sys = system.to_owned();
let response = self.llm_call_chain(&chain, &sys, &user, image, "assertion");
response.map_or_else(
|e| StepResult {
name: "[assert] custom".into(),
status: StepStatus::Failed,
message: format!("LLM assertion call failed: {e}"),
},
|(lr, idx)| {
self.usage.record_llm_call(
&chain[idx].name,
chain[idx],
lr.usage.prompt_tokens,
lr.usage.completion_tokens,
);
let content_lower = lr.content.to_lowercase().trim().to_owned();
if content_lower.starts_with("pass") {
StepResult {
name: "[assert] custom".into(),
status: StepStatus::Passed,
message: "PASS".into(),
}
} else {
StepResult {
name: "[assert] custom".into(),
status: StepStatus::Failed,
message: lr.content,
}
}
},
)
}
fn run_screenshot(path: Option<&str>, tab: &Tab) -> StepResult {
let path = path.unwrap_or("screenshot.png");
match tab.capture_screenshot(
headless_chrome::protocol::cdp::Page::CaptureScreenshotFormatOption::Png,
None,
None,
true,
) {
Ok(data) => {
if let Err(e) = std::fs::write(path, &data) {
return StepResult {
name: format!("[screenshot] {path}"),
status: StepStatus::Failed,
message: format!("failed to write screenshot: {e}"),
};
}
StepResult {
name: format!("[screenshot] {path}"),
status: StepStatus::Passed,
message: format!("saved to {path}"),
}
}
Err(e) => StepResult {
name: format!("[screenshot] {path}"),
status: StepStatus::Failed,
message: format!("screenshot failed: {e}"),
},
}
}
/// Runs an A2A agent step.
#[allow(clippy::literal_string_with_formatting_args)]
fn run_agent(
&self,
agent_name: &str,
task: &str,
definition: Option<&str>,
_test_endpoint: Option<&str>,
) -> StepResult {
// If a definition is specified, look up the task template
let resolved_task = if let Some(def_name) = definition {
if let Some(def) = self.definitions.get(def_name) {
let tmpl = def.task_template.as_deref().unwrap_or(task);
tmpl.replace("{task}", task)
} else {
return StepResult {
name: format!("[agent] {def_name}"),
status: StepStatus::Failed,
message: format!("definition '{def_name}' not found"),
};
}
} else {
task.to_owned()
};
self.run_agent_step(agent_name, &resolved_task, &format!("agent:{agent_name}"))
}
fn run_agent_step(&self, agent_name: &str, task: &str, display_name: &str) -> StepResult {
let Some(ep) = self.endpoints.get(agent_name) else {
return StepResult {
name: format!("[agent] {display_name}"),
status: StepStatus::Failed,
message: format!("agent endpoint '{agent_name}' not found"),
};
};
if ep.url.is_empty() {
return StepResult {
name: format!("[agent] {display_name}"),
status: StepStatus::Failed,
message: format!("agent endpoint '{agent_name}' has no URL"),
};
}
self.reporter.debug(format!("agent {agent_name}: {task}"));
let url = ep.url.clone();
let client = A2aClient::new(&url, self.timeout);
let task_clone = task.to_owned();
let response = std::thread::spawn(move || {
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
rt.block_on(client.send_task(&task_clone))
})
.join()
.unwrap();
// Record the flat-cost call
self.usage.record_flat_call(agent_name, ep);
match response {
Ok(text) => {
let clean = text.trim().to_owned();
let lower = clean.to_lowercase();
if lower.starts_with("pass") {
StepResult {
name: format!("[agent] {display_name}"),
status: StepStatus::Passed,
message: format!("PASS: {clean}"),
}
} else if lower.starts_with("fail") {
StepResult {
name: format!("[agent] {display_name}"),
status: StepStatus::Failed,
message: clean,
}
} else {
StepResult {
name: format!("[agent] {display_name}"),
status: StepStatus::Passed,
message: format!("response: {clean}"),
}
}
}
Err(e) => StepResult {
name: format!("[agent] {display_name}"),
status: StepStatus::Failed,
message: format!("agent call failed: {e}"),
},
}
}
/// Runs an MCP tool call step.
fn run_mcp(
&self,
server_name: &str,
tool_name: &str,
args: Option<&serde_json::Value>,
) -> StepResult {
let Some(ep) = self.endpoints.get(server_name) else {
return StepResult {
name: format!("[mcp] {server_name}:{tool_name}"),
status: StepStatus::Failed,
message: format!("MCP server endpoint '{server_name}' not found"),
};
};
let cmd = ep.command.as_deref().unwrap_or("");
if cmd.is_empty() {
return StepResult {
name: format!("[mcp] {server_name}:{tool_name}"),
status: StepStatus::Failed,
message: format!("MCP server '{server_name}' has no command configured"),
};
}
self.reporter
.debug(format!("mcp {server_name} {tool_name}"));
let args_val = args.cloned().unwrap_or(serde_json::Value::Null);
let command = cmd.to_owned();
let args_vec = ep.args.clone();
let tool = tool_name.to_owned();
let response = std::thread::spawn(move || {
let mut mcp_client =
McpClient::connect_stdio(&command, &args_vec).map_err(|e| e.to_string())?;
mcp_client
.call_tool(&tool, &args_val)
.map_err(|e| e.to_string())
})
.join()
.unwrap();
// Record the flat-cost call
self.usage.record_flat_call(server_name, ep);
match response {
Ok(result) => {
if result.isError {
StepResult {
name: format!("[mcp] {server_name}:{tool_name}"),
status: StepStatus::Failed,
message: result.to_string(),
}
} else {
StepResult {
name: format!("[mcp] {server_name}:{tool_name}"),
status: StepStatus::Passed,
message: result.to_string(),
}
}
}
Err(e) => StepResult {
name: format!("[mcp] {server_name}:{tool_name}"),
status: StepStatus::Failed,
message: format!("MCP call failed: {e}"),
},
}
}
// ── helpers ──────────────────────────────────────────────────────────
/// Builds an `LlmConfig` from a resolved endpoint, falling back to
/// the runner's default LLM config for any unset fields.
fn build_llm_for_endpoint(&self, endpoint: &crate::endpoints::ResolvedEndpoint) -> LlmConfig {
LlmConfig {
url: if endpoint.provider == crate::scenario::Provider::Bedrock {
// Bedrock builds its endpoint from the resolved AWS region
// when no URL is given — never inherit the default LLM URL.
endpoint.url.clone()
} else if endpoint.url.is_empty() {
self.llm.url.clone()
} else {
endpoint.url.clone()
},
model: endpoint
.model
.clone()
.unwrap_or_else(|| self.llm.model.clone()),
api_key: endpoint
.api_key
.clone()
.or_else(|| self.llm.api_key.clone()),
headers: if endpoint.headers.is_empty() {
self.llm.headers.clone()
} else {
endpoint.headers.clone()
},
timeout: self.llm.timeout,
temperature: self.llm.temperature,
thinking: self.llm.thinking,
model_params: self.llm.model_params.clone(),
max_attempts: endpoint.max_attempts.max(1),
provider: endpoint.provider,
deployment: endpoint.deployment.clone(),
api_version: endpoint.api_version.clone(),
auth: endpoint.auth.clone(),
header_commands: endpoint.header_commands.clone(),
aws: endpoint.aws.clone(),
}
}
/// Runs a single LLM call against an ordered endpoint chain (primary +
/// fallbacks). Every endpoint gets its own `max_attempts` retry budget;
/// the first endpoint that answers wins. Returns the response together
/// with the chain index of the answering endpoint (0 = primary) so the
/// caller can attribute usage to the correct endpoint.
///
/// Emits an `LlmCallStarted`/`LlmCallFinished` event pair so the report
/// shows duration, tokens, cost and the answering endpoint per call.
#[allow(clippy::cast_possible_truncation)]
fn llm_call_chain(
&self,
chain: &[&ResolvedEndpoint],
system: &str,
user: &str,
image: Option<&[String]>,
purpose: &str,
) -> Result<(crate::costs::LlmResponse, usize), String> {
if chain.is_empty() {
return Err("empty LLM endpoint chain".into());
}
let primary = self.build_llm_for_endpoint(chain[0]);
let fallbacks: Vec<LlmConfig> = chain[1..]
.iter()
.map(|e| self.build_llm_for_endpoint(e))
.collect();
let (test, index) = self
.current_step
.borrow()
.as_ref()
.map_or_else(|| ("-".to_owned(), 0), |(t, i)| (t.clone(), *i));
let primary_endpoint = chain[0].name.clone();
let primary_model = primary.model.clone();
self.emit_event(&TestEvent::LlmCallStarted {
test: test.clone(),
index,
endpoint: primary_endpoint.clone(),
model: primary_model.clone(),
purpose: purpose.to_owned(),
});
let started = Instant::now();
let sys = system.to_owned();
let user = user.to_owned();
let image = image.map(<[String]>::to_vec);
let result = std::thread::spawn(move || {
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
let call = async {
match image.as_deref() {
Some(img) => {
llm_chat_vision_with_usage_chain(
&primary,
&fallbacks,
&sys,
&user,
Some(img),
)
.await
}
None => llm_chat_with_usage_chain(&primary, &fallbacks, &sys, &user).await,
}
};
rt.block_on(call)
})
.join()
.unwrap();
let duration_ms = started.elapsed().as_millis() as u64;
match result {
Ok((lr, idx)) => {
let cost = calculate_llm_cost(
chain[idx],
lr.usage.prompt_tokens,
lr.usage.completion_tokens,
);
let answering = chain[idx].name.clone();
let model = chain[idx]
.model
.clone()
.unwrap_or_else(|| primary_model.clone());
self.emit_event(&TestEvent::LlmCallFinished {
test,
index,
endpoint: answering,
model,
purpose: purpose.to_owned(),
ok: true,
duration_ms,
input_tokens: lr.usage.prompt_tokens,
output_tokens: lr.usage.completion_tokens,
cost,
error: None,
});
Ok((lr, idx))
}
Err(e) => {
self.emit_event(&TestEvent::LlmCallFinished {
test,
index,
endpoint: primary_endpoint,
model: primary_model,
purpose: purpose.to_owned(),
ok: false,
duration_ms,
input_tokens: 0,
output_tokens: 0,
cost: 0.0,
error: Some(e.clone()),
});
Err(e)
}
}
}
/// Resolves a CSS selector for the target element. Uses the explicit
/// `selector` if provided, otherwise asks the LLM to find the element
/// from the natural language `target` description and page DOM.
///
/// LLM responses are sanitized and verified against the live page: a
/// response that is not a selector (empty, `:not(*)`, `null`, …) fails
/// immediately with the raw LLM output, and a selector that matches
/// nothing triggers one retry with feedback before failing.
#[allow(clippy::too_many_lines)]
fn resolve_selector(
&self,
css_override: Option<&str>,
target: &str,
step_endpoint: Option<&str>,
test_endpoint: Option<&str>,
tab: &Tab,
) -> Result<String, String> {
if let Some(explicit) = css_override {
return Ok(explicit.to_owned());
}
let dom_info = extract_dom_info(tab)?;
let page_content = get_page_text(tab);
let system = concat!(
"You are a browser automation selector generator. ",
"Given a web page's content and interactive elements, ",
"return ONLY the best CSS selector for the described element. ",
"Output nothing except the CSS selector. ",
"Prefer selectors in this order: #id, [data-testid=\"...\"], ",
"[name=\"...\"], tag.class, tag. ",
"Never output explanations, markdown, or extra text."
);
let user = format!(
"Page URL: {}\nPage Title: {}\n\nPage body text (first 4000 chars):\n{}\n\nInteractive elements:\n{}\n\nFind the CSS selector for: {}",
page_content.url,
page_content.title,
truncate(&page_content.body_text, 4000),
dom_info,
target,
);
let retry_user = format!(
"Your previous answer was not usable. {}\n\nPage URL: {}\nPage Title: {}\n\nPage body text (first 4000 chars):\n{}\n\nInteractive elements:\n{}\n\nFind the CSS selector for: {}\nReturn ONLY a single CSS selector that matches an existing element. No explanations.",
"The selector must match at least one element currently present on the page.",
page_content.url,
page_content.title,
truncate(&page_content.body_text, 4000),
dom_info,
target,
);
self.reporter.debug(format!("LLM targeting: {target}"));
let chain = self
.endpoints
.resolve_chain(step_endpoint.or(test_endpoint), TaskType::Targeting);
let sys = system.to_owned();
let call_llm = |prompt: &str| self.llm_call_chain(&chain, &sys, prompt, None, "targeting");
let first = call_llm(&user);
let (lr, idx) = match first {
Ok(lr) => lr,
Err(e) => {
return Err(format!("LLM element targeting failed: {e}"));
}
};
self.usage.record_llm_call(
&chain[idx].name,
chain[idx],
lr.usage.prompt_tokens,
lr.usage.completion_tokens,
);
let clean = sanitize_selector(&lr.content);
self.reporter.debug(format!("resolved selector: {clean}"));
if selector_is_useless(&clean) {
return Err(format!(
"LLM element targeting failed: the LLM did not return a usable selector for {target:?} (got {raw:?}). Check the page state in the diagnostics above.",
raw = lr.content.trim(),
));
}
if let Err(reason) = validate_selector(&clean) {
return Err(format!(
"LLM element targeting failed: invalid selector for {target:?}: {reason} (LLM response: {raw:?})",
raw = lr.content.trim(),
));
}
if !selector_matches(tab, &clean).unwrap_or(false) {
// One retry with feedback: flaky models occasionally invent a
// selector that does not exist on the page.
self.reporter.warn(format!(
"selector {clean} matches nothing — retrying LLM targeting with feedback"
));
let second = call_llm(&retry_user);
let (lr2, idx2) = match second {
Ok(lr2) => lr2,
Err(e) => {
return Err(format!(
"LLM element targeting failed: first answer {clean:?} matched nothing, retry also failed: {e}"
));
}
};
self.usage.record_llm_call(
&chain[idx2].name,
chain[idx2],
lr2.usage.prompt_tokens,
lr2.usage.completion_tokens,
);
let clean2 = sanitize_selector(&lr2.content);
self.reporter
.debug(format!("resolved selector (retry): {clean2}"));
if selector_is_useless(&clean2) {
return Err(format!(
"LLM element targeting failed: selector {clean:?} matched nothing; the retry returned no usable selector for {target:?} (got {raw:?}). Page excerpt: {excerpt}",
raw = lr2.content.trim(),
excerpt = truncate(&page_content.body_text, 300),
));
}
if !selector_matches(tab, &clean2).unwrap_or(false) {
return Err(format!(
"LLM element targeting failed: selector {clean2:?} does not match any element on the page for {target:?}. Verify the page state in the diagnostics; the login/SPA may not have rendered."
));
}
return Ok(clean2);
}
Ok(clean)
}
}
/// Evaluates a JS expression that is expected to return a boolean.
fn eval_bool(tab: &Tab, js: &str) -> Result<bool, String> {
tab.evaluate(js, false)
.map_err(|e| format!("evaluate failed: {e}"))?
.value
.and_then(|v| v.as_bool())
.ok_or_else(|| "evaluate returned non-boolean".to_owned())
}
/// Checks whether a CSS selector matches at least one current element.
fn selector_matches(tab: &Tab, selector: &str) -> Result<bool, String> {
eval_bool(tab, &crate::selectors::selector_matches_js(selector))
}
// ── Free helper functions ──────────────────────────────────────────────
fn run_navigate_step(full_url: &str, tab: &Tab) -> StepResult {
let name = format!("[navigate] {full_url}");
match tab.navigate_to(full_url) {
Ok(_) => {
let _ = tab.wait_until_navigated();
StepResult {
name,
status: StepStatus::Passed,
message: format!("navigated to {full_url}"),
}
}
Err(e) => StepResult {
name,
status: StepStatus::Failed,
message: format!("navigation failed: {e}"),
},
}
}
fn extract_dom_info(tab: &Tab) -> Result<String, String> {
let result = tab
.evaluate(DOM_EXTRACT_JS, false)
.map_err(|e| format!("DOM extraction failed: {e}"))?;
let json_str = result
.value
.as_ref()
.and_then(|v| v.as_str())
.unwrap_or("[]");
let elements: Vec<String> = serde_json::from_str(json_str).unwrap_or_default();
if elements.is_empty() {
return Ok("(no interactive elements found)".to_owned());
}
Ok(elements.join("\n"))
}
fn get_page_text(tab: &Tab) -> PageContent {
let url = tab.get_url();
let title = tab
.evaluate("document.title", false)
.ok()
.and_then(|r| r.value)
.and_then(|v| v.as_str().map(String::from))
.unwrap_or_else(|| "unknown".to_owned());
let body_text = tab
.evaluate(
"document.body ? document.body.innerText : document.documentElement.innerText",
false,
)
.ok()
.and_then(|r| r.value)
.and_then(|v| v.as_str().map(String::from))
.unwrap_or_default();
PageContent {
url,
title,
body_text: truncate(&body_text, 8000),
}
}
fn resolve_url(url: &str, base_url: &str) -> String {
if url.starts_with("http://") || url.starts_with("https://") {
return url.to_owned();
}
let base = base_url.trim_end_matches('/');
if url.starts_with('/') {
format!("{base}{url}")
} else {
format!("{base}/{url}")
}
}
/// Human-readable label for a step, used when steps are skipped after an
/// earlier failure.
fn step_label(step: &TestStep) -> String {
match step {
TestStep::Navigate { url, .. } => format!("[navigate] {url}"),
TestStep::Click { target, .. } => format!("[click] {target}"),
TestStep::Type { target, .. } => format!("[type] {target}"),
TestStep::Wait { target, .. } => format!("[wait] {target}"),
TestStep::Assert {
definition,
preset,
prompt,
..
} => definition.as_ref().map_or_else(
|| {
preset.as_ref().map_or_else(
|| {
prompt.as_ref().map_or_else(
|| "[assert]".to_owned(),
|pr| format!("[assert] custom ({})", truncate(pr, 60)),
)
},
|p| format!("[assert] {p}"),
)
},
|d| format!("[assert] {d}"),
),
TestStep::Screenshot { .. } => "[screenshot]".to_owned(),
TestStep::Agent { agent, .. } => format!("[agent] {agent}"),
TestStep::Mcp { server, tool, .. } => format!("[mcp] {server}:{tool}"),
}
}
/// Short kind word for artifact file names (e.g. `click`, `wait`, `assert`).
#[must_use]
const fn step_kind_label(step: &TestStep) -> &'static str {
match step {
TestStep::Navigate { .. } => "navigate",
TestStep::Click { .. } => "click",
TestStep::Type { .. } => "type",
TestStep::Wait { .. } => "wait",
TestStep::Assert { .. } => "assert",
TestStep::Screenshot { .. } => "screenshot",
TestStep::Agent { .. } => "agent",
TestStep::Mcp { .. } => "mcp",
}
}
// ── Support types ──────────────────────────────────────────────────────
#[derive(Default)]
struct TestRunResult {
passed: u32,
failed: u32,
skipped: u32,
total: u32,
details: Vec<StepResult>,
}
struct PageContent {
url: String,
title: String,
body_text: String,
}