llm_browser_testkit/

runner.rs

use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};

use anyhow::Context;
use headless_chrome::{Browser, LaunchOptions, Tab};

use crate::a2a::A2aClient;
use crate::budgets::{BudgetStatus, BudgetTracker};
use crate::costs::{calculate_llm_cost, UsageTracker};
use crate::diagnostics;
use crate::endpoints::{EndpointRegistry, ResolvedEndpoint, TaskType};
use crate::events::TestEvent;
use crate::llm_chat_vision_with_usage_chain;
use crate::llm_chat_with_usage_chain;
use crate::mcp_client::McpClient;
use crate::reporting::Reporter;
use crate::scenario::{AssertDefinition, ScenarioConfig, TestGroup, TestStep};
use crate::selectors::{sanitize_selector, selector_is_useless, validate_selector};
use crate::truncate;
use crate::LlmConfig;
use crate::DOM_EXTRACT_JS;

/// Formats a UNIX epoch timestamp as an RFC 3339 UTC string
/// (`2026-09-19T16:04:00Z`) without pulling in a date dependency.
/// Civil-from-days conversion after Howard Hinnant.
#[must_use]
#[allow(clippy::many_single_char_names)]
fn unix_to_rfc3339(secs: u64) -> String {
    let secs = i64::try_from(secs).unwrap_or(0);
    let days = secs.div_euclid(86_400);
    let rem = secs.rem_euclid(86_400);
    let (h, m, s) = (rem / 3_600, (rem % 3_600) / 60, rem % 60);
    let z = days + 719_468;
    let era = z.div_euclid(146_097);
    let doe = z.rem_euclid(146_097);
    let yoe = (doe - doe / 1_460 + doe / 36_524 - doe / 146_096) / 365;
    let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
    let mp = (5 * doy + 2) / 153;
    let d = doy - (153 * mp + 2) / 5 + 1;
    let mth = if mp < 10 { mp + 3 } else { mp - 9 };
    let y = yoe + era * 400 + i64::from(mth <= 2);
    format!("{y:04}-{mth:02}-{d:02}T{h:02}:{m:02}:{s:02}Z")
}

/// One detected layout defect (`layout_no_issues` preset).
#[derive(Debug, serde::Deserialize)]
struct LayoutIssue {
    #[serde(rename = "type")]
    issue_type: String,
    element: String,
    detail: String,
}

/// In-browser DOM layout scan for `layout_no_issues`.
///
/// Geometry-only checks (no LLM, no pixels):
/// 1. page horizontal overflow (`scrollWidth` > viewport width);
/// 2. elements outside the viewport that scrolling cannot reveal
///    (fixed elements off-screen, left/negative overflow, right-edge
///    overflow beyond the horizontally scrollable area, and bottom
///    overflow on a page that cannot scroll down) — below-the-fold
///    content on a tall scrollable page is normal flow, NOT a defect;
/// 3. text clipped by `overflow: hidden` containers whose content
///    is measurably larger than the box;
/// 4. interactive elements (buttons/links/inputs) whose center point
///    is covered by a different element that would intercept the click.
///
/// Elements whose class matches a configured ignore prefix (default:
/// the Angular CDK screen-reader helpers `cdk-visually-hidden`,
/// `cdk-describedby-message-container`, `cdk-overlay-container`) are
/// skipped — they are intentionally 1x1 / off-screen. The prefix list
/// is injected at `__IGNORE_CLASSES__` from
/// [`ScenarioConfig::layout_ignore_classes`].
///
/// Intentional stacking (off-canvas drawers, dropdown menus, badges,
/// sticky headers) is excluded by the position/relation filters.
const LAYOUT_SCAN_JS: &str = r#"
(() => {
  const issues = [];
  const push = (type, el, detail) => {
    if (issues.length >= 30) return;
    let element = el.tagName.toLowerCase();
    if (el.id) element += '#' + el.id;
    else if (typeof el.className === 'string' && el.className.trim())
      element += '.' + el.className.trim().split(/\s+/).join('.');
    issues.push({ type, element, detail: String(detail).slice(0, 220) });
  };
  const vw = document.documentElement.clientWidth || window.innerWidth;
  const vh = document.documentElement.clientHeight || window.innerHeight;
  if (!vw || !vh) return JSON.stringify(issues);
  const de = document.documentElement;
  // 1. Page-level horizontal overflow.
  if (maxSW > vw + 2)
    push('page-overflow-x', de,
      'page scrollWidth ' + maxSW + ' exceeds viewport width ' + vw);
  // Class-name prefixes to skip (injected; default = Angular CDK
  // screen-reader helpers, which are intentionally 1x1 / off-screen).
  const ignorePrefixes = __IGNORE_CLASSES__;
  const isIgnored = (el) => {
    if (typeof el.className !== 'string' || !el.className.trim()) return false;
    const classes = el.className.trim().split(/\s+/);
    return classes.some((c) => ignorePrefixes.some((p) => c.startsWith(p)));
  };
  const body = document.body;
  // The document element is not always the scroll container: the app may
  // scroll via <body> (html{overflow:hidden} + body scroll, e.g. sticky
  // navs) or via an inner overflow-y:auto panel. Never treat those pages as
  // "not scrollable" — measure the scrollable content height/width of the
  // document element AND the body, and take the max.
  const maxSH = Math.max(de.scrollHeight, body ? body.scrollHeight : 0);
  const maxSW = Math.max(de.scrollWidth, body ? body.scrollWidth : 0);
  const vScrollable = maxSH > vh + 2;
  const hScrollable = maxSW > vw + 2;
  // True when an ancestor panel (overflow:auto/scroll/overlay) can scroll
  // this element into view on the given axis — i.e. the element is inside a
  // scrolling region, so lying beyond the viewport cut is not a defect.
  const reachableByScroller = (el, axis) => {
    const ovProp = axis === 'y' ? 'overflowY' : 'overflowX';
    const dim = axis === 'y' ? 'scrollHeight' : 'scrollWidth';
    const dimClient = axis === 'y' ? 'clientHeight' : 'clientWidth';
    let p = el.parentElement;
    while (p && p !== body) {
      const pcs = getComputedStyle(p);
      const o = pcs[ovProp];
      if ((o === 'auto' || o === 'scroll' || o === 'overlay') &&
          p[dim] > p[dimClient] + 2) return true;
      p = p.parentElement;
    }
    return false;
  };
  const selfOverflowing = (el, cs, axis) => {
    const ov = axis === 'y' ? cs.overflowY : cs.overflowX;
    const dim = axis === 'y' ? 'scrollHeight' : 'scrollWidth';
    const dimClient = axis === 'y' ? 'clientHeight' : 'clientWidth';
    return (ov === 'auto' || ov === 'scroll' || ov === 'overlay') &&
      el[dim] > el[dimClient] + 2;
  };
  // True when an ancestor clips this axis with overflow:hidden/clip — the
  // element's overhang is not visible, so treat it as reachable.
  const clippedByAncestor = (el, axis) => {
    const ovProp = axis === 'y' ? 'overflowY' : 'overflowX';
    let p = el.parentElement;
    while (p && p !== body) {
      const o = getComputedStyle(p)[ovProp];
      if (o === 'hidden' || o === 'clip') return true;
      p = p.parentElement;
    }
    return false;
  };
  const all = Array.prototype.slice.call(document.querySelectorAll('body *'));
  const visible = (cs) => cs.display !== 'none' && cs.visibility !== 'hidden' && parseFloat(cs.opacity || '1') !== 0;
  const hasContent = (el) =>
    ((el.textContent || '').trim().length > 0) ||
    !!el.querySelector('img,svg,video,canvas,iframe,button,input,textarea,select');
  // 2. Elements outside the viewport that scrolling cannot reveal.
  for (const el of all) {
    const cs = getComputedStyle(el);
    if (!visible(cs)) continue;
    if (isIgnored(el)) continue;
    const r = el.getBoundingClientRect();
    if (r.width < 2 || r.height < 2) continue;
    if (!hasContent(el) && el.children.length === 0) continue;
    if (cs.position === 'fixed') {
      // Fixed elements never move with the scroll: any edge outside the
      // viewport is unreachable content and therefore a defect.
      const overTop = -r.top;
      const overLeft = -r.left;
      const overRight = r.right - vw;
      const overBottom = r.bottom - vh;
      // Top/left overshoot is never reachable; bottom/right overshoot is
      // only a defect when the fixed element cannot scroll that content
      // into view itself (e.g. an opened Material drawer whose inner
      // container scrolls is not a "cut-off" defect).
      if (overTop > 2 || overLeft > 2 ||
          (overRight > 2 && !selfOverflowing(el, cs, 'x')) ||
          (overBottom > 2 && !selfOverflowing(el, cs, 'y'))) {
        let where = '';
        if (overTop > 2 && overLeft > 2) where = 'top+left edges';
        else if (overTop > 2) where = 'top edge (' + Math.round(r.top) + ' < 0)';
        else if (overLeft > 2) where = 'left edge (' + Math.round(r.left) + ' < 0)';
        else if (overRight > 2 && overBottom > 2) where = 'right+bottom edges';
        else if (overRight > 2) where = 'right edge (' + Math.round(r.right) + ' > ' + vw + ')';
        else where = 'bottom edge (' + Math.round(r.bottom) + ' > ' + vh + ')';
        push('element-out-of-viewport', el,
          'fixed element lies ' + Math.max(overTop, overLeft, overRight, overBottom).toFixed(0) + 'px past the ' + where);
      }
      continue;
    }
    if (cs.position === 'sticky') continue;
    // Negative left overflow cannot be reached by scrolling (scrollLeft
    // never goes below 0).
    if (r.left < -2) {
      push('element-out-of-viewport', el,
        'extends ' + Math.round(-r.left).toFixed(0) + 'px past the left edge (' + Math.round(r.left) + ' < 0)');
      continue;
    }
    // Negative top with the page at the top means the element sits above
    // the document origin — also unreachable.
    if (r.top < -2 && de.scrollTop <= 2) {
      push('element-out-of-viewport', el,
        'extends ' + Math.round(-r.top).toFixed(0) + 'px past the top edge (' + Math.round(r.top) + ' < 0)');
      continue;
    }
    const overRight = r.right - vw;
    // Right-edge overflow is only a defect when the page cannot scroll
    // horizontally to reveal it (or the element sticks out past the
    // scrollable content width itself).
    if (overRight > 2 && (!hScrollable || r.right > maxSW + 2) && !reachableByScroller(el, 'x') && !clippedByAncestor(el, 'x')) {
      push('element-out-of-viewport', el,
        'extends ' + overRight.toFixed(0) + 'px past the right edge (' + Math.round(r.right) + ' > ' + vw + ')');
      continue;
    }
    // Below-the-fold content on a scrollable page is normal (tall landing
    // pages); only flag bottom overflow the user can never scroll to.
    const overBottom = r.bottom - vh;
    if (overBottom > 2 && (!vScrollable || r.bottom > maxSH + 2) && !reachableByScroller(el, 'y') && !clippedByAncestor(el, 'y')) {
      push('element-out-of-viewport', el,
        'extends ' + overBottom.toFixed(0) + 'px past the bottom edge (' + Math.round(r.bottom) + ' > ' + vh + ')');
    }
  }
  // 3. Text clipped by overflow:hidden containers.
  for (const el of all) {
    if (isIgnored(el)) continue;
    const cs = getComputedStyle(el);
    if (cs.overflowX !== 'hidden' && cs.overflowY !== 'hidden') continue;
    if (el.scrollWidth <= el.clientWidth + 2 && el.scrollHeight <= el.clientHeight + 2) continue;
    if (!(el.textContent || '').trim()) continue;
    // Single-line truncation with an ellipsis is an intentional design
    // pattern (Tailwind .truncate etc.), not a clipping defect.
    if (cs.textOverflow === 'ellipsis') continue;
    push('text-clipped', el,
      'content ' + el.scrollWidth + 'x' + el.scrollHeight +
      ' clipped to ' + el.clientWidth + 'x' + el.clientHeight);
  }
  // 4. Interactive elements covered by a different element.
  const interactive =
    'button, a[href], input, textarea, select, [role="button"], [role="link"], [role="menuitem"], label, .mdc-button, .mat-mdc-button, .mat-mdc-icon-button, .mdc-fab';
  const targets = document.querySelectorAll(interactive);
  for (const el of targets) {
    if (isIgnored(el)) continue;
    const r = el.getBoundingClientRect();
    if (r.width < 6 || r.height < 6) continue;
    const cs = getComputedStyle(el);
    if (!visible(cs)) continue;
    const cx = r.left + r.width / 2;
    const cy = r.top + r.height / 2;
    if (cx < 0 || cy < 0 || cx > vw || cy > vh) continue;
    const top = document.elementFromPoint(cx, cy);
    if (!top || top === el || el.contains(top) || top.contains(el)) continue;
    if (isIgnored(top)) continue;
    const tcs = getComputedStyle(top);
    if (!visible(tcs)) continue;
    if (tcs.pointerEvents === 'none') continue;
    const tr = top.getBoundingClientRect();
    if (tr.width * tr.height < r.width * r.height * 0.25) continue;
    const tname = top.tagName.toLowerCase() + (top.id ? '#' + top.id : '') +
      (typeof top.className === 'string' && top.className.trim() ? '.' + top.className.trim().split(/\s+/).join('.') : '');
    push('element-overlap', el,
      'center point at ' + Math.round(cx) + ',' + Math.round(cy) +
      ' is covered by <' + tname + '>');
  }
  return JSON.stringify(issues);
})()
"#;

/// How long the CDP connection stays open after the browser goes quiet.
///
/// `headless_chrome` ships a 30s default and tears down the entire connection
/// when no traffic arrives for that long; a run must own its connection for
/// its full duration instead.
const BROWSER_IDLE_TIMEOUT: Duration = Duration::from_secs(6 * 60 * 60);

/// Executes a [`Scenario`] against a real browser with optional LLM
/// assistance for element targeting and assertions.
pub struct ScenarioRunner {
    config: ScenarioConfig,
    definitions: HashMap<String, AssertDefinition>,
    llm: LlmConfig,
    timeout: Duration,
    viewport_width: u32,
    viewport_height: u32,
    /// The viewport currently applied in the browser (CDP emulation).
    /// Per-test overrides switch it mid-run; `(0, 0)` = not yet applied.
    applied_viewport: std::cell::Cell<(u32, u32)>,
    endpoints: EndpointRegistry,
    usage: Arc<UsageTracker>,
    budgets: BudgetTracker,
    /// Directory for failure artifacts (screenshots).
    artifacts_dir: PathBuf,
    /// Event sink for test reporting (console, `JSONL`, `JUnit`, trace).
    reporter: Arc<Reporter>,
    /// The test + step index currently executing, for LLM-call events.
    current_step: std::cell::RefCell<Option<(String, u32)>>,
    /// Epoch seconds of runner creation — constant for the whole run so
    /// the LLM context preamble below is cacheable by upstream providers.
    run_started: u64,
    /// Whether this runner emits the `RunStarted`/`RunFinished` events.
    /// When several scenario files are run concurrently by
    /// [`crate::parallel::run_scenarios`], the orchestrator owns those two
    /// events (one per batch), so each per-file runner suppresses its own.
    emit_run_events: bool,
}

/// Aggregated results from a scenario run.
#[derive(Debug, Default)]
pub struct RunReport {
    /// Number of tests that passed.
    pub tests_passed: u32,
    /// Number of tests that failed.
    pub tests_failed: u32,
    /// Number of steps that passed.
    pub passed: u32,
    /// Number of steps that failed.
    pub failed: u32,
    /// Number of steps that were skipped.
    pub skipped: u32,
    /// Per-step details.
    pub details: Vec<StepResult>,
}

/// Result of a single step execution.
#[derive(Debug, Clone)]
pub struct StepResult {
    /// The step name.
    pub name: String,
    /// Whether the step passed, failed, or was skipped.
    pub status: StepStatus,
    /// Human-readable result message.
    pub message: String,
}

/// Outcome for a single step.
pub use crate::events::StepStatus;

/// Predefined assertion preset definition.
struct AssertPreset {
    name: &'static str,
    system: &'static str,
    user_template: &'static str,
}

/// Built-in assertion presets.
#[allow(clippy::literal_string_with_formatting_args)]
const ASSERTION_PRESETS: &[AssertPreset] = &[
    AssertPreset {
        name: "no_error_on_page",
        system: "You are a QA tester. Evaluate if a web page contains error messages, stack traces, exception text, HTTP error codes, 'undefined' errors, or any indication of a malfunction. Be strict — even minor rendering glitches count as errors.",
        user_template: "Check if the following page content contains ANY errors or malfunctions:\n\nURL: {url}\nTitle: {title}\n\nPage Content:\n{content}\n\nRespond with exactly \"PASS\" if there are NO errors, or \"FAIL: <reason>\" if there are errors. Only respond with PASS or FAIL.",
    },
    AssertPreset {
        name: "text_visible",
        system: "You are a QA tester. Your task is to check if specific text is visible in the page content.",
        user_template: "Check if the following text appears in the page content:\n\nTEXT TO FIND: \"{expected_text}\"\n\nURL: {url}\n\nPage Content:\n{content}\n\nRespond with exactly \"PASS\" if the text is present (even partial match is OK), or \"FAIL: text not found\" if it is not.",
    },
    AssertPreset {
        name: "element_exists",
        system: "You are a QA tester. Check if a described UI element exists on a web page.",
        user_template: "Check if the following element exists on the page:\n\nELEMENT: \"{description}\"\n\nURL: {url}\n\nPage Content:\n{content}\n\nRespond with exactly \"PASS\" if the element exists, or \"FAIL: <reason>\" if it does not.",
    },
    AssertPreset {
        name: "visual_no_issues",
        system: "You are a visual QA engineer inspecting a website screenshot. Detect clearly visible layout and rendering defects: overlapping elements that hide content or controls, clipped or truncated text, content cut off at the viewport edges, misaligned or broken UI, blank/empty panels where content is expected, broken or missing images, duplicated elements, or rendering glitches. Ignore subjective aesthetics, intentional stacking (dropdowns, tooltips, layered design), and content that is simply not loaded (empty states). Only report defects that a user would actually see or be blocked by.",
        user_template: "Inspect the attached screenshot(s) (ordered from the top of the page down) and the page text below.\n\nURL: {url}\nTitle: {title}\n\nPage Content:\n{content}\n\nAre there any clearly visible layout or rendering defects (overlaps, clipping, cut-off content, broken images, blank panels)? Respond with exactly \"PASS\" if the page renders cleanly, or \"FAIL: <describe each defect and where it appears>\" otherwise.",
    },
    AssertPreset {
        name: "visual_no_overlaps",
        system: "You are a visual QA engineer inspecting a website screenshot for OVERLAPPING elements that harm usability: one element covering another element's text, buttons, links, or input fields (cookie banners, modals, popovers, chat widgets, sticky headers, or mispositioned layers that hide content or intercept clicks). Ignore intentional, non-harmful stacking (dropdowns, tooltips, badges over avatars, layered design where nothing is hidden or unclickable). Only fail on overlaps that visibly hide content or would block a click.",
        user_template: "Inspect the attached screenshot(s) (ordered from the top of the page down) and the page text below.\n\nURL: {url}\nTitle: {title}\n\nPage Content:\n{content}\n\nAre any elements overlapping in a way that hides page content, text, or interactive controls, or that would block clicks? Respond with exactly \"PASS\" if there are no such overlaps, or \"FAIL: <describe the overlapping elements and what they hide>\" otherwise.",
    },
    AssertPreset {
        name: "visual_fills_viewport",
        system: "You are a visual QA engineer inspecting a website screenshot for UNDER-FILLED layouts: the rendered content stops at a fraction of the viewport and the remaining area is a large blank region the layout should have filled. Typical defects: page content ending around the halfway mark of the viewport with an empty, background-only area below it; an app shell, dashboard, or main panel collapsing to part of its expected height and leaving dead space; content that fills only the left portion of a wide viewport with unused space to the right. Do NOT fail intentionally short or centered pages: a login or signup card, a landing hero, a short article, a confirmation step, or an empty state that does not fill the viewport is normal design. Only fail when the blank region is clearly a layout defect — the page looks cut off or collapsed, or a content container (calendar, table, chart, map, panel) visibly fails to stretch to the space its own layout provides.",
        user_template: "Inspect the attached screenshot(s) (ordered from the top of the page down) and the page text below.\n\nURL: {url}\nTitle: {title}\n\nPage Content:\n{content}\n\nDoes the page content fill the viewport as its layout intends — with no large blank region where the page appears to end partway down or partway across? Respond with exactly \"PASS\" if the page fills the viewport or is intentionally short, or \"FAIL: <describe the under-filled region and where it appears>\" if the content clearly stops at a fraction of the available page.",
    },
    AssertPreset {
        name: "visual_text_visible",
        system: "You are a visual QA engineer inspecting a website screenshot. Determine whether a specific text is FULLY visible and readable: present in the viewport, not clipped, not cut off, not covered by another element, and not obscured by overlays or low contrast stacking. A partial word or a covered text counts as FAIL.",
        user_template: "Inspect the attached screenshot(s) (ordered from the top of the page down) and the page text below.\n\nTEXT TO CHECK: \"{expected_text}\"\n\nURL: {url}\nTitle: {title}\n\nPage Content:\n{content}\n\nIs the text fully visible and readable in the screenshot (not clipped, covered, or hidden)? Respond with exactly \"PASS\" if it is fully visible, or \"FAIL: <explain what hides or clips it>\" otherwise.",
    },
    AssertPreset {
        name: "layout_no_issues",
        system: "DOM layout scanner: reports horizontal page overflow, elements sticking out of the viewport, clipped text in overflow:hidden containers, and interactive elements covered by other elements. Deterministic in-browser checks — no LLM call.",
        user_template: "Runs a deterministic DOM layout scan in the browser (no LLM call). Fails with the list of detected issues.",
    },
];

/// Whether an LLM verdict should be read as PASS.
///
/// Models routinely wrap the verdict in markdown (`**PASS** …`) or prefix it
/// with punctuation, which a bare `starts_with("pass")` misreads as a failure.
/// Strip any leading non-alphanumerics, then match the leading word; this also
/// accepts the natural-language forms `pass` / `passes`.
fn verdict_is_pass(content: &str) -> bool {
    content
        .trim()
        .trim_start_matches(|c: char| !c.is_alphanumeric())
        .to_lowercase()
        .starts_with("pass")
}

/// Whether an LLM verdict should be read as FAIL (see [`verdict_is_pass`]).
fn verdict_is_fail(content: &str) -> bool {
    content
        .trim()
        .trim_start_matches(|c: char| !c.is_alphanumeric())
        .to_lowercase()
        .starts_with("fail")
}

impl ScenarioRunner {
    /// Creates a new runner with the given scenario configuration and
    /// assertion definitions.
    #[must_use]
    #[allow(clippy::needless_pass_by_value)]
    pub fn new(scenario_config: ScenarioConfig, definitions: Vec<AssertDefinition>) -> Self {
        Self::with_reporter(scenario_config, definitions, Arc::new(Reporter::default()))
    }

    /// Creates a runner that reports run events through the given reporter.
    #[must_use]
    #[allow(clippy::needless_pass_by_value)]
    pub fn with_reporter(
        scenario_config: ScenarioConfig,
        definitions: Vec<AssertDefinition>,
        reporter: Arc<Reporter>,
    ) -> Self {
        Self::with_reporter_mode(scenario_config, definitions, reporter, true)
    }

    /// Creates a runner that reports run events through the given reporter
    /// but suppresses the `RunStarted`/`RunFinished` events. Used by the
    /// parallel orchestrator, which emits those once per batch.
    #[must_use]
    #[allow(clippy::needless_pass_by_value)]
    pub fn with_reporter_parallel(
        scenario_config: ScenarioConfig,
        definitions: Vec<AssertDefinition>,
        reporter: Arc<Reporter>,
    ) -> Self {
        Self::with_reporter_mode(scenario_config, definitions, reporter, false)
    }

    #[must_use]
    #[allow(clippy::needless_pass_by_value)]
    fn with_reporter_mode(
        scenario_config: ScenarioConfig,
        definitions: Vec<AssertDefinition>,
        reporter: Arc<Reporter>,
        emit_run_events: bool,
    ) -> Self {
        reporter.add_redaction_secrets(crate::redact::collect_secrets_from_scenario_config(
            &scenario_config,
        ));
        let llm = LlmConfig {
            url: scenario_config
                .llm_url
                .clone()
                .unwrap_or_else(crate::llm_base_url),
            model: scenario_config
                .llm_model
                .clone()
                .unwrap_or_else(crate::llm_model),
            api_key: scenario_config
                .llm_api_key
                .clone()
                .or_else(|| std::env::var("HARNESS_LLM_API_KEY").ok()),
            headers: if scenario_config.llm_headers.is_empty() {
                crate::parse_headers_env()
            } else {
                scenario_config.llm_headers.clone()
            },
            timeout: Duration::from_secs(scenario_config.timeout_secs.unwrap_or(60)),
            temperature: scenario_config.temperature,
            thinking: scenario_config.thinking,
            model_params: scenario_config.model_params.clone(),
            cache: scenario_config.cache.unwrap_or(true),
            max_attempts: crate::default_llm_attempts(),
            provider: crate::scenario::Provider::Openai,
            deployment: None,
            api_version: None,
            auth: crate::scenario::AuthConfig::default(),
            header_commands: std::collections::HashMap::new(),
            aws: crate::scenario::AwsConfig::default(),
        };
        let endpoints = EndpointRegistry::from_config(
            &scenario_config.endpoints,
            Some(&llm),
            crate::endpoints::EndpointDefaults {
                cache: scenario_config.cache,
                cache_pricing: scenario_config.cache_pricing,
            },
        );
        let budgets = BudgetTracker::from_config(&scenario_config.budgets);
        let defs_map: HashMap<String, AssertDefinition> = definitions
            .into_iter()
            .map(|d| (d.name.clone(), d))
            .collect();

        Self {
            timeout: Duration::from_secs(scenario_config.timeout_secs.unwrap_or(60)),
            viewport_width: scenario_config.viewport_width.unwrap_or(1280),
            viewport_height: scenario_config.viewport_height.unwrap_or(720),
            applied_viewport: std::cell::Cell::new((0, 0)),
            config: scenario_config.clone(),
            definitions: defs_map,
            llm,
            endpoints,
            usage: Arc::new(UsageTracker::new()),
            budgets,
            artifacts_dir: PathBuf::from(
                scenario_config
                    .artifacts_dir
                    .unwrap_or_else(|| "artifacts".to_owned()),
            ),
            reporter,
            current_step: std::cell::RefCell::new(None),
            run_started: SystemTime::now()
                .duration_since(UNIX_EPOCH)
                .map_or(0, |d| d.as_secs()),
            emit_run_events,
        }
    }

    /// Emits an event; a sink failure degrades to a console warning so a
    /// broken log file can never mask the run itself.
    fn emit_event(&self, event: &TestEvent) {
        if let Err(err) = self.reporter.emit(event) {
            use std::io::Write as _;
            let mut out = std::io::stderr().lock();
            let _ = out.write_fmt(format_args!("  ! failed to record event: {err}\n"));
        }
    }

    /// Returns a clone of the [`UsageTracker`] for reporting.
    #[must_use]
    pub fn usage_tracker(&self) -> Arc<UsageTracker> {
        Arc::clone(&self.usage)
    }

    /// Returns a reference to the [`BudgetTracker`].
    #[must_use]
    pub const fn budget_tracker(&self) -> &BudgetTracker {
        &self.budgets
    }

    /// Executes all test groups in the scenario and returns a report.
    ///
    /// # Errors
    ///
    /// Returns an error if the browser fails to launch.
    #[allow(clippy::too_many_lines, clippy::cast_possible_truncation)]
    pub fn run(&self, tests: &[TestGroup]) -> anyhow::Result<RunReport> {
        let mut report = RunReport::default();

        if tests.is_empty() {
            self.reporter.warn("No tests defined in scenario.");
            if self.emit_run_events {
                self.emit_event(&TestEvent::RunFinished {
                    tests_passed: 0,
                    tests_failed: 0,
                    steps_passed: 0,
                    steps_failed: 0,
                    steps_skipped: 0,
                    total_cost: 0.0,
                    total_tokens: 0,
                    total_input_tokens: 0,
                    total_output_tokens: 0,
                    total_cached_input_tokens: 0,
                    total_cache_creation_input_tokens: 0,
                    models: Vec::new(),
                    total_calls: 0,
                });
            }
            return Ok(report);
        }

        if self.emit_run_events {
            self.emit_event(&TestEvent::RunStarted {
                total_tests: tests.len() as u32,
            });
        }

        let browser_headless = self.config.browser_headless.unwrap_or(true);

        let launch_opts = LaunchOptions {
            headless: browser_headless,
            window_size: Some((self.viewport_width, self.viewport_height)),
            sandbox: false,
            // headless_chrome defaults this to 30s and shuts down the whole CDP
            // connection when no messages arrive for that long. A scenario can
            // easily exceed 30s of browser silence (slow LLM targeting/assertion
            // calls, page waits, budget checks between steps), after which every
            // remaining step fails with "Unable to make method calls because
            // underlying connection is closed" — one quiet gap kills the run.
            // Open-ended scenarios must own the connection for their full
            // duration, so keep it alive for 6 hours.
            idle_browser_timeout: BROWSER_IDLE_TIMEOUT,
            ..LaunchOptions::default()
        };

        let browser = Browser::new(launch_opts).context("failed to launch browser")?;
        let tab = browser.new_tab().context("failed to open browser tab")?;
        match (
            self.config.browser_basic_auth_user.clone(),
            self.config.browser_basic_auth_password.clone(),
        ) {
            (Some(username), Some(password)) if !username.is_empty() && !password.is_empty() => {
                tab.authenticate(Some(username), Some(password))
                    .context("failed to configure browser HTTP Basic Auth")?;
                tab.enable_fetch(None, Some(true))
                    .context("failed to enable browser HTTP authentication")?;
            }
            (None, None) => {}
            _ => anyhow::bail!("browser Basic Auth requires both username and password"),
        }
        let _ = tab.set_default_timeout(self.timeout);

        // Start MCP server if configured
        #[cfg(feature = "mcp-server")]
        if let Some(ref mcp_cfg) = self.config.mcp_server {
            if mcp_cfg.enabled {
                let port = mcp_cfg.port;
                std::thread::spawn(move || {
                    let _ = crate::mcp_server::start_mcp_server(port);
                });
            }
        }
        #[cfg(not(feature = "mcp-server"))]
        if let Some(mcp_cfg) = &self.config.mcp_server {
            if mcp_cfg.enabled {
                self.reporter
                    .warn("MCP server configured but 'mcp-server' feature not enabled");
            }
        }

        // Start A2A agent server if configured
        #[cfg(feature = "a2a-server")]
        if let Some(ref a2a_cfg) = self.config.a2a_server {
            if a2a_cfg.enabled {
                let port = a2a_cfg.port;
                tokio::spawn(crate::a2a_server::start_a2a_server(port));
            }
        }
        #[cfg(not(feature = "a2a-server"))]
        if let Some(a2a_cfg) = &self.config.a2a_server {
            if a2a_cfg.enabled {
                self.reporter
                    .warn("A2A server configured but 'a2a-server' feature not enabled");
            }
        }

        let retry_budget = self.config.retry_failed_tests.unwrap_or(0);

        for test in tests {
            // A shared tab on a contended runner makes some page loads
            // stall (a JS chunk or a GraphQL call hangs mid-flight) and
            // the test fails on a wait/assert that a fresh run passes.
            // Re-run the WHOLE test when it failed and the retry budget
            // allows; the fresh per-test isolation (login clear +
            // auto-navigate) applies to the retry too. Both attempts'
            // LLM spend stays in the budget accounting (each attempt
            // commits its usage); only the final attempt is reported.
            let details_len = report.details.len();
            let passed_before = report.passed;
            let failed_before = report.failed;
            let skipped_before = report.skipped;
            #[allow(unused_assignments)]
            let mut final_result = None;
            let mut attempt = 0;
            loop {
                attempt += 1;
                self.emit_event(&TestEvent::TestStarted {
                    test: test.name.clone(),
                });

                self.usage.reset_per_test();

                let test_started = Instant::now();
                let test_result = self.run_test(test, &tab);
                let duration_ms = test_started.elapsed().as_millis() as u64;
                let usage = self.usage.current_test_snapshot();
                self.usage.commit_test(&test.name);

                self.emit_event(&TestEvent::TestFinished {
                    test: test.name.clone(),
                    passed: test_result.passed,
                    failed: test_result.failed,
                    skipped: test_result.skipped,
                    duration_ms,
                    cost: usage.total_cost,
                    tokens: usage.total_tokens,
                    input_tokens: usage.total_input_tokens,
                    output_tokens: usage.total_output_tokens,
                    cached_input_tokens: usage.total_cached_input_tokens,
                    cache_creation_input_tokens: usage.total_cache_creation_input_tokens,
                    models: usage.models.clone(),
                    calls: usage.total_calls,
                });

                final_result = Some(test_result);
                let result = final_result.as_ref().expect("just assigned");
                if result.failed == 0 || result.total == 0 || attempt > retry_budget {
                    break;
                }
                self.reporter.warn(format!(
                    "! retrying failed test '{}' (attempt {}/{}) — a fresh run passes on \
                     transient page-load stalls",
                    test.name,
                    attempt + 1,
                    retry_budget + 1
                ));
                // Roll this failed attempt out of the report so the
                // retry's outcome replaces it.
                report.details.truncate(details_len);
                report.passed = passed_before;
                report.failed = failed_before;
                report.skipped = skipped_before;
            }

            let test_result = final_result.unwrap_or_default();
            if test_result.failed == 0 && test_result.total > 0 {
                report.tests_passed += 1;
            } else if test_result.total > 0 {
                report.tests_failed += 1;
            }

            report.passed += test_result.passed;
            report.failed += test_result.failed;
            report.skipped += test_result.skipped;
            report.details.extend(test_result.details);
        }

        let global = self.usage.global_snapshot();
        if self.emit_run_events {
            self.emit_event(&TestEvent::RunFinished {
                tests_passed: report.tests_passed,
                tests_failed: report.tests_failed,
                steps_passed: report.passed,
                steps_failed: report.failed,
                steps_skipped: report.skipped,
                total_cost: global.total_cost,
                total_tokens: global.total_tokens,
                total_input_tokens: global.total_input_tokens,
                total_output_tokens: global.total_output_tokens,
                total_cached_input_tokens: global.total_cached_input_tokens,
                total_cache_creation_input_tokens: global.total_cache_creation_input_tokens,
                models: global.models.clone(),
                total_calls: global.total_calls,
            });
        }

        Ok(report)
    }

    #[allow(clippy::too_many_lines, clippy::cast_possible_truncation)]
    fn run_test(&self, test: &TestGroup, tab: &Tab) -> TestRunResult {
        let base_url = test
            .base_url
            .clone()
            .or_else(|| self.config.base_url.clone())
            .unwrap_or_else(crate::base_url);

        // Per-test viewport override: switch the browser via CDP
        // device-metrics emulation before this test runs.
        let vw = test.viewport_width.unwrap_or(self.viewport_width);
        let vh = test.viewport_height.unwrap_or(self.viewport_height);
        if self.applied_viewport.get() != (vw, vh) {
            self.apply_viewport(tab, vw, vh);
            self.applied_viewport.set((vw, vh));
        }

        // Per-test isolation: every test starts from its own start_url
        // (unless auto_navigate is disabled), so a test never inherits the
        // previous test's page state.
        let auto_navigate = test.auto_navigate.unwrap_or(self.config.auto_navigate);

        let start_url = test
            .start_url
            .clone()
            .or_else(|| self.config.start_url.clone())
            .unwrap_or_else(|| "/dashboard".to_owned());

        // Per-test state isolation for LOGIN tests: the tab is shared
        // across the file's tests, and a previous test's login persists
        // (Cognito tokens in localStorage + the hosted-UI cookies). The
        // SPA's login pages then auto-continue authenticated visitors on
        // boot, so a later login test never sees the form and times out
        // waiting for `#email` (observed on immosai runs #1641/#1642: a
        // shard's first three logins pass, the fourth onward time out).
        // Clear cookies + origin storage ONLY for tests that perform
        // their own login (their first navigate step targets a login
        // route, or they have no navigate step and ride the auto-nav
        // onto a login start_url). The many "page loads" tests that
        // navigate app pages directly RELY on the shared session — a
        // blanket clear bounces them off their target (run #1643: every
        // shard failed with "the current page is the login page, not
        // the mailboxes page"). The HTTP cache is deliberately NOT
        // cleared — re-downloading the SPA bundle per test would only
        // add boot time on a slow runner egress.
        if auto_navigate && test_targets_login(&start_url, &test.steps) {
            use headless_chrome::protocol::cdp::{Network, Storage};
            if let Err(e) = tab.call_method(Network::ClearBrowserCookies(None)) {
                self.reporter
                    .warn(format!("per-test cookie clear failed: {e}"));
            }
            if let Some(origin) = origin_of(&base_url) {
                if let Err(e) = tab.call_method(Storage::ClearDataForOrigin {
                    origin,
                    storage_Types: "all".to_string(),
                }) {
                    self.reporter
                        .warn(format!("per-test storage clear failed: {e}"));
                }
            }
        }
        if auto_navigate {
            let full_url = resolve_url(&start_url, &base_url);
            self.reporter.debug(format!("auto-navigate: {full_url}"));
            let _ = tab.navigate_to(&full_url);
            let _ = tab.wait_until_navigated();
            std::thread::sleep(Duration::from_secs(4));
        }

        let mut result = TestRunResult::default();

        for (step_index, step) in test.steps.iter().enumerate() {
            result.total += 1;

            let wait_ms = match step {
                TestStep::Navigate { wait_after_ms, .. }
                | TestStep::Click { wait_after_ms, .. }
                | TestStep::Type { wait_after_ms, .. } => *wait_after_ms,
                _ => None,
            };

            self.current_step
                .replace(Some((test.name.clone(), step_index as u32)));
            self.emit_event(&TestEvent::StepStarted {
                test: test.name.clone(),
                index: step_index as u32,
                label: step_label(step),
            });
            let step_started = Instant::now();

            let mut step_result = match step {
                TestStep::Navigate { url, .. } => {
                    let full_url = resolve_url(url, &base_url);
                    run_navigate_step(&full_url, tab)
                }
                TestStep::Click {
                    target,
                    selector,
                    endpoint,
                    idempotent,
                    ..
                } => self.run_click(
                    target,
                    selector.as_deref(),
                    endpoint.as_deref(),
                    test.endpoint.as_deref(),
                    *idempotent,
                    tab,
                ),
                TestStep::Type {
                    target,
                    text,
                    selector,
                    endpoint,
                    idempotent,
                    ..
                } => self.run_type(
                    target,
                    text,
                    selector.as_deref(),
                    endpoint.as_deref(),
                    test.endpoint.as_deref(),
                    *idempotent,
                    tab,
                ),
                TestStep::Wait {
                    target,
                    selector,
                    text,
                    timeout_ms,
                    endpoint,
                    idempotent,
                } => self.run_wait(
                    target,
                    selector.as_deref(),
                    text.as_deref(),
                    *timeout_ms,
                    endpoint.as_deref(),
                    test.endpoint.as_deref(),
                    *idempotent,
                    tab,
                ),
                TestStep::Assert {
                    definition,
                    preset,
                    prompt,
                    assert_text,
                    endpoint,
                    screenshot,
                } => self.run_assert(
                    definition.as_deref(),
                    preset.as_deref(),
                    prompt.as_deref(),
                    assert_text.as_deref(),
                    *screenshot,
                    endpoint.as_deref(),
                    test.endpoint.as_deref(),
                    tab,
                ),
                TestStep::Screenshot { path } => Self::run_screenshot(path.as_deref(), tab),
                TestStep::Agent {
                    agent,
                    task,
                    definition,
                } => self.run_agent(agent, task, definition.as_deref(), test.endpoint.as_deref()),
                TestStep::Mcp { server, tool, args } => self.run_mcp(server, tool, args.as_ref()),
            };

            // Failure diagnostics: capture the page state and a screenshot so
            // CI logs say WHAT the page looked like when the step failed,
            // instead of a bare "timed out: The event waited for never came".
            let (diagnostics_block, screenshot_path) = if step_result.status == StepStatus::Failed {
                let state = diagnostics::capture(tab);
                let screenshot = diagnostics::save_screenshot(
                    tab,
                    &self.artifacts_dir,
                    &test.name,
                    &test.name,
                    step_index,
                    step_kind_label(step),
                );
                step_result.message = format!(
                    "{base} — {excerpt}",
                    base = step_result.message,
                    excerpt = diagnostics::inline_excerpt(&state),
                );
                (Some(diagnostics::full_context(&state)), screenshot)
            } else {
                (None, None)
            };

            let duration_ms = step_started.elapsed().as_millis() as u64;
            self.emit_event(&TestEvent::StepFinished {
                test: test.name.clone(),
                index: step_index as u32,
                label: step_result.name.clone(),
                status: step_result.status,
                duration_ms,
                message: step_result.message.clone(),
                diagnostics: diagnostics_block,
                screenshot: screenshot_path,
            });
            self.current_step.replace(None);

            match step_result.status {
                StepStatus::Passed => result.passed += 1,
                StepStatus::Failed => result.failed += 1,
                StepStatus::Skipped => result.skipped += 1,
            }

            // Fail fast: the first failed step ends the test and the
            // remaining steps are reported as skipped (no LLM budget is
            // burned asserting against a page that is already known broken).
            if step_result.status == StepStatus::Failed
                && !self.config.continue_on_failure
                && step_index + 1 < test.steps.len()
            {
                self.emit_event(&TestEvent::Warning {
                    message: format!(
                        "failing fast: {} remaining step(s) skipped (set continue_on_failure = true in [config] to disable)",
                        test.steps.len() - step_index - 1
                    ),
                });
                for (offset, skipped) in test.steps[step_index + 1..].iter().enumerate() {
                    let skipped_index = step_index + 1 + offset;
                    let label = step_label(skipped);
                    result.total += 1;
                    result.skipped += 1;
                    self.emit_event(&TestEvent::StepStarted {
                        test: test.name.clone(),
                        index: skipped_index as u32,
                        label: label.clone(),
                    });
                    self.emit_event(&TestEvent::StepFinished {
                        test: test.name.clone(),
                        index: skipped_index as u32,
                        label,
                        status: StepStatus::Skipped,
                        duration_ms: 0,
                        message: "skipped: previous step failed".into(),
                        diagnostics: None,
                        screenshot: None,
                    });
                    result.details.push(StepResult {
                        name: step_label(skipped),
                        status: StepStatus::Skipped,
                        message: "skipped: previous step failed".into(),
                    });
                }
                result.details.push(step_result);
                return result;
            }

            // Check per-test budget after each step
            let test_usage = self.usage.current_test_snapshot();
            let global_usage = self.usage.global_snapshot();
            let budget_status = self.budgets.check_all(
                &test.name,
                &test_usage,
                &global_usage,
                test.budget.as_ref(),
            );
            match budget_status {
                BudgetStatus::HardExceeded { message, .. } => {
                    self.emit_event(&TestEvent::Warning {
                        message: format!("budget exceeded: {message}"),
                    });
                    result.details.push(StepResult {
                        name: "[budget]".into(),
                        status: StepStatus::Failed,
                        message,
                    });
                    result.failed += 1;
                    return result;
                }
                BudgetStatus::SoftExceeded { message, .. } => {
                    self.emit_event(&TestEvent::Warning {
                        message: format!("budget warning: {message}"),
                    });
                }
                BudgetStatus::Ok => {}
            }

            if let Some(ms) = wait_ms {
                std::thread::sleep(Duration::from_millis(ms));
            }

            result.details.push(step_result);
        }

        result
    }

    /// Applies a viewport size to the current tab via CDP
    /// `Emulation.setDeviceMetricsOverride`. Used by per-test viewport
    /// overrides and the viewport matrix. The initial window size set at
    /// browser launch is replaced by emulation; failures are logged but
    /// do not fail the test (a mismatched viewport only weakens coverage).
    fn apply_viewport(&self, tab: &Tab, width: u32, height: u32) {
        use headless_chrome::protocol::cdp::Emulation;
        let _ = self;
        let params = Emulation::SetDeviceMetricsOverride {
            width,
            height,
            device_scale_factor: 1.0,
            mobile: false,
            scale: None,
            screen_width: Some(width),
            screen_height: Some(height),
            position_x: None,
            position_y: None,
            dont_set_visible_size: None,
            screen_orientation: None,
            viewport: None,
            display_feature: None,
            device_posture: None,
        };
        self.reporter.debug(format!("viewport: {width}x{height}"));
        if let Err(e) = tab.call_method(params) {
            self.reporter
                .warn(format!("viewport switch to {width}x{height} failed: {e}"));
        }
    }

    /// Height (px) covered by assert-step screenshots: the configured
    /// `screenshot_max_height` (absolute px or viewport multiple, default
    /// `"20x"`) resolved against the currently applied viewport, raised to
    /// at least the viewport height so the visible screen is always fully
    /// included. The capture is split into viewport-tall tiles, so this
    /// value bounds total coverage (and hence the number of image parts).
    #[must_use]
    fn screenshot_height_cap(&self) -> u32 {
        let viewport_height = self.current_viewport_height();
        let cap = self
            .config
            .screenshot_max_height
            .as_ref()
            .map_or(viewport_height * 20, |h| h.to_px(viewport_height));
        cap.max(viewport_height)
    }

    /// Height of the viewport currently emulated in the browser (falling
    /// back to the configured default before any emulation was applied).
    #[must_use]
    const fn current_viewport_height(&self) -> u32 {
        let (_, height) = self.applied_viewport.get();
        if height > 0 {
            height
        } else {
            self.viewport_height
        }
    }

    // ── step handlers ───────────────────────────────────────────────────

    #[allow(clippy::too_many_lines)]
    fn run_click(
        &self,
        target: &str,
        selector_override: Option<&str>,
        step_endpoint: Option<&str>,
        test_endpoint: Option<&str>,
        idempotent: bool,
        tab: &Tab,
    ) -> StepResult {
        let name = format!("[click] {target}");
        let selector = match self.resolve_selector(
            selector_override,
            target,
            step_endpoint,
            test_endpoint,
            tab,
        ) {
            Ok(s) => s,
            Err(msg) => {
                if idempotent {
                    return StepResult {
                        name,
                        status: StepStatus::Skipped,
                        message: format!("skipped (idempotent): no target found — {msg}"),
                    };
                }
                return StepResult {
                    name,
                    status: StepStatus::Failed,
                    message: msg,
                };
            }
        };

        // Idempotent steps probe briefly: a missing target means the
        // action was already done / not applicable (e.g. an
        // already-authenticated session), and skipping is the success
        // path, not a failure.
        let probe_secs = if idempotent { 5 } else { 10 };
        match tab.wait_for_element_with_custom_timeout(&selector, Duration::from_secs(probe_secs)) {
            Ok(element) => match element.click() {
                Ok(_) => StepResult {
                    name,
                    status: StepStatus::Passed,
                    message: format!("clicked {selector}"),
                },
                Err(e) => StepResult {
                    name,
                    status: StepStatus::Failed,
                    message: format!("click failed on {selector}: {e}"),
                },
            },
            Err(e) if idempotent => StepResult {
                name,
                status: StepStatus::Skipped,
                message: format!("skipped (idempotent): element {selector} not present — {e}"),
            },
            Err(e) => StepResult {
                name,
                status: StepStatus::Failed,
                message: format!("element {selector} not found: {e}"),
            },
        }
    }

    #[allow(clippy::too_many_arguments)]
    fn run_type(
        &self,
        target: &str,
        text: &str,
        selector_override: Option<&str>,
        step_endpoint: Option<&str>,
        test_endpoint: Option<&str>,
        idempotent: bool,
        tab: &Tab,
    ) -> StepResult {
        let name = format!("[type] {target}");
        let selector = match self.resolve_selector(
            selector_override,
            target,
            step_endpoint,
            test_endpoint,
            tab,
        ) {
            Ok(s) => s,
            Err(msg) => {
                if idempotent {
                    return StepResult {
                        name,
                        status: StepStatus::Skipped,
                        message: format!("skipped (idempotent): no target found — {msg}"),
                    };
                }
                return StepResult {
                    name,
                    status: StepStatus::Failed,
                    message: msg,
                };
            }
        };

        let probe_secs = if idempotent { 5 } else { 10 };
        match tab.wait_for_element_with_custom_timeout(&selector, Duration::from_secs(probe_secs)) {
            Ok(element) => {
                if let Err(e) = element.click() {
                    return StepResult {
                        name,
                        status: StepStatus::Failed,
                        message: format!("click to focus {selector} failed: {e}"),
                    };
                }

                let js = format!(
                    "document.querySelector('{}').value = '';",
                    selector.replace('\'', "\\'")
                );
                let _ = tab.evaluate(&js, false);

                match element.type_into(text) {
                    Ok(_) => StepResult {
                        name,
                        status: StepStatus::Passed,
                        message: format!("typed {text:?} into {selector}"),
                    },
                    Err(e) => StepResult {
                        name,
                        status: StepStatus::Failed,
                        message: format!("type into {selector} failed: {e}"),
                    },
                }
            }
            Err(e) if idempotent => StepResult {
                name,
                status: StepStatus::Skipped,
                message: format!("skipped (idempotent): element {selector} not present — {e}"),
            },
            Err(e) => StepResult {
                name,
                status: StepStatus::Failed,
                message: format!("element {selector} not found: {e}"),
            },
        }
    }

    #[allow(clippy::too_many_arguments)]
    #[allow(clippy::too_many_lines)]
    fn run_wait(
        &self,
        target: &str,
        selector_override: Option<&str>,
        text: Option<&str>,
        timeout_ms: Option<u64>,
        step_endpoint: Option<&str>,
        test_endpoint: Option<&str>,
        idempotent: bool,
        tab: &Tab,
    ) -> StepResult {
        let timeout = Duration::from_millis(timeout_ms.unwrap_or(10_000));
        let step_name = format!("[wait] {target}");

        // Resolve an explicit selector only (text-only waits are LLM-free).
        let selector = match selector_override {
            Some(s) => Some(s.to_owned()),
            None if text.is_some() => None,
            None => match self.resolve_selector(None, target, step_endpoint, test_endpoint, tab) {
                Ok(s) => Some(s),
                Err(msg) => {
                    if idempotent {
                        return StepResult {
                            name: step_name,
                            status: StepStatus::Skipped,
                            message: format!("skipped (idempotent): no target found — {msg}"),
                        };
                    }
                    return StepResult {
                        name: step_name,
                        status: StepStatus::Failed,
                        message: msg,
                    };
                }
            },
        };

        if text.is_some() {
            let sel_js = selector
                .as_deref()
                .map(crate::selectors::selector_matches_js);
            let text_js = text.map(|t| {
                let escaped = t.replace('\\', "\\\\").replace('\'', "\\'");
                format!("document.body ? document.body.innerText.includes('{escaped}') : false")
            });

            let deadline = Instant::now() + timeout;
            loop {
                let sel_ok = sel_js
                    .as_ref()
                    .is_none_or(|js| eval_bool(tab, js).unwrap_or(false));
                let text_ok = text_js
                    .as_ref()
                    .is_none_or(|js| eval_bool(tab, js).unwrap_or(false));
                if sel_ok && text_ok {
                    let mut what = Vec::new();
                    if let Some(sel) = &selector {
                        what.push(format!("found {sel}"));
                    }
                    if let Some(t) = text {
                        what.push(format!("text {t:?} visible"));
                    }
                    return StepResult {
                        name: step_name,
                        status: StepStatus::Passed,
                        message: what.join(" and "),
                    };
                }
                if Instant::now() >= deadline {
                    let mut what = Vec::new();
                    if let Some(sel) = &selector {
                        what.push(sel.clone());
                    }
                    if let Some(t) = text {
                        what.push(format!("text {t:?}"));
                    }
                    let message = format!(
                        "wait for {} timed out after {}ms: the event waited for never came",
                        what.join(" / "),
                        timeout.as_millis(),
                    );
                    if idempotent {
                        return StepResult {
                            name: step_name,
                            status: StepStatus::Skipped,
                            message: format!("skipped (idempotent): {message}"),
                        };
                    }
                    return StepResult {
                        name: step_name,
                        status: StepStatus::Failed,
                        message,
                    };
                }
                std::thread::sleep(Duration::from_millis(250));
            }
        }

        match selector.as_deref() {
            Some(sel) => match tab.wait_for_element_with_custom_timeout(sel, timeout) {
                Ok(_) => StepResult {
                    name: step_name,
                    status: StepStatus::Passed,
                    message: format!("found {sel}"),
                },
                Err(e) if idempotent => StepResult {
                    name: step_name,
                    status: StepStatus::Skipped,
                    message: format!(
                        "skipped (idempotent): wait for {sel} timed out after {}ms: {e}",
                        timeout.as_millis()
                    ),
                },
                Err(e) => StepResult {
                    name: step_name,
                    status: StepStatus::Failed,
                    message: format!(
                        "wait for {sel} timed out after {}ms: {e}",
                        timeout.as_millis()
                    ),
                },
            },
            None => StepResult {
                name: step_name,
                status: StepStatus::Failed,
                message: "wait step has neither selector nor text".into(),
            },
        }
    }

    #[allow(clippy::too_many_arguments)]
    fn run_assert(
        &self,
        definition: Option<&str>,
        preset: Option<&str>,
        prompt: Option<&str>,
        assert_text: Option<&str>,
        screenshot: bool,
        step_endpoint: Option<&str>,
        test_endpoint: Option<&str>,
        tab: &Tab,
    ) -> StepResult {
        std::thread::sleep(Duration::from_millis(500));

        let page_content = get_page_text(tab);

        // Vision attach: capture the full page once per assert step and
        // split it into viewport-tall tiles (the total coverage is bounded
        // by the configured height cap so vision tokens stay sane). All
        // tile data URLs are handed to the preset/prompt evaluation below.
        let image: Option<Vec<String>> = if screenshot {
            let endpoint = self
                .endpoints
                .resolve(step_endpoint.or(test_endpoint), TaskType::Assertion);
            if !endpoint.vision {
                return StepResult {
                    name: "[assert]".into(),
                    status: StepStatus::Failed,
                    message: format!(
                        "screenshot requested but endpoint '{name}' does not declare vision = true (add vision = true to its [config.endpoints] entry)",
                        name = endpoint.name
                    ),
                };
            }
            match crate::vision::capture_screenshot_data_urls(
                tab,
                self.config
                    .screenshot_max_dimension
                    .unwrap_or(crate::vision::DEFAULT_MAX_DIMENSION),
                self.screenshot_height_cap(),
                self.current_viewport_height(),
            ) {
                Ok(urls) => Some(urls),
                Err(e) => {
                    return StepResult {
                        name: "[assert]".into(),
                        status: StepStatus::Failed,
                        message: format!("screenshot capture failed: {e}"),
                    };
                }
            }
        } else {
            None
        };

        if let Some(def_name) = definition {
            if let Some(def) = self.definitions.get(def_name) {
                return self.run_assert_def(
                    def,
                    &page_content,
                    image.as_deref(),
                    step_endpoint,
                    test_endpoint,
                    tab,
                );
            }
            return StepResult {
                name: format!("[assert] {def_name}"),
                status: StepStatus::Failed,
                message: format!("definition '{def_name}' not found"),
            };
        }

        if let Some(preset_name) = preset {
            // Deterministic DOM layout scan — runs JS in the browser and
            // never calls the LLM (free, fast, no pixel budget).
            if preset_name == "layout_no_issues" {
                return self.run_layout_preset(tab);
            }
            return self.run_preset(
                preset_name,
                assert_text,
                &page_content,
                image.as_deref(),
                step_endpoint,
                test_endpoint,
            );
        }

        if let Some(prompt_text) = prompt {
            return self.run_custom(
                prompt_text,
                &page_content,
                image.as_deref(),
                step_endpoint,
                test_endpoint,
            );
        }

        StepResult {
            name: "[assert]".into(),
            status: StepStatus::Skipped,
            message: "no definition, preset, or prompt specified".into(),
        }
    }

    fn run_assert_def(
        &self,
        def: &AssertDefinition,
        page_content: &PageContent,
        image: Option<&[String]>,
        step_endpoint: Option<&str>,
        test_endpoint: Option<&str>,
        tab: &Tab,
    ) -> StepResult {
        // Agent-based definition: delegate to an A2A agent
        if let Some(ref agent) = def.agent {
            if image.is_some() {
                return StepResult {
                    name: format!("[assert] {}", def.name),
                    status: StepStatus::Failed,
                    message: "agent-backed assertions do not support screenshots".into(),
                };
            }
            let task = def
                .task_template
                .as_deref()
                .unwrap_or("Evaluate the assertion")
                .replace("{url}", &page_content.url)
                .replace("{title}", &page_content.title)
                .replace("{content}", &page_content.body_text)
                .replace("{expected_text}", def.assert_text.as_deref().unwrap_or(""));

            return self.run_agent_step(agent, &task, &def.name);
        }

        // Custom preset: system + user_template provided in the definition
        if let (Some(system), Some(template)) = (&def.system, &def.user_template) {
            return self.run_custom_preset(
                &def.name,
                system,
                template,
                def.assert_text.as_deref(),
                page_content,
                image,
                step_endpoint,
                test_endpoint,
            );
        }

        def.preset.as_ref().map_or_else(
            || {
                def.prompt.as_ref().map_or_else(
                    || StepResult {
                        name: format!("[assert] {}", def.name),
                        status: StepStatus::Failed,
                        message: "definition has no preset, prompt, or system+user_template".into(),
                    },
                    |prompt| {
                        self.run_custom(prompt, page_content, image, step_endpoint, test_endpoint)
                    },
                )
            },
            |preset_name| {
                if preset_name == "layout_no_issues" {
                    return self.run_layout_preset(tab);
                }
                self.run_preset(
                    preset_name,
                    def.assert_text.as_deref(),
                    page_content,
                    image,
                    step_endpoint,
                    test_endpoint,
                )
            },
        )
    }

    #[allow(clippy::too_many_arguments)]
    fn run_custom_preset(
        &self,
        name: &str,
        system: &str,
        template: &str,
        assert_text: Option<&str>,
        page_content: &PageContent,
        image: Option<&[String]>,
        step_endpoint: Option<&str>,
        test_endpoint: Option<&str>,
    ) -> StepResult {
        let user_prompt = template
            .replace("{url}", &page_content.url)
            .replace("{title}", &page_content.title)
            .replace("{content}", &page_content.body_text)
            .replace("{expected_text}", assert_text.unwrap_or(""))
            .replace("{description}", "");

        // Custom preset definitions frequently forget the {content}
        // placeholder — without it the LLM has no page to evaluate and
        // answers "I can't determine that without seeing the page". Always
        // append the page context unless the template already references it.
        let user_prompt = if template.contains("{content}") {
            user_prompt
        } else {
            format!(
                "{user_prompt}\n\nPage URL: {url}\nPage Title: {title}\n\nPage Content:\n{content}",
                url = page_content.url,
                title = page_content.title,
                content = page_content.body_text,
            )
        };

        self.reporter
            .debug(format!("assert: {name} (custom preset)"));

        let chain = self
            .endpoints
            .resolve_chain(step_endpoint.or(test_endpoint), TaskType::Assertion);
        let sys = system.to_owned();

        let response = self.llm_call_chain(&chain, &sys, &user_prompt, image, "assertion");

        response.map_or_else(
            |e| StepResult {
                name: format!("[assert] {name}"),
                status: StepStatus::Failed,
                message: format!("LLM assertion call failed: {e}"),
            },
            |(lr, _idx)| {
                if verdict_is_pass(&lr.content) {
                    StepResult {
                        name: format!("[assert] {name}"),
                        status: StepStatus::Passed,
                        message: "PASS".into(),
                    }
                } else {
                    StepResult {
                        name: format!("[assert] {name}"),
                        status: StepStatus::Failed,
                        message: lr.content,
                    }
                }
            },
        )
    }

    fn run_preset(
        &self,
        preset_name: &str,
        assert_text: Option<&str>,
        page_content: &PageContent,
        image: Option<&[String]>,
        step_endpoint: Option<&str>,
        test_endpoint: Option<&str>,
    ) -> StepResult {
        let Some(preset) = ASSERTION_PRESETS.iter().find(|p| p.name == preset_name) else {
            return StepResult {
                name: format!("[assert] {preset_name}"),
                status: StepStatus::Failed,
                message: format!("unknown assertion preset: {preset_name}"),
            };
        };
        if preset_name.starts_with("visual_") && image.is_none() {
            return StepResult {
                name: format!("[assert] {preset_name}"),
                status: StepStatus::Failed,
                message: format!(
                    "preset '{preset_name}' evaluates the page screenshot — set screenshot = true on the assert step and point it at a vision endpoint (vision = true)"
                ),
            };
        }

        let user_prompt = preset
            .user_template
            .replace("{url}", &page_content.url)
            .replace("{title}", &page_content.title)
            .replace("{content}", &page_content.body_text)
            .replace("{expected_text}", assert_text.unwrap_or(""))
            .replace("{description}", "");

        // Same safety net as custom presets: never let the LLM answer with
        // no page context at all.
        let user_prompt = if preset.user_template.contains("{content}") {
            user_prompt
        } else {
            format!(
                "{user_prompt}\n\nPage URL: {url}\nPage Title: {title}\n\nPage Content:\n{content}",
                url = page_content.url,
                title = page_content.title,
                content = page_content.body_text,
            )
        };

        self.reporter.debug(format!("assert: {preset_name}"));

        let chain = self
            .endpoints
            .resolve_chain(step_endpoint.or(test_endpoint), TaskType::Assertion);
        let sys = preset.system.to_owned();

        let response = self.llm_call_chain(&chain, &sys, &user_prompt, image, "assertion");

        response.map_or_else(
            |e| StepResult {
                name: format!("[assert] {preset_name}"),
                status: StepStatus::Failed,
                message: format!("LLM assertion call failed: {e}"),
            },
            |(lr, _idx)| {
                if verdict_is_pass(&lr.content) {
                    StepResult {
                        name: format!("[assert] {preset_name}"),
                        status: StepStatus::Passed,
                        message: "PASS".into(),
                    }
                } else {
                    StepResult {
                        name: format!("[assert] {preset_name}"),
                        status: StepStatus::Failed,
                        message: lr.content,
                    }
                }
            },
        )
    }

    /// Runs the deterministic DOM layout scan (`layout_no_issues`).
    ///
    /// Evaluates the layout-scan JS in the page and fails with the list of
    /// detected issues: horizontal page overflow, visible elements sticking
    /// out of the viewport, text clipped by `overflow: hidden` containers,
    /// and interactive elements covered by other elements. No LLM call —
    /// checks are geometry-based so the check is free, deterministic, and
    /// safe to run on every page × viewport variant.
    fn run_layout_preset(&self, tab: &Tab) -> StepResult {
        let name = "[assert] layout_no_issues".to_owned();
        self.reporter
            .debug("assert: layout_no_issues (DOM layout scan)");
        let js = LAYOUT_SCAN_JS.replace(
            "__IGNORE_CLASSES__",
            &serde_json::to_string(&self.config.layout_ignore_classes)
                .unwrap_or_else(|_| "[]".to_owned()),
        );
        let result = tab.evaluate(&js, false);
        let json_str = match result {
            Ok(r) => r
                .value
                .as_ref()
                .and_then(|v| v.as_str().map(String::from))
                .unwrap_or_else(|| "[]".to_owned()),
            Err(e) => {
                return StepResult {
                    name,
                    status: StepStatus::Failed,
                    message: format!("layout scan JS failed: {e}"),
                };
            }
        };
        let issues: Vec<LayoutIssue> = serde_json::from_str(&json_str).unwrap_or_default();
        if issues.is_empty() {
            return StepResult {
                name,
                status: StepStatus::Passed,
                message: "PASS — no layout defects detected".into(),
            };
        }
        let mut lines: Vec<String> = issues
            .iter()
            .take(10)
            .map(|i| {
                format!(
                    "- [{type_}] {element}: {detail}",
                    type_ = i.issue_type,
                    element = i.element,
                    detail = i.detail
                )
            })
            .collect();
        if issues.len() > 10 {
            lines.push(format!("- … and {} more", issues.len() - 10));
        }
        StepResult {
            name,
            status: StepStatus::Failed,
            message: format!(
                "FAIL — {} layout defect(s) detected:\n{}",
                issues.len(),
                lines.join("\n")
            ),
        }
    }

    fn run_custom(
        &self,
        prompt: &str,
        page_content: &PageContent,
        image: Option<&[String]>,
        step_endpoint: Option<&str>,
        test_endpoint: Option<&str>,
    ) -> StepResult {
        let system = "You are a QA tester evaluating a web page. Respond with exactly \"PASS\" if the assertion holds, or \"FAIL: <reason>\" if it does not.";

        let mut user = format!(
            "Page URL: {url}\nPage Title: {title}\n\nPage Content:\n{content}\n\nAssertion: {prompt}",
            url = page_content.url,
            title = page_content.title,
            content = page_content.body_text,
        );
        if image.is_some() {
            user.push_str(
                "\n\nA screenshot of the page is attached — inspect it for visual evidence when answering.",
            );
        }

        self.reporter.debug("custom assert");

        let chain = self
            .endpoints
            .resolve_chain(step_endpoint.or(test_endpoint), TaskType::Assertion);
        let sys = system.to_owned();

        let response = self.llm_call_chain(&chain, &sys, &user, image, "assertion");

        response.map_or_else(
            |e| StepResult {
                name: "[assert] custom".into(),
                status: StepStatus::Failed,
                message: format!("LLM assertion call failed: {e}"),
            },
            |(lr, _idx)| {
                if verdict_is_pass(&lr.content) {
                    StepResult {
                        name: "[assert] custom".into(),
                        status: StepStatus::Passed,
                        message: "PASS".into(),
                    }
                } else {
                    StepResult {
                        name: "[assert] custom".into(),
                        status: StepStatus::Failed,
                        message: lr.content,
                    }
                }
            },
        )
    }

    fn run_screenshot(path: Option<&str>, tab: &Tab) -> StepResult {
        let path = path.unwrap_or("screenshot.png");

        match tab.capture_screenshot(
            headless_chrome::protocol::cdp::Page::CaptureScreenshotFormatOption::Png,
            None,
            None,
            true,
        ) {
            Ok(data) => {
                if let Err(e) = std::fs::write(path, &data) {
                    return StepResult {
                        name: format!("[screenshot] {path}"),
                        status: StepStatus::Failed,
                        message: format!("failed to write screenshot: {e}"),
                    };
                }
                StepResult {
                    name: format!("[screenshot] {path}"),
                    status: StepStatus::Passed,
                    message: format!("saved to {path}"),
                }
            }
            Err(e) => StepResult {
                name: format!("[screenshot] {path}"),
                status: StepStatus::Failed,
                message: format!("screenshot failed: {e}"),
            },
        }
    }

    /// Runs an A2A agent step.
    #[allow(clippy::literal_string_with_formatting_args)]
    fn run_agent(
        &self,
        agent_name: &str,
        task: &str,
        definition: Option<&str>,
        _test_endpoint: Option<&str>,
    ) -> StepResult {
        // If a definition is specified, look up the task template
        let resolved_task = if let Some(def_name) = definition {
            if let Some(def) = self.definitions.get(def_name) {
                let tmpl = def.task_template.as_deref().unwrap_or(task);
                tmpl.replace("{task}", task)
            } else {
                return StepResult {
                    name: format!("[agent] {def_name}"),
                    status: StepStatus::Failed,
                    message: format!("definition '{def_name}' not found"),
                };
            }
        } else {
            task.to_owned()
        };

        self.run_agent_step(agent_name, &resolved_task, &format!("agent:{agent_name}"))
    }

    fn run_agent_step(&self, agent_name: &str, task: &str, display_name: &str) -> StepResult {
        let Some(ep) = self.endpoints.get(agent_name) else {
            return StepResult {
                name: format!("[agent] {display_name}"),
                status: StepStatus::Failed,
                message: format!("agent endpoint '{agent_name}' not found"),
            };
        };

        if ep.url.is_empty() {
            return StepResult {
                name: format!("[agent] {display_name}"),
                status: StepStatus::Failed,
                message: format!("agent endpoint '{agent_name}' has no URL"),
            };
        }

        self.reporter.debug(format!("agent {agent_name}: {task}"));

        let url = ep.url.clone();
        let client = A2aClient::new(&url, self.timeout);
        let task_clone = task.to_owned();

        let response = std::thread::spawn(move || {
            let rt = tokio::runtime::Builder::new_current_thread()
                .enable_all()
                .build()
                .unwrap();
            rt.block_on(client.send_task(&task_clone))
        })
        .join()
        .unwrap();

        // Record the flat-cost call
        self.usage.record_flat_call(agent_name, ep);

        match response {
            Ok(text) => {
                let clean = text.trim().to_owned();
                if verdict_is_pass(&clean) {
                    StepResult {
                        name: format!("[agent] {display_name}"),
                        status: StepStatus::Passed,
                        message: format!("PASS: {clean}"),
                    }
                } else if verdict_is_fail(&clean) {
                    StepResult {
                        name: format!("[agent] {display_name}"),
                        status: StepStatus::Failed,
                        message: clean,
                    }
                } else {
                    StepResult {
                        name: format!("[agent] {display_name}"),
                        status: StepStatus::Passed,
                        message: format!("response: {clean}"),
                    }
                }
            }
            Err(e) => StepResult {
                name: format!("[agent] {display_name}"),
                status: StepStatus::Failed,
                message: format!("agent call failed: {e}"),
            },
        }
    }

    /// Runs an MCP tool call step.
    fn run_mcp(
        &self,
        server_name: &str,
        tool_name: &str,
        args: Option<&serde_json::Value>,
    ) -> StepResult {
        let Some(ep) = self.endpoints.get(server_name) else {
            return StepResult {
                name: format!("[mcp] {server_name}:{tool_name}"),
                status: StepStatus::Failed,
                message: format!("MCP server endpoint '{server_name}' not found"),
            };
        };

        let cmd = ep.command.as_deref().unwrap_or("");
        if cmd.is_empty() {
            return StepResult {
                name: format!("[mcp] {server_name}:{tool_name}"),
                status: StepStatus::Failed,
                message: format!("MCP server '{server_name}' has no command configured"),
            };
        }

        self.reporter
            .debug(format!("mcp {server_name} {tool_name}"));

        let args_val = args.cloned().unwrap_or(serde_json::Value::Null);

        let command = cmd.to_owned();
        let args_vec = ep.args.clone();
        let tool = tool_name.to_owned();

        let response = std::thread::spawn(move || {
            let mut mcp_client =
                McpClient::connect_stdio(&command, &args_vec).map_err(|e| e.to_string())?;
            mcp_client
                .call_tool(&tool, &args_val)
                .map_err(|e| e.to_string())
        })
        .join()
        .unwrap();

        // Record the flat-cost call
        self.usage.record_flat_call(server_name, ep);

        match response {
            Ok(result) => {
                if result.isError {
                    StepResult {
                        name: format!("[mcp] {server_name}:{tool_name}"),
                        status: StepStatus::Failed,
                        message: result.to_string(),
                    }
                } else {
                    StepResult {
                        name: format!("[mcp] {server_name}:{tool_name}"),
                        status: StepStatus::Passed,
                        message: result.to_string(),
                    }
                }
            }
            Err(e) => StepResult {
                name: format!("[mcp] {server_name}:{tool_name}"),
                status: StepStatus::Failed,
                message: format!("MCP call failed: {e}"),
            },
        }
    }

    // ── helpers ──────────────────────────────────────────────────────────

    /// Builds an `LlmConfig` from a resolved endpoint, falling back to
    /// the runner's default LLM config for any unset fields.
    fn build_llm_for_endpoint(&self, endpoint: &crate::endpoints::ResolvedEndpoint) -> LlmConfig {
        LlmConfig {
            url: if endpoint.provider == crate::scenario::Provider::Bedrock {
                // Bedrock builds its endpoint from the resolved AWS region
                // when no URL is given — never inherit the default LLM URL.
                endpoint.url.clone()
            } else if endpoint.url.is_empty() {
                self.llm.url.clone()
            } else {
                endpoint.url.clone()
            },
            model: endpoint
                .model
                .clone()
                .unwrap_or_else(|| self.llm.model.clone()),
            api_key: endpoint
                .api_key
                .clone()
                .or_else(|| self.llm.api_key.clone()),
            headers: if endpoint.headers.is_empty() {
                self.llm.headers.clone()
            } else {
                endpoint.headers.clone()
            },
            timeout: self.llm.timeout,
            temperature: self.llm.temperature,
            thinking: self.llm.thinking,
            model_params: self.llm.model_params.clone(),
            cache: endpoint.cache_markers,
            max_attempts: endpoint.max_attempts.max(1),
            provider: endpoint.provider,
            deployment: endpoint.deployment.clone(),
            api_version: endpoint.api_version.clone(),
            auth: endpoint.auth.clone(),
            header_commands: endpoint.header_commands.clone(),
            aws: endpoint.aws.clone(),
        }
    }

    /// Runs a single LLM call against an ordered endpoint chain (primary +
    /// fallbacks). Every endpoint gets its own `max_attempts` retry budget;
    /// the first endpoint that answers wins. Returns the response together
    /// with the chain index of the answering endpoint (0 = primary) so the
    /// caller can attribute usage to the correct endpoint.
    ///
    /// Emits an `LlmCallStarted`/`LlmCallFinished` event pair so the report
    /// shows duration, tokens, cost and the answering endpoint per call.
    /// Run-level context handed to every LLM call as the FIRST block of the
    /// user message. Contents that are stable for the whole run ("run
    /// started", "target site") come first so upstream provider prefix
    /// caching stays effective; the current time is the last line because
    /// it changes on every call.
    fn run_context(&self) -> String {
        let mut parts = vec![
            "RUN CONTEXT (use this to interpret the page, never repeat it back)".into(),
            "================================================================".into(),
            format!("Run started: {} UTC", unix_to_rfc3339(self.run_started)),
        ];
        if let Some(base) = self.config.base_url.as_deref() {
            parts.push(format!("Target site: {base}"));
        }
        let now = SystemTime::now()
            .duration_since(UNIX_EPOCH)
            .map_or(0, |d| d.as_secs());
        parts.push(format!("Current time: {} UTC", unix_to_rfc3339(now)));
        parts.join("\n")
    }

    #[allow(clippy::cast_possible_truncation, clippy::too_many_lines)]
    fn llm_call_chain(
        &self,
        chain: &[&ResolvedEndpoint],
        system: &str,
        user: &str,
        image: Option<&[String]>,
        purpose: &str,
    ) -> Result<(crate::costs::LlmResponse, usize), String> {
        if chain.is_empty() {
            return Err("empty LLM endpoint chain".into());
        }
        let primary = self.build_llm_for_endpoint(chain[0]);
        let fallbacks: Vec<LlmConfig> = chain[1..]
            .iter()
            .map(|e| self.build_llm_for_endpoint(e))
            .collect();

        let (test, index) = self
            .current_step
            .borrow()
            .as_ref()
            .map_or_else(|| ("-".to_owned(), 0), |(t, i)| (t.clone(), *i));
        let primary_endpoint = chain[0].name.clone();
        let primary_model = primary.model.clone();
        self.emit_event(&TestEvent::LlmCallStarted {
            test: test.clone(),
            index,
            endpoint: primary_endpoint.clone(),
            model: primary_model.clone(),
            purpose: purpose.to_owned(),
        });

        let started = Instant::now();
        let sys = system.to_owned();
        let context = self.run_context();
        let user = if context.is_empty() {
            user.to_owned()
        } else {
            format!("{context}\n\n{user}")
        };
        let image = image.map(<[String]>::to_vec);

        let result = std::thread::spawn(move || {
            let rt = tokio::runtime::Builder::new_current_thread()
                .enable_all()
                .build()
                .unwrap();
            let call = async {
                match image.as_deref() {
                    Some(img) => {
                        llm_chat_vision_with_usage_chain(
                            &primary,
                            &fallbacks,
                            &sys,
                            &user,
                            Some(img),
                        )
                        .await
                    }
                    None => llm_chat_with_usage_chain(&primary, &fallbacks, &sys, &user).await,
                }
            };
            rt.block_on(call)
        })
        .join()
        .unwrap();

        let duration_ms = started.elapsed().as_millis() as u64;
        match result {
            Ok((lr, idx)) => {
                let cost = calculate_llm_cost(chain[idx], &lr.usage);
                let answering = chain[idx].name.clone();
                let model = chain[idx]
                    .model
                    .clone()
                    .unwrap_or_else(|| primary_model.clone());
                self.usage
                    .record_llm_call(&answering, chain[idx], &model, &lr.usage);
                self.emit_event(&TestEvent::LlmCallFinished {
                    test,
                    index,
                    endpoint: answering,
                    model,
                    purpose: purpose.to_owned(),
                    ok: true,
                    duration_ms,
                    input_tokens: lr.usage.prompt_tokens,
                    output_tokens: lr.usage.completion_tokens,
                    cached_input_tokens: lr.usage.cached_input_tokens,
                    cache_creation_input_tokens: lr.usage.cache_creation_input_tokens,
                    cost,
                    error: None,
                });
                Ok((lr, idx))
            }
            Err(e) => {
                self.emit_event(&TestEvent::LlmCallFinished {
                    test,
                    index,
                    endpoint: primary_endpoint,
                    model: primary_model,
                    purpose: purpose.to_owned(),
                    ok: false,
                    duration_ms,
                    input_tokens: 0,
                    output_tokens: 0,
                    cached_input_tokens: 0,
                    cache_creation_input_tokens: 0,
                    cost: 0.0,
                    error: Some(e.clone()),
                });
                Err(e)
            }
        }
    }

    /// Resolves a CSS selector for the target element. Uses the explicit
    /// `selector` if provided, otherwise asks the LLM to find the element
    /// from the natural language `target` description and page DOM.
    ///
    /// LLM responses are sanitized and verified against the live page: a
    /// response that is not a selector (empty, `:not(*)`, `null`, …) fails
    /// immediately with the raw LLM output, and a selector that matches
    /// nothing triggers one retry with feedback before failing.
    #[allow(clippy::too_many_lines)]
    fn resolve_selector(
        &self,
        css_override: Option<&str>,
        target: &str,
        step_endpoint: Option<&str>,
        test_endpoint: Option<&str>,
        tab: &Tab,
    ) -> Result<String, String> {
        if let Some(explicit) = css_override {
            return Ok(explicit.to_owned());
        }

        let dom_info = extract_dom_info(tab)?;
        let page_content = get_page_text(tab);

        let system = concat!(
            "You are a browser automation selector generator. ",
            "Given a web page's content and interactive elements, ",
            "return ONLY the best CSS selector for the described element. ",
            "Output nothing except the CSS selector. ",
            "Prefer selectors in this order: #id, [data-testid=\"...\"], ",
            "[name=\"...\"], tag.class, tag. ",
            "Never output explanations, markdown, or extra text."
        );

        let user = format!(
            "Page URL: {}\nPage Title: {}\n\nPage body text (first 4000 chars):\n{}\n\nInteractive elements:\n{}\n\nFind the CSS selector for: {}",
            page_content.url,
            page_content.title,
            truncate(&page_content.body_text, 4000),
            dom_info,
            target,
        );

        let retry_user = format!(
            "Your previous answer was not usable. {}\n\nPage URL: {}\nPage Title: {}\n\nPage body text (first 4000 chars):\n{}\n\nInteractive elements:\n{}\n\nFind the CSS selector for: {}\nReturn ONLY a single CSS selector that matches an existing element. No explanations.",
            "The selector must match at least one element currently present on the page.",
            page_content.url,
            page_content.title,
            truncate(&page_content.body_text, 4000),
            dom_info,
            target,
        );

        self.reporter.debug(format!("LLM targeting: {target}"));

        let chain = self
            .endpoints
            .resolve_chain(step_endpoint.or(test_endpoint), TaskType::Targeting);
        let sys = system.to_owned();

        let call_llm = |prompt: &str| self.llm_call_chain(&chain, &sys, prompt, None, "targeting");

        let first = call_llm(&user);
        let (lr, _idx) = match first {
            Ok(lr) => lr,
            Err(e) => {
                return Err(format!("LLM element targeting failed: {e}"));
            }
        };
        let clean = sanitize_selector(&lr.content);
        self.reporter.debug(format!("resolved selector: {clean}"));

        if selector_is_useless(&clean) {
            return Err(format!(
                "LLM element targeting failed: the LLM did not return a usable selector for {target:?} (got {raw:?}). Check the page state in the diagnostics above.",
                raw = lr.content.trim(),
            ));
        }
        if let Err(reason) = validate_selector(&clean) {
            return Err(format!(
                "LLM element targeting failed: invalid selector for {target:?}: {reason} (LLM response: {raw:?})",
                raw = lr.content.trim(),
            ));
        }
        if !selector_matches(tab, &clean).unwrap_or(false) {
            // One retry with feedback: flaky models occasionally invent a
            // selector that does not exist on the page.
            self.reporter.warn(format!(
                "selector {clean} matches nothing — retrying LLM targeting with feedback"
            ));
            let second = call_llm(&retry_user);
            let (lr2, _idx2) = match second {
                Ok(lr2) => lr2,
                Err(e) => {
                    return Err(format!(
                        "LLM element targeting failed: first answer {clean:?} matched nothing, retry also failed: {e}"
                    ));
                }
            };
            let clean2 = sanitize_selector(&lr2.content);
            self.reporter
                .debug(format!("resolved selector (retry): {clean2}"));
            if selector_is_useless(&clean2) {
                return Err(format!(
                    "LLM element targeting failed: selector {clean:?} matched nothing; the retry returned no usable selector for {target:?} (got {raw:?}). Page excerpt: {excerpt}",
                    raw = lr2.content.trim(),
                    excerpt = truncate(&page_content.body_text, 300),
                ));
            }
            if !selector_matches(tab, &clean2).unwrap_or(false) {
                return Err(format!(
                    "LLM element targeting failed: selector {clean2:?} does not match any element on the page for {target:?}. Verify the page state in the diagnostics; the login/SPA may not have rendered."
                ));
            }
            return Ok(clean2);
        }

        Ok(clean)
    }
}

/// Evaluates a JS expression that is expected to return a boolean.
fn eval_bool(tab: &Tab, js: &str) -> Result<bool, String> {
    tab.evaluate(js, false)
        .map_err(|e| format!("evaluate failed: {e}"))?
        .value
        .and_then(|v| v.as_bool())
        .ok_or_else(|| "evaluate returned non-boolean".to_owned())
}

/// Checks whether a CSS selector matches at least one current element.
fn selector_matches(tab: &Tab, selector: &str) -> Result<bool, String> {
    eval_bool(tab, &crate::selectors::selector_matches_js(selector))
}

// ── Free helper functions ──────────────────────────────────────────────

fn run_navigate_step(full_url: &str, tab: &Tab) -> StepResult {
    let name = format!("[navigate] {full_url}");
    match tab.navigate_to(full_url) {
        Ok(_) => {
            let _ = tab.wait_until_navigated();
            StepResult {
                name,
                status: StepStatus::Passed,
                message: format!("navigated to {full_url}"),
            }
        }
        Err(e) => StepResult {
            name,
            status: StepStatus::Failed,
            message: format!("navigation failed: {e}"),
        },
    }
}

fn extract_dom_info(tab: &Tab) -> Result<String, String> {
    let result = tab
        .evaluate(DOM_EXTRACT_JS, false)
        .map_err(|e| format!("DOM extraction failed: {e}"))?;

    let json_str = result
        .value
        .as_ref()
        .and_then(|v| v.as_str())
        .unwrap_or("[]");

    let elements: Vec<String> = serde_json::from_str(json_str).unwrap_or_default();

    if elements.is_empty() {
        return Ok("(no interactive elements found)".to_owned());
    }

    Ok(elements.join("\n"))
}

fn get_page_text(tab: &Tab) -> PageContent {
    let url = tab.get_url();

    let title = tab
        .evaluate("document.title", false)
        .ok()
        .and_then(|r| r.value)
        .and_then(|v| v.as_str().map(String::from))
        .unwrap_or_else(|| "unknown".to_owned());

    let body_text = tab
        .evaluate(
            "document.body ? document.body.innerText : document.documentElement.innerText",
            false,
        )
        .ok()
        .and_then(|r| r.value)
        .and_then(|v| v.as_str().map(String::from))
        .unwrap_or_default();

    PageContent {
        url,
        title,
        body_text: truncate(&body_text, 8000),
    }
}

fn resolve_url(url: &str, base_url: &str) -> String {
    if url.starts_with("http://") || url.starts_with("https://") {
        return url.to_owned();
    }
    let base = base_url.trim_end_matches('/');
    if url.starts_with('/') {
        format!("{base}{url}")
    } else {
        format!("{base}/{url}")
    }
}

/// Origin (`scheme://host[:port]`) of a base URL, used to scope the
/// per-test `Storage.clearDataForOrigin` call. Returns `None` when the
/// URL has no recognizable scheme/host (the clear is skipped).
#[must_use]
fn origin_of(base_url: &str) -> Option<String> {
    let url = if base_url.contains("://") {
        base_url.to_owned()
    } else {
        format!("https://{base_url}")
    };
    let (scheme, rest) = url.split_once("://")?;
    let authority = rest
        .split(['/', '?', '#'])
        .next()
        .filter(|a| !a.is_empty())?;
    Some(format!("{scheme}://{authority}"))
}

/// True when a test performs its own login: its first navigate step
/// targets a login route (org `/auth/login`, tenant `/tenant/login`),
/// or it has no navigate step at all and rides the auto-navigate onto a
/// login `start_url`. Such tests need a cleared session — with a live one
/// the SPA bounces the login page before the form ever mounts. Tests
/// whose own first navigate goes to an app page keep the shared session
/// (most "page loads" tests rely on it) — a blanket clear broke exactly
/// those on immosai run #1643.
#[must_use]
fn test_targets_login(start_url: &str, steps: &[TestStep]) -> bool {
    for step in steps {
        if let TestStep::Navigate { url, .. } = step {
            return url.contains("login");
        }
    }
    start_url.contains("login")
}

/// Human-readable label for a step, used when steps are skipped after an
/// earlier failure.
fn step_label(step: &TestStep) -> String {
    match step {
        TestStep::Navigate { url, .. } => format!("[navigate] {url}"),
        TestStep::Click { target, .. } => format!("[click] {target}"),
        TestStep::Type { target, .. } => format!("[type] {target}"),
        TestStep::Wait { target, .. } => format!("[wait] {target}"),
        TestStep::Assert {
            definition,
            preset,
            prompt,
            ..
        } => definition.as_ref().map_or_else(
            || {
                preset.as_ref().map_or_else(
                    || {
                        prompt.as_ref().map_or_else(
                            || "[assert]".to_owned(),
                            |pr| format!("[assert] custom ({})", truncate(pr, 60)),
                        )
                    },
                    |p| format!("[assert] {p}"),
                )
            },
            |d| format!("[assert] {d}"),
        ),
        TestStep::Screenshot { .. } => "[screenshot]".to_owned(),
        TestStep::Agent { agent, .. } => format!("[agent] {agent}"),
        TestStep::Mcp { server, tool, .. } => format!("[mcp] {server}:{tool}"),
    }
}

/// Short kind word for artifact file names (e.g. `click`, `wait`, `assert`).
#[must_use]
const fn step_kind_label(step: &TestStep) -> &'static str {
    match step {
        TestStep::Navigate { .. } => "navigate",
        TestStep::Click { .. } => "click",
        TestStep::Type { .. } => "type",
        TestStep::Wait { .. } => "wait",
        TestStep::Assert { .. } => "assert",
        TestStep::Screenshot { .. } => "screenshot",
        TestStep::Agent { .. } => "agent",
        TestStep::Mcp { .. } => "mcp",
    }
}

// ── Support types ──────────────────────────────────────────────────────

#[derive(Default)]
struct TestRunResult {
    passed: u32,
    failed: u32,
    skipped: u32,
    total: u32,
    details: Vec<StepResult>,
}

struct PageContent {
    url: String,
    title: String,
    body_text: String,
}

#[cfg(test)]
mod tests {
    use super::unix_to_rfc3339;

    #[test]
    fn rfc3339_epoch_and_reference_dates() {
        assert_eq!(unix_to_rfc3339(0), "1970-01-01T00:00:00Z");
        assert_eq!(unix_to_rfc3339(1_736_840_000), "2025-01-14T07:33:20Z");
        assert_eq!(unix_to_rfc3339(1_784_469_000), "2026-07-19T13:50:00Z");
        assert_eq!(unix_to_rfc3339(9_999_999_999), "2286-11-20T17:46:39Z");
    }

    #[test]
    fn rfc3339_handles_leap_years() {
        assert_eq!(unix_to_rfc3339(1_582_905_600), "2020-02-28T16:00:00Z");
        assert_eq!(unix_to_rfc3339(1_707_408_000), "2024-02-08T16:00:00Z");
    }
}

#[cfg(test)]
mod verdict_parse_tests {
    use super::{verdict_is_fail, verdict_is_pass};

    #[test]
    fn tolerates_markdown_punctuation_and_natural_language() {
        assert!(verdict_is_pass("PASS"));
        assert!(verdict_is_pass("pass"));
        assert!(verdict_is_pass("**PASS**\n\nThe page is clean."));
        assert!(verdict_is_pass("passes - no explicit error visible"));
        assert!(verdict_is_pass("  \"pass\""));
        assert!(!verdict_is_pass("FAIL: something broke"));
        assert!(!verdict_is_pass("**FAIL** broken"));

        assert!(verdict_is_fail("**FAIL** broken"));
        assert!(verdict_is_fail("fails - error toast shown"));
        assert!(!verdict_is_fail("passes - ok"));
    }
}