llm_browser_testkit/

runner.rs

use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::{Duration, Instant};

use anyhow::Context;
use headless_chrome::{Browser, LaunchOptions, Tab};

use crate::a2a::A2aClient;
use crate::budgets::{BudgetStatus, BudgetTracker};
use crate::costs::UsageTracker;
use crate::diagnostics;
use crate::endpoints::{EndpointRegistry, TaskType};
use crate::llm_chat_with_usage;
use crate::mcp_client::McpClient;
use crate::scenario::{AssertDefinition, ScenarioConfig, TestGroup, TestStep};
use crate::selectors::{sanitize_selector, selector_is_useless, validate_selector};
use crate::truncate;
use crate::LlmConfig;
use crate::DOM_EXTRACT_JS;

/// How long the CDP connection stays open after the browser goes quiet.
///
/// `headless_chrome` ships a 30s default and tears down the entire connection
/// when no traffic arrives for that long; a run must own its connection for
/// its full duration instead.
const BROWSER_IDLE_TIMEOUT: Duration = Duration::from_secs(6 * 60 * 60);

/// Executes a [`Scenario`] against a real browser with optional LLM
/// assistance for element targeting and assertions.
pub struct ScenarioRunner {
    config: ScenarioConfig,
    definitions: HashMap<String, AssertDefinition>,
    llm: LlmConfig,
    timeout: Duration,
    viewport_width: u32,
    viewport_height: u32,
    endpoints: EndpointRegistry,
    usage: Arc<UsageTracker>,
    budgets: BudgetTracker,
    /// Directory for failure artifacts (screenshots).
    artifacts_dir: PathBuf,
}

/// Aggregated results from a scenario run.
#[derive(Debug, Default)]
pub struct RunReport {
    /// Number of tests that passed.
    pub tests_passed: u32,
    /// Number of tests that failed.
    pub tests_failed: u32,
    /// Number of steps that passed.
    pub passed: u32,
    /// Number of steps that failed.
    pub failed: u32,
    /// Number of steps that were skipped.
    pub skipped: u32,
    /// Per-step details.
    pub details: Vec<StepResult>,
}

/// Result of a single step execution.
#[derive(Debug)]
pub struct StepResult {
    /// The step name.
    pub name: String,
    /// Whether the step passed, failed, or was skipped.
    pub status: StepStatus,
    /// Human-readable result message.
    pub message: String,
}

/// Outcome for a single step.
#[derive(Debug, PartialEq, Eq)]
pub enum StepStatus {
    /// Step executed successfully and all assertions passed.
    Passed,
    /// Step execution or assertion failed.
    Failed,
    /// Step was skipped.
    Skipped,
}

/// Predefined assertion preset definition.
struct AssertPreset {
    name: &'static str,
    system: &'static str,
    user_template: &'static str,
}

/// Built-in assertion presets.
#[allow(clippy::literal_string_with_formatting_args)]
const ASSERTION_PRESETS: &[AssertPreset] = &[
    AssertPreset {
        name: "no_error_on_page",
        system: "You are a QA tester. Evaluate if a web page contains error messages, stack traces, exception text, HTTP error codes, 'undefined' errors, or any indication of a malfunction. Be strict — even minor rendering glitches count as errors.",
        user_template: "Check if the following page content contains ANY errors or malfunctions:\n\nURL: {url}\nTitle: {title}\n\nPage Content:\n{content}\n\nRespond with exactly \"PASS\" if there are NO errors, or \"FAIL: <reason>\" if there are errors. Only respond with PASS or FAIL.",
    },
    AssertPreset {
        name: "text_visible",
        system: "You are a QA tester. Your task is to check if specific text is visible in the page content.",
        user_template: "Check if the following text appears in the page content:\n\nTEXT TO FIND: \"{expected_text}\"\n\nURL: {url}\n\nPage Content:\n{content}\n\nRespond with exactly \"PASS\" if the text is present (even partial match is OK), or \"FAIL: text not found\" if it is not.",
    },
    AssertPreset {
        name: "element_exists",
        system: "You are a QA tester. Check if a described UI element exists on a web page.",
        user_template: "Check if the following element exists on the page:\n\nELEMENT: \"{description}\"\n\nURL: {url}\n\nPage Content:\n{content}\n\nRespond with exactly \"PASS\" if the element exists, or \"FAIL: <reason>\" if it does not.",
    },
];

impl ScenarioRunner {
    /// Creates a new runner with the given scenario configuration and
    /// assertion definitions.
    #[must_use]
    #[allow(clippy::needless_pass_by_value)]
    pub fn new(scenario_config: ScenarioConfig, definitions: Vec<AssertDefinition>) -> Self {
        let llm = LlmConfig {
            url: scenario_config
                .llm_url
                .clone()
                .unwrap_or_else(crate::llm_base_url),
            model: scenario_config
                .llm_model
                .clone()
                .unwrap_or_else(crate::llm_model),
            api_key: scenario_config
                .llm_api_key
                .clone()
                .or_else(|| std::env::var("HARNESS_LLM_API_KEY").ok()),
            headers: if scenario_config.llm_headers.is_empty() {
                crate::parse_headers_env()
            } else {
                scenario_config.llm_headers.clone()
            },
            timeout: Duration::from_secs(scenario_config.timeout_secs.unwrap_or(60)),
            temperature: scenario_config.temperature,
            thinking: scenario_config.thinking,
            model_params: scenario_config.model_params.clone(),
        };
        let endpoints = EndpointRegistry::from_config(&scenario_config.endpoints, Some(&llm));
        let budgets = BudgetTracker::from_config(&scenario_config.budgets);
        let defs_map: HashMap<String, AssertDefinition> = definitions
            .into_iter()
            .map(|d| (d.name.clone(), d))
            .collect();

        Self {
            timeout: Duration::from_secs(scenario_config.timeout_secs.unwrap_or(60)),
            viewport_width: scenario_config.viewport_width.unwrap_or(1280),
            viewport_height: scenario_config.viewport_height.unwrap_or(720),
            config: scenario_config.clone(),
            definitions: defs_map,
            llm,
            endpoints,
            usage: Arc::new(UsageTracker::new()),
            budgets,
            artifacts_dir: PathBuf::from(
                scenario_config
                    .artifacts_dir
                    .clone()
                    .unwrap_or_else(|| "artifacts".to_owned()),
            ),
        }
    }

    /// Returns a clone of the [`UsageTracker`] for reporting.
    #[must_use]
    pub fn usage_tracker(&self) -> Arc<UsageTracker> {
        Arc::clone(&self.usage)
    }

    /// Returns a reference to the [`BudgetTracker`].
    #[must_use]
    pub const fn budget_tracker(&self) -> &BudgetTracker {
        &self.budgets
    }

    /// Executes all test groups in the scenario and returns a report.
    ///
    /// # Errors
    ///
    /// Returns an error if the browser fails to launch.
    #[allow(clippy::too_many_lines)]
    pub fn run(&self, tests: &[TestGroup]) -> anyhow::Result<RunReport> {
        let mut report = RunReport::default();

        if tests.is_empty() {
            eprintln!("No tests defined in scenario.");
            return Ok(report);
        }

        let browser_headless = self.config.browser_headless.unwrap_or(true);

        let launch_opts = LaunchOptions {
            headless: browser_headless,
            window_size: Some((self.viewport_width, self.viewport_height)),
            sandbox: false,
            // headless_chrome defaults this to 30s and shuts down the whole CDP
            // connection when no messages arrive for that long. A scenario can
            // easily exceed 30s of browser silence (slow LLM targeting/assertion
            // calls, page waits, budget checks between steps), after which every
            // remaining step fails with "Unable to make method calls because
            // underlying connection is closed" — one quiet gap kills the run.
            // Open-ended scenarios must own the connection for their full
            // duration, so keep it alive for 6 hours.
            idle_browser_timeout: BROWSER_IDLE_TIMEOUT,
            ..LaunchOptions::default()
        };

        let browser = Browser::new(launch_opts).context("failed to launch browser")?;
        let tab = browser.new_tab().context("failed to open browser tab")?;
        let _ = tab.set_default_timeout(self.timeout);

        // Start MCP server if configured
        #[cfg(feature = "mcp-server")]
        if let Some(ref mcp_cfg) = self.config.mcp_server {
            if mcp_cfg.enabled {
                let port = mcp_cfg.port;
                std::thread::spawn(move || {
                    let _ = crate::mcp_server::start_mcp_server(port);
                });
            }
        }
        #[cfg(not(feature = "mcp-server"))]
        if let Some(mcp_cfg) = &self.config.mcp_server {
            if mcp_cfg.enabled {
                eprintln!("  ⚠️  MCP server configured but 'mcp-server' feature not enabled");
            }
        }

        // Start A2A agent server if configured
        #[cfg(feature = "a2a-server")]
        if let Some(ref a2a_cfg) = self.config.a2a_server {
            if a2a_cfg.enabled {
                let port = a2a_cfg.port;
                tokio::spawn(crate::a2a_server::start_a2a_server(port));
            }
        }
        #[cfg(not(feature = "a2a-server"))]
        if let Some(a2a_cfg) = &self.config.a2a_server {
            if a2a_cfg.enabled {
                eprintln!("  ⚠️  A2A server configured but 'a2a-server' feature not enabled");
            }
        }

        for test in tests {
            eprintln!("\n╔══════════════════════════════");
            eprintln!("║  Test: {}", test.name);
            eprintln!("╚══════════════════════════════");

            self.usage.reset_per_test();

            let test_result = self.run_test(test, &tab);
            self.usage.commit_test(&test.name);

            if test_result.failed == 0 && test_result.total > 0 {
                report.tests_passed += 1;
                eprintln!("  Test ✅ Passed");
            } else if test_result.total > 0 {
                report.tests_failed += 1;
                eprintln!("  Test ❌ Failed");
            }

            report.passed += test_result.passed;
            report.failed += test_result.failed;
            report.skipped += test_result.skipped;
            report.details.extend(test_result.details);
        }

        Ok(report)
    }

    #[allow(clippy::too_many_lines)]
    fn run_test(&self, test: &TestGroup, tab: &Tab) -> TestRunResult {
        let base_url = test
            .base_url
            .clone()
            .or_else(|| self.config.base_url.clone())
            .unwrap_or_else(crate::base_url);

        let auto_navigate = test.auto_navigate.unwrap_or(self.config.auto_navigate);

        let start_url = test
            .start_url
            .clone()
            .or_else(|| self.config.start_url.clone())
            .unwrap_or_else(|| "/dashboard".to_owned());

        if auto_navigate {
            let full_url = resolve_url(&start_url, &base_url);
            eprintln!("  → auto-navigate: {full_url}");
            let _ = tab.navigate_to(&full_url);
            let _ = tab.wait_until_navigated();
            std::thread::sleep(Duration::from_secs(4));
        }

        let mut result = TestRunResult::default();

        for (step_index, step) in test.steps.iter().enumerate() {
            result.total += 1;

            let wait_ms = match step {
                TestStep::Navigate { wait_after_ms, .. }
                | TestStep::Click { wait_after_ms, .. }
                | TestStep::Type { wait_after_ms, .. } => *wait_after_ms,
                _ => None,
            };

            let mut step_result = match step {
                TestStep::Navigate { url, .. } => {
                    let full_url = resolve_url(url, &base_url);
                    run_navigate_step(&full_url, tab)
                }
                TestStep::Click {
                    target,
                    selector,
                    endpoint,
                    ..
                } => self.run_click(
                    target,
                    selector.as_deref(),
                    endpoint.as_deref(),
                    test.endpoint.as_deref(),
                    tab,
                ),
                TestStep::Type {
                    target,
                    text,
                    selector,
                    endpoint,
                    ..
                } => self.run_type(
                    target,
                    text,
                    selector.as_deref(),
                    endpoint.as_deref(),
                    test.endpoint.as_deref(),
                    tab,
                ),
                TestStep::Wait {
                    target,
                    selector,
                    text,
                    timeout_ms,
                    endpoint,
                } => self.run_wait(
                    target,
                    selector.as_deref(),
                    text.as_deref(),
                    *timeout_ms,
                    endpoint.as_deref(),
                    test.endpoint.as_deref(),
                    tab,
                ),
                TestStep::Assert {
                    definition,
                    preset,
                    prompt,
                    assert_text,
                    endpoint,
                } => self.run_assert(
                    definition.as_deref(),
                    preset.as_deref(),
                    prompt.as_deref(),
                    assert_text.as_deref(),
                    endpoint.as_deref(),
                    test.endpoint.as_deref(),
                    tab,
                ),
                TestStep::Screenshot { path } => Self::run_screenshot(path.as_deref(), tab),
                TestStep::Agent {
                    agent,
                    task,
                    definition,
                } => self.run_agent(agent, task, definition.as_deref(), test.endpoint.as_deref()),
                TestStep::Mcp { server, tool, args } => self.run_mcp(server, tool, args.as_ref()),
            };

            // Failure diagnostics: capture the page state and a screenshot so
            // CI logs say WHAT the page looked like when the step failed,
            // instead of a bare "timed out: The event waited for never came".
            if step_result.status == StepStatus::Failed {
                let state = diagnostics::capture(tab);
                let screenshot = diagnostics::save_screenshot(
                    tab,
                    &self.artifacts_dir,
                    &test.name,
                    &test.name,
                    step_index,
                    &step_kind_label(step),
                );
                step_result.message = format!(
                    "{base} — {excerpt}",
                    base = step_result.message,
                    excerpt = diagnostics::inline_excerpt(&state),
                );
                eprintln!(
                    "{}",
                    diagnostics::full_context(&state, screenshot.as_deref())
                );
            }

            eprintln!(
                "    {} {} — {}",
                if step_result.status == StepStatus::Passed {
                    "✅"
                } else if step_result.status == StepStatus::Failed {
                    "❌"
                } else {
                    "⏭️"
                },
                step_result.name,
                step_result.message,
            );

            match step_result.status {
                StepStatus::Passed => result.passed += 1,
                StepStatus::Failed => result.failed += 1,
                StepStatus::Skipped => result.skipped += 1,
            }

            // Fail fast: the first failed step ends the test and the
            // remaining steps are reported as skipped (no LLM budget is
            // burned asserting against a page that is already known broken).
            if step_result.status == StepStatus::Failed
                && !self.config.continue_on_failure
                && step_index + 1 < test.steps.len()
            {
                eprintln!(
                    "      ⏭️  {}",
                    format!(
                        "failing fast: {} remaining step(s) skipped (set continue_on_failure = true in [config] to disable)",
                        test.steps.len() - step_index - 1
                    )
                );
                for skipped in &test.steps[step_index + 1..] {
                    result.total += 1;
                    result.skipped += 1;
                    eprintln!(
                        "    ⏭️  {} — skipped: previous step failed",
                        step_label(skipped)
                    );
                    result.details.push(StepResult {
                        name: step_label(skipped),
                        status: StepStatus::Skipped,
                        message: "skipped: previous step failed".into(),
                    });
                }
                result.details.push(step_result);
                return result;
            }

            // Check per-test budget after each step
            let test_usage = self.usage.current_test_snapshot();
            let global_usage = self.usage.global_snapshot();
            let budget_status = self.budgets.check_all(
                &test.name,
                &test_usage,
                &global_usage,
                test.budget.as_ref(),
            );
            match budget_status {
                BudgetStatus::HardExceeded { message, .. } => {
                    crate::reporting::print_budget_error(&message);
                    result.details.push(StepResult {
                        name: "[budget]".into(),
                        status: StepStatus::Failed,
                        message,
                    });
                    result.failed += 1;
                    return result;
                }
                BudgetStatus::SoftExceeded { message, .. } => {
                    crate::reporting::print_budget_warning(&message);
                }
                BudgetStatus::Ok => {}
            }

            if let Some(ms) = wait_ms {
                std::thread::sleep(Duration::from_millis(ms));
            }

            result.details.push(step_result);
        }

        result
    }

    // ── step handlers ───────────────────────────────────────────────────

    fn run_click(
        &self,
        target: &str,
        selector_override: Option<&str>,
        step_endpoint: Option<&str>,
        test_endpoint: Option<&str>,
        tab: &Tab,
    ) -> StepResult {
        let selector = match self.resolve_selector(
            selector_override,
            target,
            step_endpoint,
            test_endpoint,
            tab,
        ) {
            Ok(s) => s,
            Err(msg) => {
                return StepResult {
                    name: format!("[click] {target}"),
                    status: StepStatus::Failed,
                    message: msg,
                };
            }
        };

        match tab.wait_for_element_with_custom_timeout(&selector, Duration::from_secs(10)) {
            Ok(element) => match element.click() {
                Ok(_) => StepResult {
                    name: format!("[click] {target}"),
                    status: StepStatus::Passed,
                    message: format!("clicked {selector}"),
                },
                Err(e) => StepResult {
                    name: format!("[click] {target}"),
                    status: StepStatus::Failed,
                    message: format!("click failed on {selector}: {e}"),
                },
            },
            Err(e) => StepResult {
                name: format!("[click] {target}"),
                status: StepStatus::Failed,
                message: format!("element {selector} not found: {e}"),
            },
        }
    }

    #[allow(clippy::too_many_arguments)]
    fn run_type(
        &self,
        target: &str,
        text: &str,
        selector_override: Option<&str>,
        step_endpoint: Option<&str>,
        test_endpoint: Option<&str>,
        tab: &Tab,
    ) -> StepResult {
        let selector = match self.resolve_selector(
            selector_override,
            target,
            step_endpoint,
            test_endpoint,
            tab,
        ) {
            Ok(s) => s,
            Err(msg) => {
                return StepResult {
                    name: format!("[type] {target}"),
                    status: StepStatus::Failed,
                    message: msg,
                };
            }
        };

        match tab.wait_for_element_with_custom_timeout(&selector, Duration::from_secs(10)) {
            Ok(element) => {
                if let Err(e) = element.click() {
                    return StepResult {
                        name: format!("[type] {target}"),
                        status: StepStatus::Failed,
                        message: format!("click to focus {selector} failed: {e}"),
                    };
                }

                let js = format!(
                    "document.querySelector('{}').value = '';",
                    selector.replace('\'', "\\'")
                );
                let _ = tab.evaluate(&js, false);

                match element.type_into(text) {
                    Ok(_) => StepResult {
                        name: format!("[type] {target}"),
                        status: StepStatus::Passed,
                        message: format!("typed {text:?} into {selector}"),
                    },
                    Err(e) => StepResult {
                        name: format!("[type] {target}"),
                        status: StepStatus::Failed,
                        message: format!("type into {selector} failed: {e}"),
                    },
                }
            }
            Err(e) => StepResult {
                name: format!("[type] {target}"),
                status: StepStatus::Failed,
                message: format!("element {selector} not found: {e}"),
            },
        }
    }

    #[allow(clippy::too_many_arguments)]
    fn run_wait(
        &self,
        target: &str,
        selector_override: Option<&str>,
        text: Option<&str>,
        timeout_ms: Option<u64>,
        step_endpoint: Option<&str>,
        test_endpoint: Option<&str>,
        tab: &Tab,
    ) -> StepResult {
        let timeout = Duration::from_millis(timeout_ms.unwrap_or(10_000));
        let step_name = format!("[wait] {target}");

        // Resolve an explicit selector only (text-only waits are LLM-free).
        let selector = match selector_override {
            Some(s) => Some(s.to_owned()),
            None if text.is_some() => None,
            None => match self.resolve_selector(None, target, step_endpoint, test_endpoint, tab) {
                Ok(s) => Some(s),
                Err(msg) => {
                    return StepResult {
                        name: step_name,
                        status: StepStatus::Failed,
                        message: msg,
                    };
                }
            },
        };

        if text.is_some() {
            let sel_js = selector
                .as_deref()
                .map(crate::selectors::selector_matches_js);
            let text_js = text.map(|t| {
                let escaped = t.replace('\\', "\\\\").replace('\'', "\\'");
                format!("document.body ? document.body.innerText.includes('{escaped}') : false")
            });

            let deadline = Instant::now() + timeout;
            loop {
                let sel_ok = match &sel_js {
                    Some(js) => eval_bool(tab, js).unwrap_or(false),
                    None => true,
                };
                let text_ok = match &text_js {
                    Some(js) => eval_bool(tab, js).unwrap_or(false),
                    None => true,
                };
                if sel_ok && text_ok {
                    let mut what = Vec::new();
                    if let Some(sel) = &selector {
                        what.push(format!("found {sel}"));
                    }
                    if let Some(t) = text {
                        what.push(format!("text {t:?} visible"));
                    }
                    return StepResult {
                        name: step_name,
                        status: StepStatus::Passed,
                        message: what.join(" and "),
                    };
                }
                if Instant::now() >= deadline {
                    let mut what = Vec::new();
                    if let Some(sel) = &selector {
                        what.push(format!("{sel}"));
                    }
                    if let Some(t) = text {
                        what.push(format!("text {t:?}"));
                    }
                    return StepResult {
                        name: step_name,
                        status: StepStatus::Failed,
                        message: format!(
                            "wait for {} timed out after {}ms: the event waited for never came",
                            what.join(" / "),
                            timeout.as_millis(),
                        ),
                    };
                }
                std::thread::sleep(Duration::from_millis(250));
            }
        }

        match selector.as_deref() {
            Some(sel) => match tab.wait_for_element_with_custom_timeout(sel, timeout) {
                Ok(_) => StepResult {
                    name: step_name,
                    status: StepStatus::Passed,
                    message: format!("found {sel}"),
                },
                Err(e) => StepResult {
                    name: step_name,
                    status: StepStatus::Failed,
                    message: format!(
                        "wait for {sel} timed out after {}ms: {e}",
                        timeout.as_millis()
                    ),
                },
            },
            None => StepResult {
                name: step_name,
                status: StepStatus::Failed,
                message: "wait step has neither selector nor text".into(),
            },
        }
    }

    #[allow(clippy::too_many_arguments)]
    fn run_assert(
        &self,
        definition: Option<&str>,
        preset: Option<&str>,
        prompt: Option<&str>,
        assert_text: Option<&str>,
        step_endpoint: Option<&str>,
        test_endpoint: Option<&str>,
        tab: &Tab,
    ) -> StepResult {
        std::thread::sleep(Duration::from_millis(500));

        let page_content = get_page_text(tab);

        if let Some(def_name) = definition {
            if let Some(def) = self.definitions.get(def_name) {
                return self.run_assert_def(def, &page_content, step_endpoint, test_endpoint);
            }
            return StepResult {
                name: format!("[assert] {def_name}"),
                status: StepStatus::Failed,
                message: format!("definition '{def_name}' not found"),
            };
        }

        if let Some(preset_name) = preset {
            return self.run_preset(
                preset_name,
                assert_text,
                &page_content,
                step_endpoint,
                test_endpoint,
            );
        }

        if let Some(prompt_text) = prompt {
            return self.run_custom(prompt_text, &page_content, step_endpoint, test_endpoint);
        }

        StepResult {
            name: "[assert]".into(),
            status: StepStatus::Skipped,
            message: "no definition, preset, or prompt specified".into(),
        }
    }

    fn run_assert_def(
        &self,
        def: &AssertDefinition,
        page_content: &PageContent,
        step_endpoint: Option<&str>,
        test_endpoint: Option<&str>,
    ) -> StepResult {
        // Agent-based definition: delegate to an A2A agent
        if let Some(ref agent) = def.agent {
            let task = def
                .task_template
                .as_deref()
                .unwrap_or("Evaluate the assertion")
                .replace("{url}", &page_content.url)
                .replace("{title}", &page_content.title)
                .replace("{content}", &page_content.body_text)
                .replace("{expected_text}", def.assert_text.as_deref().unwrap_or(""));

            return self.run_agent_step(agent, &task, &def.name);
        }

        // Custom preset: system + user_template provided in the definition
        if let (Some(system), Some(template)) = (&def.system, &def.user_template) {
            return self.run_custom_preset(
                &def.name,
                system,
                template,
                def.assert_text.as_deref(),
                page_content,
                step_endpoint,
                test_endpoint,
            );
        }

        def.preset.as_ref().map_or_else(
            || {
                def.prompt.as_ref().map_or_else(
                    || StepResult {
                        name: format!("[assert] {}", def.name),
                        status: StepStatus::Failed,
                        message: "definition has no preset, prompt, or system+user_template".into(),
                    },
                    |prompt| self.run_custom(prompt, page_content, step_endpoint, test_endpoint),
                )
            },
            |preset_name| {
                self.run_preset(
                    preset_name,
                    def.assert_text.as_deref(),
                    page_content,
                    step_endpoint,
                    test_endpoint,
                )
            },
        )
    }

    #[allow(clippy::too_many_arguments)]
    fn run_custom_preset(
        &self,
        name: &str,
        system: &str,
        template: &str,
        assert_text: Option<&str>,
        page_content: &PageContent,
        step_endpoint: Option<&str>,
        test_endpoint: Option<&str>,
    ) -> StepResult {
        let user_prompt = template
            .replace("{url}", &page_content.url)
            .replace("{title}", &page_content.title)
            .replace("{content}", &page_content.body_text)
            .replace("{expected_text}", assert_text.unwrap_or(""))
            .replace("{description}", "");

        // Custom preset definitions frequently forget the {content}
        // placeholder — without it the LLM has no page to evaluate and
        // answers "I can't determine that without seeing the page". Always
        // append the page context unless the template already references it.
        let user_prompt = if template.contains("{content}") {
            user_prompt
        } else {
            format!(
                "{user_prompt}\n\nPage URL: {url}\nPage Title: {title}\n\nPage Content:\n{content}",
                url = page_content.url,
                title = page_content.title,
                content = page_content.body_text,
            )
        };

        eprintln!("      assert: {name} (custom preset)");

        let endpoint = self
            .endpoints
            .resolve(step_endpoint.or(test_endpoint), TaskType::Assertion);
        let llm = self.build_llm_for_endpoint(endpoint);
        let usage = Arc::clone(&self.usage);
        let endpoint_name = endpoint.name.clone();
        let sys = system.to_owned();

        let response = std::thread::spawn(move || {
            let rt = tokio::runtime::Builder::new_current_thread()
                .enable_all()
                .build()
                .unwrap();
            rt.block_on(llm_chat_with_usage(&llm, &sys, &user_prompt))
        })
        .join()
        .unwrap();

        response.map_or_else(
            |e| StepResult {
                name: format!("[assert] {name}"),
                status: StepStatus::Failed,
                message: format!("LLM assertion call failed: {e}"),
            },
            |lr| {
                usage.record_llm_call(
                    &endpoint_name,
                    endpoint,
                    lr.usage.prompt_tokens,
                    lr.usage.completion_tokens,
                );
                let content_lower = lr.content.to_lowercase().trim().to_owned();
                if content_lower.starts_with("pass") {
                    StepResult {
                        name: format!("[assert] {name}"),
                        status: StepStatus::Passed,
                        message: "PASS".into(),
                    }
                } else {
                    StepResult {
                        name: format!("[assert] {name}"),
                        status: StepStatus::Failed,
                        message: lr.content,
                    }
                }
            },
        )
    }

    fn run_preset(
        &self,
        preset_name: &str,
        assert_text: Option<&str>,
        page_content: &PageContent,
        step_endpoint: Option<&str>,
        test_endpoint: Option<&str>,
    ) -> StepResult {
        let Some(preset) = ASSERTION_PRESETS.iter().find(|p| p.name == preset_name) else {
            return StepResult {
                name: format!("[assert] {preset_name}"),
                status: StepStatus::Failed,
                message: format!("unknown assertion preset: {preset_name}"),
            };
        };

        let user_prompt = preset
            .user_template
            .replace("{url}", &page_content.url)
            .replace("{title}", &page_content.title)
            .replace("{content}", &page_content.body_text)
            .replace("{expected_text}", assert_text.unwrap_or(""))
            .replace("{description}", "");

        // Same safety net as custom presets: never let the LLM answer with
        // no page context at all.
        let user_prompt = if preset.user_template.contains("{content}") {
            user_prompt
        } else {
            format!(
                "{user_prompt}\n\nPage URL: {url}\nPage Title: {title}\n\nPage Content:\n{content}",
                url = page_content.url,
                title = page_content.title,
                content = page_content.body_text,
            )
        };

        eprintln!("      assert: {preset_name}");

        let endpoint = self
            .endpoints
            .resolve(step_endpoint.or(test_endpoint), TaskType::Assertion);
        let llm = self.build_llm_for_endpoint(endpoint);
        let usage = Arc::clone(&self.usage);
        let endpoint_name = endpoint.name.clone();
        let sys = preset.system.to_owned();

        let response = std::thread::spawn(move || {
            let rt = tokio::runtime::Builder::new_current_thread()
                .enable_all()
                .build()
                .unwrap();
            rt.block_on(llm_chat_with_usage(&llm, &sys, &user_prompt))
        })
        .join()
        .unwrap();

        response.map_or_else(
            |e| StepResult {
                name: format!("[assert] {preset_name}"),
                status: StepStatus::Failed,
                message: format!("LLM assertion call failed: {e}"),
            },
            |lr| {
                usage.record_llm_call(
                    &endpoint_name,
                    endpoint,
                    lr.usage.prompt_tokens,
                    lr.usage.completion_tokens,
                );
                let content_lower = lr.content.to_lowercase().trim().to_owned();
                if content_lower.starts_with("pass") {
                    StepResult {
                        name: format!("[assert] {preset_name}"),
                        status: StepStatus::Passed,
                        message: "PASS".into(),
                    }
                } else {
                    StepResult {
                        name: format!("[assert] {preset_name}"),
                        status: StepStatus::Failed,
                        message: lr.content,
                    }
                }
            },
        )
    }

    fn run_custom(
        &self,
        prompt: &str,
        page_content: &PageContent,
        step_endpoint: Option<&str>,
        test_endpoint: Option<&str>,
    ) -> StepResult {
        let system = "You are a QA tester evaluating a web page. Respond with exactly \"PASS\" if the assertion holds, or \"FAIL: <reason>\" if it does not.";

        let user = format!(
            "Page URL: {url}\nPage Title: {title}\n\nPage Content:\n{content}\n\nAssertion: {prompt}",
            url = page_content.url,
            title = page_content.title,
            content = page_content.body_text,
        );

        eprintln!("      custom assert");

        let endpoint = self
            .endpoints
            .resolve(step_endpoint.or(test_endpoint), TaskType::Assertion);
        let llm = self.build_llm_for_endpoint(endpoint);
        let usage = Arc::clone(&self.usage);
        let endpoint_name = endpoint.name.clone();
        let sys = system.to_owned();

        let response = std::thread::spawn(move || {
            let rt = tokio::runtime::Builder::new_current_thread()
                .enable_all()
                .build()
                .unwrap();
            rt.block_on(llm_chat_with_usage(&llm, &sys, &user))
        })
        .join()
        .unwrap();

        response.map_or_else(
            |e| StepResult {
                name: "[assert] custom".into(),
                status: StepStatus::Failed,
                message: format!("LLM assertion call failed: {e}"),
            },
            |lr| {
                usage.record_llm_call(
                    &endpoint_name,
                    endpoint,
                    lr.usage.prompt_tokens,
                    lr.usage.completion_tokens,
                );
                let content_lower = lr.content.to_lowercase().trim().to_owned();
                if content_lower.starts_with("pass") {
                    StepResult {
                        name: "[assert] custom".into(),
                        status: StepStatus::Passed,
                        message: "PASS".into(),
                    }
                } else {
                    StepResult {
                        name: "[assert] custom".into(),
                        status: StepStatus::Failed,
                        message: lr.content,
                    }
                }
            },
        )
    }

    fn run_screenshot(path: Option<&str>, tab: &Tab) -> StepResult {
        let path = path.unwrap_or("screenshot.png");

        match tab.capture_screenshot(
            headless_chrome::protocol::cdp::Page::CaptureScreenshotFormatOption::Png,
            None,
            None,
            true,
        ) {
            Ok(data) => {
                if let Err(e) = std::fs::write(path, &data) {
                    return StepResult {
                        name: format!("[screenshot] {path}"),
                        status: StepStatus::Failed,
                        message: format!("failed to write screenshot: {e}"),
                    };
                }
                StepResult {
                    name: format!("[screenshot] {path}"),
                    status: StepStatus::Passed,
                    message: format!("saved to {path}"),
                }
            }
            Err(e) => StepResult {
                name: format!("[screenshot] {path}"),
                status: StepStatus::Failed,
                message: format!("screenshot failed: {e}"),
            },
        }
    }

    /// Runs an A2A agent step.
    #[allow(clippy::literal_string_with_formatting_args)]
    fn run_agent(
        &self,
        agent_name: &str,
        task: &str,
        definition: Option<&str>,
        _test_endpoint: Option<&str>,
    ) -> StepResult {
        // If a definition is specified, look up the task template
        let resolved_task = if let Some(def_name) = definition {
            if let Some(def) = self.definitions.get(def_name) {
                let tmpl = def.task_template.as_deref().unwrap_or(task);
                tmpl.replace("{task}", task)
            } else {
                return StepResult {
                    name: format!("[agent] {def_name}"),
                    status: StepStatus::Failed,
                    message: format!("definition '{def_name}' not found"),
                };
            }
        } else {
            task.to_owned()
        };

        self.run_agent_step(agent_name, &resolved_task, &format!("agent:{agent_name}"))
    }

    fn run_agent_step(&self, agent_name: &str, task: &str, display_name: &str) -> StepResult {
        let Some(ep) = self.endpoints.get(agent_name) else {
            return StepResult {
                name: format!("[agent] {display_name}"),
                status: StepStatus::Failed,
                message: format!("agent endpoint '{agent_name}' not found"),
            };
        };

        if ep.url.is_empty() {
            return StepResult {
                name: format!("[agent] {display_name}"),
                status: StepStatus::Failed,
                message: format!("agent endpoint '{agent_name}' has no URL"),
            };
        }

        eprintln!("      → agent {agent_name}: {task}");

        let url = ep.url.clone();
        let client = A2aClient::new(&url, self.timeout);
        let task_clone = task.to_owned();

        let response = std::thread::spawn(move || {
            let rt = tokio::runtime::Builder::new_current_thread()
                .enable_all()
                .build()
                .unwrap();
            rt.block_on(client.send_task(&task_clone))
        })
        .join()
        .unwrap();

        // Record the flat-cost call
        self.usage.record_flat_call(agent_name, ep);

        match response {
            Ok(text) => {
                let clean = text.trim().to_owned();
                let lower = clean.to_lowercase();
                if lower.starts_with("pass") {
                    StepResult {
                        name: format!("[agent] {display_name}"),
                        status: StepStatus::Passed,
                        message: format!("PASS: {clean}"),
                    }
                } else if lower.starts_with("fail") {
                    StepResult {
                        name: format!("[agent] {display_name}"),
                        status: StepStatus::Failed,
                        message: clean,
                    }
                } else {
                    StepResult {
                        name: format!("[agent] {display_name}"),
                        status: StepStatus::Passed,
                        message: format!("response: {clean}"),
                    }
                }
            }
            Err(e) => StepResult {
                name: format!("[agent] {display_name}"),
                status: StepStatus::Failed,
                message: format!("agent call failed: {e}"),
            },
        }
    }

    /// Runs an MCP tool call step.
    fn run_mcp(
        &self,
        server_name: &str,
        tool_name: &str,
        args: Option<&serde_json::Value>,
    ) -> StepResult {
        let Some(ep) = self.endpoints.get(server_name) else {
            return StepResult {
                name: format!("[mcp] {server_name}:{tool_name}"),
                status: StepStatus::Failed,
                message: format!("MCP server endpoint '{server_name}' not found"),
            };
        };

        let cmd = ep.command.as_deref().unwrap_or("");
        if cmd.is_empty() {
            return StepResult {
                name: format!("[mcp] {server_name}:{tool_name}"),
                status: StepStatus::Failed,
                message: format!("MCP server '{server_name}' has no command configured"),
            };
        }

        eprintln!("      → mcp {server_name} {tool_name}");

        let args_val = args.cloned().unwrap_or(serde_json::Value::Null);

        let command = cmd.to_owned();
        let args_vec = ep.args.clone();
        let tool = tool_name.to_owned();

        let response = std::thread::spawn(move || {
            let mut mcp_client =
                McpClient::connect_stdio(&command, &args_vec).map_err(|e| e.to_string())?;
            mcp_client
                .call_tool(&tool, &args_val)
                .map_err(|e| e.to_string())
        })
        .join()
        .unwrap();

        // Record the flat-cost call
        self.usage.record_flat_call(server_name, ep);

        match response {
            Ok(result) => {
                if result.isError {
                    StepResult {
                        name: format!("[mcp] {server_name}:{tool_name}"),
                        status: StepStatus::Failed,
                        message: result.to_string(),
                    }
                } else {
                    StepResult {
                        name: format!("[mcp] {server_name}:{tool_name}"),
                        status: StepStatus::Passed,
                        message: result.to_string(),
                    }
                }
            }
            Err(e) => StepResult {
                name: format!("[mcp] {server_name}:{tool_name}"),
                status: StepStatus::Failed,
                message: format!("MCP call failed: {e}"),
            },
        }
    }

    // ── helpers ──────────────────────────────────────────────────────────

    /// Builds an `LlmConfig` from a resolved endpoint, falling back to
    /// the runner's default LLM config for any unset fields.
    fn build_llm_for_endpoint(&self, endpoint: &crate::endpoints::ResolvedEndpoint) -> LlmConfig {
        LlmConfig {
            url: if endpoint.url.is_empty() {
                self.llm.url.clone()
            } else {
                endpoint.url.clone()
            },
            model: endpoint
                .model
                .clone()
                .unwrap_or_else(|| self.llm.model.clone()),
            api_key: endpoint
                .api_key
                .clone()
                .or_else(|| self.llm.api_key.clone()),
            headers: if endpoint.headers.is_empty() {
                self.llm.headers.clone()
            } else {
                endpoint.headers.clone()
            },
            timeout: self.llm.timeout,
            temperature: self.llm.temperature,
            thinking: self.llm.thinking,
            model_params: self.llm.model_params.clone(),
        }
    }

    /// Resolves a CSS selector for the target element. Uses the explicit
    /// `selector` if provided, otherwise asks the LLM to find the element
    /// from the natural language `target` description and page DOM.
    ///
    /// LLM responses are sanitized and verified against the live page: a
    /// response that is not a selector (empty, `:not(*)`, `null`, …) fails
    /// immediately with the raw LLM output, and a selector that matches
    /// nothing triggers one retry with feedback before failing.
    #[allow(clippy::too_many_lines)]
    fn resolve_selector(
        &self,
        css_override: Option<&str>,
        target: &str,
        step_endpoint: Option<&str>,
        test_endpoint: Option<&str>,
        tab: &Tab,
    ) -> Result<String, String> {
        if let Some(explicit) = css_override {
            return Ok(explicit.to_owned());
        }

        let dom_info = extract_dom_info(tab)?;
        let page_content = get_page_text(tab);

        let system = concat!(
            "You are a browser automation selector generator. ",
            "Given a web page's content and interactive elements, ",
            "return ONLY the best CSS selector for the described element. ",
            "Output nothing except the CSS selector. ",
            "Prefer selectors in this order: #id, [data-testid=\"...\"], ",
            "[name=\"...\"], tag.class, tag. ",
            "Never output explanations, markdown, or extra text."
        );

        let user = format!(
            "Page URL: {}\nPage Title: {}\n\nPage body text (first 4000 chars):\n{}\n\nInteractive elements:\n{}\n\nFind the CSS selector for: {}",
            page_content.url,
            page_content.title,
            truncate(&page_content.body_text, 4000),
            dom_info,
            target,
        );

        let retry_user = format!(
            "Your previous answer was not usable. {}\n\nPage URL: {}\nPage Title: {}\n\nPage body text (first 4000 chars):\n{}\n\nInteractive elements:\n{}\n\nFind the CSS selector for: {}\nReturn ONLY a single CSS selector that matches an existing element. No explanations.",
            "The selector must match at least one element currently present on the page.",
            page_content.url,
            page_content.title,
            truncate(&page_content.body_text, 4000),
            dom_info,
            target,
        );

        eprintln!("      LLM targeting: {target}");

        let endpoint = self
            .endpoints
            .resolve(step_endpoint.or(test_endpoint), TaskType::Targeting);
        let llm = self.build_llm_for_endpoint(endpoint);
        let usage = Arc::clone(&self.usage);
        let endpoint_name = endpoint.name.clone();
        let endpoint_clone = endpoint.clone();
        let sys = system.to_owned();

        let call_llm = |prompt: &str| {
            let llm = llm.clone();
            let sys = sys.clone();
            let prompt = prompt.to_owned();
            std::thread::spawn(move || {
                let rt = tokio::runtime::Builder::new_current_thread()
                    .enable_all()
                    .build()
                    .unwrap();
                rt.block_on(llm_chat_with_usage(&llm, &sys, &prompt))
            })
            .join()
            .unwrap()
        };

        let first = call_llm(&user);
        let lr = match first {
            Ok(lr) => lr,
            Err(e) => {
                return Err(format!("LLM element targeting failed: {e}"));
            }
        };
        usage.record_llm_call(
            &endpoint_name,
            &endpoint_clone,
            lr.usage.prompt_tokens,
            lr.usage.completion_tokens,
        );
        let clean = sanitize_selector(&lr.content);
        eprintln!("      resolved selector: {clean}");

        if selector_is_useless(&clean) {
            return Err(format!(
                "LLM element targeting failed: the LLM did not return a usable selector for {target:?} (got {raw:?}). Check the page state in the diagnostics above.",
                raw = lr.content.trim(),
            ));
        }
        if let Err(reason) = validate_selector(&clean) {
            return Err(format!(
                "LLM element targeting failed: invalid selector for {target:?}: {reason} (LLM response: {raw:?})",
                raw = lr.content.trim(),
            ));
        }
        if !selector_matches(tab, &clean).unwrap_or(false) {
            // One retry with feedback: flaky models occasionally invent a
            // selector that does not exist on the page.
            eprintln!(
                "      selector {clean} matches nothing — retrying LLM targeting with feedback"
            );
            let second = call_llm(&retry_user);
            let lr2 = match second {
                Ok(lr2) => lr2,
                Err(e) => {
                    return Err(format!(
                        "LLM element targeting failed: first answer {clean:?} matched nothing, retry also failed: {e}"
                    ));
                }
            };
            usage.record_llm_call(
                &endpoint_name,
                &endpoint_clone,
                lr2.usage.prompt_tokens,
                lr2.usage.completion_tokens,
            );
            let clean2 = sanitize_selector(&lr2.content);
            eprintln!("      resolved selector (retry): {clean2}");
            if selector_is_useless(&clean2) {
                return Err(format!(
                    "LLM element targeting failed: selector {clean:?} matched nothing; the retry returned no usable selector for {target:?} (got {raw:?}). Page excerpt: {excerpt}",
                    raw = lr2.content.trim(),
                    excerpt = truncate(&page_content.body_text, 300),
                ));
            }
            if !selector_matches(tab, &clean2).unwrap_or(false) {
                return Err(format!(
                    "LLM element targeting failed: selector {clean2:?} does not match any element on the page for {target:?}. Verify the page state in the diagnostics; the login/SPA may not have rendered."
                ));
            }
            return Ok(clean2);
        }

        Ok(clean)
    }
}

/// Evaluates a JS expression that is expected to return a boolean.
fn eval_bool(tab: &Tab, js: &str) -> Result<bool, String> {
    tab.evaluate(js, false)
        .map_err(|e| format!("evaluate failed: {e}"))?
        .value
        .and_then(|v| v.as_bool())
        .ok_or_else(|| "evaluate returned non-boolean".to_owned())
}

/// Checks whether a CSS selector matches at least one current element.
fn selector_matches(tab: &Tab, selector: &str) -> Result<bool, String> {
    eval_bool(tab, &crate::selectors::selector_matches_js(selector))
}

// ── Free helper functions ──────────────────────────────────────────────

fn run_navigate_step(full_url: &str, tab: &Tab) -> StepResult {
    let name = format!("[navigate] {full_url}");
    match tab.navigate_to(full_url) {
        Ok(_) => {
            let _ = tab.wait_until_navigated();
            StepResult {
                name,
                status: StepStatus::Passed,
                message: format!("navigated to {full_url}"),
            }
        }
        Err(e) => StepResult {
            name,
            status: StepStatus::Failed,
            message: format!("navigation failed: {e}"),
        },
    }
}

fn extract_dom_info(tab: &Tab) -> Result<String, String> {
    let result = tab
        .evaluate(DOM_EXTRACT_JS, false)
        .map_err(|e| format!("DOM extraction failed: {e}"))?;

    let json_str = result
        .value
        .as_ref()
        .and_then(|v| v.as_str())
        .unwrap_or("[]");

    let elements: Vec<String> = serde_json::from_str(json_str).unwrap_or_default();

    if elements.is_empty() {
        return Ok("(no interactive elements found)".to_owned());
    }

    Ok(elements.join("\n"))
}

fn get_page_text(tab: &Tab) -> PageContent {
    let url = tab.get_url();

    let title = tab
        .evaluate("document.title", false)
        .ok()
        .and_then(|r| r.value)
        .and_then(|v| v.as_str().map(String::from))
        .unwrap_or_else(|| "unknown".to_owned());

    let body_text = tab
        .evaluate(
            "document.body ? document.body.innerText : document.documentElement.innerText",
            false,
        )
        .ok()
        .and_then(|r| r.value)
        .and_then(|v| v.as_str().map(String::from))
        .unwrap_or_default();

    PageContent {
        url,
        title,
        body_text: truncate(&body_text, 8000),
    }
}

fn resolve_url(url: &str, base_url: &str) -> String {
    if url.starts_with("http://") || url.starts_with("https://") {
        return url.to_owned();
    }
    let base = base_url.trim_end_matches('/');
    if url.starts_with('/') {
        format!("{base}{url}")
    } else {
        format!("{base}/{url}")
    }
}

/// Human-readable label for a step, used when steps are skipped after an
/// earlier failure.
fn step_label(step: &TestStep) -> String {
    match step {
        TestStep::Navigate { url, .. } => format!("[navigate] {url}"),
        TestStep::Click { target, .. } => format!("[click] {target}"),
        TestStep::Type { target, .. } => format!("[type] {target}"),
        TestStep::Wait { target, .. } => format!("[wait] {target}"),
        TestStep::Assert {
            definition,
            preset,
            prompt,
            ..
        } => {
            if let Some(d) = definition {
                format!("[assert] {d}")
            } else if let Some(p) = preset {
                format!("[assert] {p}")
            } else if let Some(pr) = prompt {
                format!("[assert] custom ({})", truncate(pr, 60))
            } else {
                "[assert]".to_owned()
            }
        }
        TestStep::Screenshot { .. } => "[screenshot]".to_owned(),
        TestStep::Agent { agent, .. } => format!("[agent] {agent}"),
        TestStep::Mcp { server, tool, .. } => format!("[mcp] {server}:{tool}"),
    }
}

/// Short kind word for artifact file names (e.g. `click`, `wait`, `assert`).
#[must_use]
fn step_kind_label(step: &TestStep) -> &'static str {
    match step {
        TestStep::Navigate { .. } => "navigate",
        TestStep::Click { .. } => "click",
        TestStep::Type { .. } => "type",
        TestStep::Wait { .. } => "wait",
        TestStep::Assert { .. } => "assert",
        TestStep::Screenshot { .. } => "screenshot",
        TestStep::Agent { .. } => "agent",
        TestStep::Mcp { .. } => "mcp",
    }
}

// ── Support types ──────────────────────────────────────────────────────

#[derive(Default)]
struct TestRunResult {
    passed: u32,
    failed: u32,
    skipped: u32,
    total: u32,
    details: Vec<StepResult>,
}

struct PageContent {
    url: String,
    title: String,
    body_text: String,
}