llm_browser_testkit/
runner.rs

1use std::collections::HashMap;
2use std::time::Duration;
3
4use anyhow::Context;
5use headless_chrome::{Browser, LaunchOptions, Tab};
6
7use crate::llm_chat;
8use crate::scenario::{AssertDefinition, ScenarioConfig, TestGroup, TestStep};
9use crate::truncate;
10use crate::LlmConfig;
11use crate::DOM_EXTRACT_JS;
12
13/// Executes a [`Scenario`] against a real browser with optional LLM
14/// assistance for element targeting and assertions.
15pub struct ScenarioRunner {
16    config: ScenarioConfig,
17    definitions: HashMap<String, AssertDefinition>,
18    llm: LlmConfig,
19    timeout: Duration,
20    viewport_width: u32,
21    viewport_height: u32,
22}
23
24/// Aggregated results from a scenario run.
25#[derive(Debug, Default)]
26pub struct RunReport {
27    /// Number of tests that passed.
28    pub tests_passed: u32,
29    /// Number of tests that failed.
30    pub tests_failed: u32,
31    /// Number of steps that passed.
32    pub passed: u32,
33    /// Number of steps that failed.
34    pub failed: u32,
35    /// Number of steps that were skipped.
36    pub skipped: u32,
37    /// Per-step details.
38    pub details: Vec<StepResult>,
39}
40
41/// Result of a single step execution.
42#[derive(Debug)]
43pub struct StepResult {
44    /// The step name.
45    pub name: String,
46    /// Whether the step passed, failed, or was skipped.
47    pub status: StepStatus,
48    /// Human-readable result message.
49    pub message: String,
50}
51
52/// Outcome for a single step.
53#[derive(Debug, PartialEq, Eq)]
54pub enum StepStatus {
55    /// Step executed successfully and all assertions passed.
56    Passed,
57    /// Step execution or assertion failed.
58    Failed,
59    /// Step was skipped.
60    Skipped,
61}
62
63/// Predefined assertion preset definition.
64struct AssertPreset {
65    name: &'static str,
66    system: &'static str,
67    user_template: &'static str,
68}
69
70/// Built-in assertion presets.
71#[allow(clippy::literal_string_with_formatting_args)]
72const ASSERTION_PRESETS: &[AssertPreset] = &[
73    AssertPreset {
74        name: "no_error_on_page",
75        system: "You are a QA tester. Evaluate if a web page contains error messages, stack traces, exception text, HTTP error codes, 'undefined' errors, or any indication of a malfunction. Be strict — even minor rendering glitches count as errors.",
76        user_template: "Check if the following page content contains ANY errors or malfunctions:\n\nURL: {url}\nTitle: {title}\n\nPage Content:\n{content}\n\nRespond with exactly \"PASS\" if there are NO errors, or \"FAIL: <reason>\" if there are errors. Only respond with PASS or FAIL.",
77    },
78    AssertPreset {
79        name: "text_visible",
80        system: "You are a QA tester. Your task is to check if specific text is visible in the page content.",
81        user_template: "Check if the following text appears in the page content:\n\nTEXT TO FIND: \"{expected_text}\"\n\nURL: {url}\n\nPage Content:\n{content}\n\nRespond with exactly \"PASS\" if the text is present (even partial match is OK), or \"FAIL: text not found\" if it is not.",
82    },
83    AssertPreset {
84        name: "element_exists",
85        system: "You are a QA tester. Check if a described UI element exists on a web page.",
86        user_template: "Check if the following element exists on the page:\n\nELEMENT: \"{description}\"\n\nURL: {url}\n\nPage Content:\n{content}\n\nRespond with exactly \"PASS\" if the element exists, or \"FAIL: <reason>\" if it does not.",
87    },
88];
89
90impl ScenarioRunner {
91    /// Creates a new runner with the given scenario configuration and
92    /// assertion definitions.
93    #[must_use]
94    pub fn new(scenario_config: ScenarioConfig, definitions: Vec<AssertDefinition>) -> Self {
95        let llm = LlmConfig {
96            url: scenario_config
97                .llm_url
98                .clone()
99                .unwrap_or_else(crate::llm_base_url),
100            model: scenario_config
101                .llm_model
102                .clone()
103                .unwrap_or_else(crate::llm_model),
104            api_key: scenario_config.llm_api_key.clone().or_else(|| {
105                std::env::var("HARNESS_LLM_API_KEY").ok()
106            }),
107            headers: if scenario_config.llm_headers.is_empty() {
108                crate::parse_headers_env()
109            } else {
110                scenario_config.llm_headers.clone()
111            },
112            timeout: Duration::from_secs(scenario_config.timeout_secs.unwrap_or(60)),
113            temperature: scenario_config.temperature,
114            thinking: scenario_config.thinking,
115        };
116        let timeout = Duration::from_secs(scenario_config.timeout_secs.unwrap_or(60));
117        let viewport_width = scenario_config.viewport_width.unwrap_or(1280);
118        let viewport_height = scenario_config.viewport_height.unwrap_or(720);
119        let defs_map: HashMap<String, AssertDefinition> = definitions
120            .into_iter()
121            .map(|d| (d.name.clone(), d))
122            .collect();
123
124        Self {
125            config: scenario_config,
126            definitions: defs_map,
127            llm,
128            timeout,
129            viewport_width,
130            viewport_height,
131        }
132    }
133
134    /// Executes all test groups in the scenario and returns a report.
135    ///
136    /// # Errors
137    ///
138    /// Returns an error if the browser fails to launch.
139    pub fn run(&self, tests: &[TestGroup]) -> anyhow::Result<RunReport> {
140        let mut report = RunReport::default();
141
142        if tests.is_empty() {
143            eprintln!("No tests defined in scenario.");
144            return Ok(report);
145        }
146
147        let browser_headless = self.config.browser_headless.unwrap_or(true);
148
149        let launch_opts = LaunchOptions {
150            headless: browser_headless,
151            window_size: Some((self.viewport_width, self.viewport_height)),
152            sandbox: false,
153            ..LaunchOptions::default()
154        };
155
156        let browser = Browser::new(launch_opts).context("failed to launch browser")?;
157        let tab = browser.new_tab().context("failed to open browser tab")?;
158        let _ = tab.set_default_timeout(self.timeout);
159
160        for test in tests {
161            eprintln!("\n══════════════════════════════");
162            eprintln!("  TEST: {}", test.name);
163            eprintln!("══════════════════════════════");
164
165            let test_result = self.run_test(test, &tab);
166            if test_result.failed == 0 && test_result.total > 0 {
167                report.tests_passed += 1;
168                eprintln!("  TEST ✅ PASSED");
169            } else if test_result.total > 0 {
170                report.tests_failed += 1;
171                eprintln!("  TEST ❌ FAILED");
172            }
173
174            report.passed += test_result.passed;
175            report.failed += test_result.failed;
176            report.skipped += test_result.skipped;
177            report.details.extend(test_result.details);
178        }
179
180        Ok(report)
181    }
182
183    fn run_test(&self, test: &TestGroup, tab: &Tab) -> TestRunResult {
184        let base_url = test
185            .base_url
186            .clone()
187            .or_else(|| self.config.base_url.clone())
188            .unwrap_or_else(crate::base_url);
189
190        let auto_navigate = test.auto_navigate.unwrap_or(self.config.auto_navigate);
191
192        let start_url = test
193            .start_url
194            .clone()
195            .or_else(|| self.config.start_url.clone())
196            .unwrap_or_else(|| "/dashboard".to_owned());
197
198        if auto_navigate {
199            let full_url = resolve_url(&start_url, &base_url);
200            eprintln!("  → auto-navigate: {full_url}");
201            let _ = tab.navigate_to(&full_url);
202            let _ = tab.wait_until_navigated();
203            std::thread::sleep(Duration::from_secs(4));
204        }
205
206        let mut result = TestRunResult {
207            passed: 0,
208            failed: 0,
209            skipped: 0,
210            total: 0,
211            details: Vec::new(),
212        };
213
214        for step in &test.steps {
215            result.total += 1;
216
217            // Extract per-step wait_after_ms before the step consumes the data
218            let wait_ms = match step {
219                TestStep::Navigate { wait_after_ms, .. }
220                | TestStep::Click { wait_after_ms, .. }
221                | TestStep::Type { wait_after_ms, .. } => *wait_after_ms,
222                _ => None,
223            };
224
225            let step_result = match step {
226                TestStep::Navigate { url, .. } => {
227                    let full_url = resolve_url(url, &base_url);
228                    run_navigate_step(&full_url, tab)
229                }
230                TestStep::Click {
231                    target, selector, ..
232                } => self.run_click(target, selector.as_deref(), tab),
233                TestStep::Type {
234                    target,
235                    text,
236                    selector,
237                    ..
238                } => self.run_type(target, text, selector.as_deref(), tab),
239                TestStep::Wait {
240                    target,
241                    selector,
242                    timeout_ms,
243                } => self.run_wait(target, selector.as_deref(), *timeout_ms, tab),
244                TestStep::Assert {
245                    definition,
246                    preset,
247                    prompt,
248                    assert_text,
249                } => self.run_assert(
250                    definition.as_deref(),
251                    preset.as_deref(),
252                    prompt.as_deref(),
253                    assert_text.as_deref(),
254                    tab,
255                ),
256                TestStep::Screenshot { path } => Self::run_screenshot(path.as_deref(), tab),
257            };
258
259            eprintln!(
260                "    {} {} — {}",
261                if step_result.status == StepStatus::Passed {
262                    "✅"
263                } else if step_result.status == StepStatus::Failed {
264                    "❌"
265                } else {
266                    "⏭️"
267                },
268                step_result.name,
269                step_result.message,
270            );
271
272            match step_result.status {
273                StepStatus::Passed => result.passed += 1,
274                StepStatus::Failed => result.failed += 1,
275                StepStatus::Skipped => result.skipped += 1,
276            }
277
278            if let Some(ms) = wait_ms {
279                std::thread::sleep(Duration::from_millis(ms));
280            }
281
282            result.details.push(step_result);
283        }
284
285        result
286    }
287
288    // ── step handlers ───────────────────────────────────────────────────
289
290    fn run_click(&self, target: &str, selector_override: Option<&str>, tab: &Tab) -> StepResult {
291        let selector = match self.resolve_selector(selector_override, target, tab) {
292            Ok(s) => s,
293            Err(msg) => {
294                return StepResult {
295                    name: format!("[click] {target}"),
296                    status: StepStatus::Failed,
297                    message: msg,
298                };
299            }
300        };
301
302        match tab.wait_for_element(&selector) {
303            Ok(element) => match element.click() {
304                Ok(_) => StepResult {
305                    name: format!("[click] {target}"),
306                    status: StepStatus::Passed,
307                    message: format!("clicked {selector}"),
308                },
309                Err(e) => StepResult {
310                    name: format!("[click] {target}"),
311                    status: StepStatus::Failed,
312                    message: format!("click failed on {selector}: {e}"),
313                },
314            },
315            Err(e) => StepResult {
316                name: format!("[click] {target}"),
317                status: StepStatus::Failed,
318                message: format!("element {selector} not found: {e}"),
319            },
320        }
321    }
322
323    fn run_type(
324        &self,
325        target: &str,
326        text: &str,
327        selector_override: Option<&str>,
328        tab: &Tab,
329    ) -> StepResult {
330        let selector = match self.resolve_selector(selector_override, target, tab) {
331            Ok(s) => s,
332            Err(msg) => {
333                return StepResult {
334                    name: format!("[type] {target}"),
335                    status: StepStatus::Failed,
336                    message: msg,
337                };
338            }
339        };
340
341        match tab.wait_for_element(&selector) {
342            Ok(element) => {
343                if let Err(e) = element.click() {
344                    return StepResult {
345                        name: format!("[type] {target}"),
346                        status: StepStatus::Failed,
347                        message: format!("click to focus {selector} failed: {e}"),
348                    };
349                }
350
351                let js = format!(
352                    "document.querySelector('{}').value = '';",
353                    selector.replace('\'', "\\'")
354                );
355                let _ = tab.evaluate(&js, false);
356
357                match element.type_into(text) {
358                    Ok(_) => StepResult {
359                        name: format!("[type] {target}"),
360                        status: StepStatus::Passed,
361                        message: format!("typed {text:?} into {selector}"),
362                    },
363                    Err(e) => StepResult {
364                        name: format!("[type] {target}"),
365                        status: StepStatus::Failed,
366                        message: format!("type into {selector} failed: {e}"),
367                    },
368                }
369            }
370            Err(e) => StepResult {
371                name: format!("[type] {target}"),
372                status: StepStatus::Failed,
373                message: format!("element {selector} not found: {e}"),
374            },
375        }
376    }
377
378    fn run_wait(
379        &self,
380        target: &str,
381        selector_override: Option<&str>,
382        timeout_ms: Option<u64>,
383        tab: &Tab,
384    ) -> StepResult {
385        let selector = match self.resolve_selector(selector_override, target, tab) {
386            Ok(s) => s,
387            Err(msg) => {
388                return StepResult {
389                    name: format!("[wait] {target}"),
390                    status: StepStatus::Failed,
391                    message: msg,
392                };
393            }
394        };
395
396        let timeout = Duration::from_millis(timeout_ms.unwrap_or(10_000));
397
398        match tab.wait_for_element_with_custom_timeout(&selector, timeout) {
399            Ok(_) => StepResult {
400                name: format!("[wait] {target}"),
401                status: StepStatus::Passed,
402                message: format!("found {selector}"),
403            },
404            Err(e) => StepResult {
405                name: format!("[wait] {target}"),
406                status: StepStatus::Failed,
407                message: format!("wait for {selector} timed out: {e}"),
408            },
409        }
410    }
411
412    fn run_assert(
413        &self,
414        definition: Option<&str>,
415        preset: Option<&str>,
416        prompt: Option<&str>,
417        assert_text: Option<&str>,
418        tab: &Tab,
419    ) -> StepResult {
420        std::thread::sleep(Duration::from_millis(500));
421
422        let page_content = get_page_text(tab);
423
424        if let Some(def_name) = definition {
425            if let Some(def) = self.definitions.get(def_name) {
426                return self.run_assert_def(def, &page_content);
427            }
428            return StepResult {
429                name: format!("[assert] {def_name}"),
430                status: StepStatus::Failed,
431                message: format!("definition '{def_name}' not found"),
432            };
433        }
434
435        if let Some(preset_name) = preset {
436            return self.run_preset(preset_name, assert_text, &page_content);
437        }
438
439        if let Some(prompt_text) = prompt {
440            return self.run_custom(prompt_text, &page_content);
441        }
442
443        StepResult {
444            name: "[assert]".into(),
445            status: StepStatus::Skipped,
446            message: "no definition, preset, or prompt specified".into(),
447        }
448    }
449
450    fn run_assert_def(&self, def: &AssertDefinition, page_content: &PageContent) -> StepResult {
451        def.preset.as_ref().map_or_else(
452            || {
453                def.prompt.as_ref().map_or_else(
454                    || StepResult {
455                        name: format!("[assert] {}", def.name),
456                        status: StepStatus::Failed,
457                        message: "definition has no preset or prompt".into(),
458                    },
459                    |prompt| self.run_custom(prompt, page_content),
460                )
461            },
462            |preset_name| self.run_preset(preset_name, def.assert_text.as_deref(), page_content),
463        )
464    }
465
466    fn run_preset(
467        &self,
468        preset_name: &str,
469        assert_text: Option<&str>,
470        page_content: &PageContent,
471    ) -> StepResult {
472        let Some(preset) = ASSERTION_PRESETS.iter().find(|p| p.name == preset_name) else {
473            return StepResult {
474                name: format!("[assert] {preset_name}"),
475                status: StepStatus::Failed,
476                message: format!("unknown assertion preset: {preset_name}"),
477            };
478        };
479
480        let user_prompt = preset
481            .user_template
482            .replace("{url}", &page_content.url)
483            .replace("{title}", &page_content.title)
484            .replace("{content}", &page_content.body_text)
485            .replace("{expected_text}", assert_text.unwrap_or(""))
486            .replace("{description}", "");
487
488        eprintln!("      assert: {preset_name}");
489
490        let (llm, sys, prompt) = (
491            self.llm.clone(),
492            preset.system.to_owned(),
493            user_prompt,
494        );
495        let response = std::thread::spawn(move || {
496            let rt = tokio::runtime::Builder::new_current_thread()
497                .enable_all()
498                .build()
499                .unwrap();
500            rt.block_on(llm_chat(&llm, &sys, &prompt))
501        })
502        .join()
503        .unwrap();
504
505        response.map_or_else(
506            || StepResult {
507                name: format!("[assert] {preset_name}"),
508                status: StepStatus::Failed,
509                message: "LLM assertion call failed (server down?)".into(),
510            },
511            |content| {
512                let content_lower = content.to_lowercase().trim().to_owned();
513                if content_lower.starts_with("pass") {
514                    StepResult {
515                        name: format!("[assert] {preset_name}"),
516                        status: StepStatus::Passed,
517                        message: "PASS".into(),
518                    }
519                } else {
520                    StepResult {
521                        name: format!("[assert] {preset_name}"),
522                        status: StepStatus::Failed,
523                        message: content,
524                    }
525                }
526            },
527        )
528    }
529
530    fn run_custom(&self, prompt: &str, page_content: &PageContent) -> StepResult {
531        let system = "You are a QA tester evaluating a web page. Respond with exactly \"PASS\" if the assertion holds, or \"FAIL: <reason>\" if it does not.";
532
533        let user = format!(
534            "Page URL: {url}\nPage Title: {title}\n\nPage Content:\n{content}\n\nAssertion: {prompt}",
535            url = page_content.url,
536            title = page_content.title,
537            content = page_content.body_text,
538        );
539
540        eprintln!("      custom assert");
541
542        let (llm, sys, user_prompt) = (
543            self.llm.clone(),
544            system.to_owned(),
545            user,
546        );
547        let response = std::thread::spawn(move || {
548            let rt = tokio::runtime::Builder::new_current_thread()
549                .enable_all()
550                .build()
551                .unwrap();
552            rt.block_on(llm_chat(&llm, &sys, &user_prompt))
553        })
554        .join()
555        .unwrap();
556
557        response.map_or_else(
558            || StepResult {
559                name: "[assert] custom".into(),
560                status: StepStatus::Failed,
561                message: "LLM assertion call failed (server down?)".into(),
562            },
563            |content| {
564                let content_lower = content.to_lowercase().trim().to_owned();
565                if content_lower.starts_with("pass") {
566                    StepResult {
567                        name: "[assert] custom".into(),
568                        status: StepStatus::Passed,
569                        message: "PASS".into(),
570                    }
571                } else {
572                    StepResult {
573                        name: "[assert] custom".into(),
574                        status: StepStatus::Failed,
575                        message: content,
576                    }
577                }
578            },
579        )
580    }
581
582    fn run_screenshot(path: Option<&str>, tab: &Tab) -> StepResult {
583        let path = path.unwrap_or("screenshot.png");
584
585        match tab.capture_screenshot(
586            headless_chrome::protocol::cdp::Page::CaptureScreenshotFormatOption::Png,
587            None,
588            None,
589            true,
590        ) {
591            Ok(data) => {
592                if let Err(e) = std::fs::write(path, &data) {
593                    return StepResult {
594                        name: format!("[screenshot] {path}"),
595                        status: StepStatus::Failed,
596                        message: format!("failed to write screenshot: {e}"),
597                    };
598                }
599                StepResult {
600                    name: format!("[screenshot] {path}"),
601                    status: StepStatus::Passed,
602                    message: format!("saved to {path}"),
603                }
604            }
605            Err(e) => StepResult {
606                name: format!("[screenshot] {path}"),
607                status: StepStatus::Failed,
608                message: format!("screenshot failed: {e}"),
609            },
610        }
611    }
612
613    // ── helpers ──────────────────────────────────────────────────────────
614
615    /// Resolves a CSS selector for the target element. Uses the explicit
616    /// `selector` if provided, otherwise asks the LLM to find the element
617    /// from the natural language `target` description and page DOM.
618    fn resolve_selector(
619        &self,
620        css_override: Option<&str>,
621        target: &str,
622        tab: &Tab,
623    ) -> Result<String, String> {
624        if let Some(explicit) = css_override {
625            return Ok(explicit.to_owned());
626        }
627
628        let dom_info = extract_dom_info(tab)?;
629        let page_content = get_page_text(tab);
630
631        let system = concat!(
632            "You are a browser automation selector generator. ",
633            "Given a web page's content and interactive elements, ",
634            "return ONLY the best CSS selector for the described element. ",
635            "Output nothing except the CSS selector. ",
636            "Prefer selectors in this order: #id, [data-testid=\"...\"], ",
637            "[name=\"...\"], tag.class, tag. ",
638            "Never output explanations, markdown, or extra text."
639        );
640
641        let user = format!(
642            "Page URL: {}\nPage Title: {}\n\nPage body text (first 4000 chars):\n{}\n\nInteractive elements:\n{}\n\nFind the CSS selector for: {}",
643            page_content.url,
644            page_content.title,
645            truncate(&page_content.body_text, 4000),
646            dom_info,
647            target,
648        );
649
650        eprintln!("      LLM targeting: {target}");
651
652        let (llm, sys, user_prompt) = (
653            self.llm.clone(),
654            system.to_owned(),
655            user,
656        );
657        let selector = std::thread::spawn(move || {
658            let rt = tokio::runtime::Builder::new_current_thread()
659                .enable_all()
660                .build()
661                .unwrap();
662            rt.block_on(llm_chat(&llm, &sys, &user_prompt))
663        })
664        .join()
665        .unwrap()
666        .ok_or_else(|| "LLM element targeting failed (server down?)".to_owned())?;
667
668        let clean = selector
669            .trim()
670            .trim_matches('"')
671            .trim_matches('\'')
672            .trim_matches('`')
673            .to_owned();
674        eprintln!("      resolved selector: {clean}");
675
676        Ok(clean)
677    }
678}
679
680// ── Free helper functions ──────────────────────────────────────────────
681
682fn run_navigate_step(full_url: &str, tab: &Tab) -> StepResult {
683    let name = format!("[navigate] {full_url}");
684    match tab.navigate_to(full_url) {
685        Ok(_) => {
686            let _ = tab.wait_until_navigated();
687            StepResult {
688                name,
689                status: StepStatus::Passed,
690                message: format!("navigated to {full_url}"),
691            }
692        }
693        Err(e) => StepResult {
694            name,
695            status: StepStatus::Failed,
696            message: format!("navigation failed: {e}"),
697        },
698    }
699}
700
701fn extract_dom_info(tab: &Tab) -> Result<String, String> {
702    let result = tab
703        .evaluate(DOM_EXTRACT_JS, false)
704        .map_err(|e| format!("DOM extraction failed: {e}"))?;
705
706    let json_str = result
707        .value
708        .as_ref()
709        .and_then(|v| v.as_str())
710        .unwrap_or("[]");
711
712    let elements: Vec<String> = serde_json::from_str(json_str).unwrap_or_default();
713
714    if elements.is_empty() {
715        return Ok("(no interactive elements found)".to_owned());
716    }
717
718    Ok(elements.join("\n"))
719}
720
721fn get_page_text(tab: &Tab) -> PageContent {
722    let url = tab.get_url();
723
724    let title = tab
725        .evaluate("document.title", false)
726        .ok()
727        .and_then(|r| r.value)
728        .and_then(|v| v.as_str().map(String::from))
729        .unwrap_or_else(|| "unknown".to_owned());
730
731    let body_text = tab
732        .evaluate("document.body ? document.body.innerText : ''", false)
733        .ok()
734        .and_then(|r| r.value)
735        .and_then(|v| v.as_str().map(String::from))
736        .unwrap_or_default();
737
738    PageContent {
739        url,
740        title,
741        body_text: truncate(&body_text, 8000),
742    }
743}
744
745fn resolve_url(url: &str, base_url: &str) -> String {
746    if url.starts_with("http://") || url.starts_with("https://") {
747        return url.to_owned();
748    }
749    let base = base_url.trim_end_matches('/');
750    if url.starts_with('/') {
751        format!("{base}{url}")
752    } else {
753        format!("{base}/{url}")
754    }
755}
756
757// ── Support types ──────────────────────────────────────────────────────
758
759struct TestRunResult {
760    passed: u32,
761    failed: u32,
762    skipped: u32,
763    total: u32,
764    details: Vec<StepResult>,
765}
766
767struct PageContent {
768    url: String,
769    title: String,
770    body_text: String,
771}
llm_browser_testkit/runner.rs

llm_browser_testkit/
runner.rs
