llm_browser_testkit/
diagnostics.rs

1//! Failure diagnostics — page-state capture and artifact writing.
2//!
3//! When a step fails, the runner captures the current page state (URL,
4//! title, visible text, error-ish alert elements) and saves a screenshot.
5//! The formatted context is appended to step messages so CI logs answer the
6//! classic "why did the wait time out?" question instead of printing a bare
7//! timeout message.
8
9use std::fmt::Write as _;
10use std::path::Path;
11
12use headless_chrome::{protocol::cdp::Page::CaptureScreenshotFormatOption, Tab};
13
14use crate::truncate;
15
16/// Maximum length of visible text kept in the in-message excerpt.
17const EXCERPT_LEN: usize = 160;
18/// Maximum length of visible text kept for the printed diagnostics block.
19const FULL_TEXT_LEN: usize = 1500;
20
21/// JavaScript that collects visible "alert-like" elements — error banners,
22/// toast/snackbars, validation messages, error cards.
23pub const DIAGNOSTIC_ALERTS_JS: &str = r#"
24(() => {
25  const sel = '[role="alert"], [aria-live="assertive"], .alert, .error, .error-message, .errorMessage, .error-text, [data-error], [class*="error"], [class*="Error"], mat-snack-bar-container, .mat-mdc-snack-bar-container, .toast, .notification';
26  const els = document.querySelectorAll(sel);
27  const out = [];
28  const seen = new Set();
29  els.forEach((el) => {
30    if (el.offsetParent === null && !el.closest('mat-snack-bar-container')) return;
31    const text = (el.innerText || el.textContent || '').trim();
32    if (!text || seen.has(text)) return;
33    seen.add(text);
34    out.push(text.substring(0, 200));
35  });
36  return JSON.stringify(out.slice(0, 6));
37})()
38"#;
39
40/// Snapshot of the page at the moment a step failed.
41pub struct PageState {
42    /// Current URL.
43    pub url: String,
44    /// `document.title`.
45    pub title: String,
46    /// Visible body text (truncated).
47    pub visible_text: String,
48    /// Text of visible error/alert elements (truncated, deduplicated).
49    pub alerts: Vec<String>,
50}
51
52/// Captures the current page state via CDP evaluation. Never fails — every
53/// extractor degrades to a default value so diagnostic capture cannot mask
54/// the original step failure.
55#[must_use]
56pub fn capture(tab: &Tab) -> PageState {
57    let url = tab.get_url();
58    let title = tab
59        .evaluate("document.title", false)
60        .ok()
61        .and_then(|r| r.value)
62        .and_then(|v| v.as_str().map(String::from))
63        .unwrap_or_else(|| "unknown".to_owned());
64
65    let visible_text = tab
66        .evaluate(
67            "document.body ? document.body.innerText : document.documentElement.innerText",
68            false,
69        )
70        .ok()
71        .and_then(|r| r.value)
72        .and_then(|v| v.as_str().map(String::from))
73        .unwrap_or_default();
74
75    let alerts = tab
76        .evaluate(DIAGNOSTIC_ALERTS_JS, false)
77        .ok()
78        .and_then(|r| r.value)
79        .and_then(|v| v.as_str().map(String::from))
80        .and_then(|json| serde_json::from_str::<Vec<String>>(&json).ok())
81        .unwrap_or_default();
82
83    PageState {
84        url,
85        title,
86        visible_text: truncate(&visible_text, FULL_TEXT_LEN),
87        alerts,
88    }
89}
90
91/// Compact one-line excerpt of the page state, appended to `StepResult`
92/// messages so the summary line in CI logs stays self-contained.
93#[must_use]
94pub fn inline_excerpt(state: &PageState) -> String {
95    let mut out = format!("page: {} ({})", state.url, state.title);
96    let trimmed = state.visible_text.trim();
97    if !trimmed.is_empty() {
98        let excerpt = truncate(trimmed, EXCERPT_LEN);
99        let one_line = excerpt.replace('\n', " ⏎ ");
100        let _ = write!(out, " — visible: \"{one_line}\"");
101    }
102    if !state.alerts.is_empty() {
103        let _ = write!(out, " — alerts: {}", state.alerts.join(" | "));
104    }
105    out
106}
107
108/// Multi-line diagnostics block printed to stderr after a step failure.
109#[must_use]
110pub fn full_context(state: &PageState, screenshot: Option<&str>) -> String {
111    let mut lines = Vec::new();
112    lines.push(format!("    │ url:      {}", state.url));
113    lines.push(format!("    │ title:    {}", state.title));
114    if !state.alerts.is_empty() {
115        for alert in &state.alerts {
116            lines.push(format!("    │ alert:    {alert}"));
117        }
118    }
119    let trimmed = state.visible_text.trim();
120    if !trimmed.is_empty() {
121        lines.push(format!("    │ content:  {trimmed}"));
122    }
123    if let Some(path) = screenshot {
124        lines.push(format!("    📸 screenshot: {path}"));
125    }
126    lines.join("\n")
127}
128
129/// Sanitizes a test/step name for use in artifact file names.
130#[must_use]
131pub fn slugify(name: &str, max_len: usize) -> String {
132    let mut slug: String = name
133        .chars()
134        .map(|c| {
135            if c.is_ascii_alphanumeric() {
136                c.to_ascii_lowercase()
137            } else {
138                '-'
139            }
140        })
141        .collect();
142    while slug.contains("--") {
143        slug = slug.replace("--", "-");
144    }
145    let slug = slug.trim_matches('-');
146    if slug.len() > max_len {
147        // Slug is pure ASCII (letters/digits/dashes), so byte slicing is
148        // safe and keeps the file name at exactly max_len characters.
149        slug[..max_len].to_owned()
150    } else {
151        slug.to_owned()
152    }
153}
154
155/// Saves a PNG screenshot of the current tab under `dir`, named after the
156/// scenario/test/step. Returns the absolute-ish path, or `None` when the
157/// screenshot fails (best effort — never fails the step).
158#[must_use]
159pub fn save_screenshot(
160    tab: &Tab,
161    dir: &Path,
162    scenario: &str,
163    test: &str,
164    step_index: usize,
165    step_kind: &str,
166) -> Option<String> {
167    let data = tab
168        .capture_screenshot(CaptureScreenshotFormatOption::Png, None, None, true)
169        .ok()?;
170    let file_name = format!(
171        "{scenario}__{test}__{step_index:03}-{step_kind}.png",
172        scenario = slugify(scenario, 40),
173        test = slugify(test, 40),
174    );
175    let path = dir.join(file_name);
176    if let Some(parent) = path.parent() {
177        if std::fs::create_dir_all(parent).is_err() {
178            return None;
179        }
180    }
181    if std::fs::write(&path, &data).is_err() {
182        return None;
183    }
184    Some(path.to_string_lossy().into_owned())
185}
186
187#[cfg(test)]
188mod tests {
189    use super::{full_context, inline_excerpt, slugify};
190    use crate::diagnostics::PageState;
191
192    fn state() -> PageState {
193        PageState {
194            url: "http://127.0.0.1:8082/auth/login".into(),
195            title: "Immosai — Sign in".into(),
196            visible_text: "Email address\nPassword\nSign in\nInvalid credentials.".into(),
197            alerts: vec!["Invalid credentials.".into()],
198        }
199    }
200
201    #[test]
202    fn test_inline_excerpt_contains_url_and_text() {
203        let s = inline_excerpt(&state());
204        assert!(s.contains("auth/login"));
205        assert!(s.contains("visible:"));
206        assert!(s.contains("alerts:"));
207    }
208
209    #[test]
210    fn test_inline_excerpt_newlines_flattened() {
211        let s = inline_excerpt(&state());
212        assert!(!s.contains('\n'));
213    }
214
215    #[test]
216    fn test_full_context_lists_alerts_and_screenshot() {
217        let s = full_context(&state(), Some("artifacts/x.png"));
218        assert!(s.contains("url:"));
219        assert!(s.contains("alert:"));
220        assert!(s.contains("screenshot: artifacts/x.png"));
221    }
222
223    #[test]
224    fn test_slugify() {
225        assert_eq!(
226            slugify("Login and open account", 60),
227            "login-and-open-account"
228        );
229        assert_eq!(slugify("Äpfel & Birnen!", 60), "pfel-birnen");
230        assert_eq!(slugify("a".repeat(100).as_str(), 20), "a".repeat(20));
231    }
232}
llm_browser_testkit/diagnostics.rs

llm_browser_testkit/
diagnostics.rs
