llm_browser_testkit/
lib.rs

1//! LLM-driven browser test framework.
2//!
3//! Provides reusable building blocks for browser-based test scenarios:
4//! - Browser client via Chrome `DevTools` Protocol (headless)
5//! - LLM client for natural language element targeting and assertions
6//! - A2A agent integration for agent-to-agent communication
7//! - MCP client/server integration for tool-calling
8//! - Cost tracking, token counting, and budget enforcement
9//! - Declarative TOML scenario runner
10//! - `#[browser_test]` macros for `cargo test` integration
11
12#![allow(
13    clippy::expect_used,
14    clippy::unwrap_used,
15    clippy::panic,
16    clippy::missing_panics_doc
17)]
18
19/// A2A agent protocol client.
20pub mod a2a;
21/// Budget tracking and enforcement.
22pub mod budgets;
23/// Cost calculation, usage tracking, and pricing.
24pub mod costs;
25/// Failure diagnostics — page-state capture and artifact writing.
26pub mod diagnostics;
27/// Endpoint registry and routing resolver.
28pub mod endpoints;
29/// MCP client for connecting to external MCP servers.
30pub mod mcp_client;
31/// MCP server for exposing the framework as an MCP server.
32pub mod mcp_server;
33/// Cost and token report printer.
34pub mod reporting;
35/// Step-by-step scenario executor (navigate, click, type, wait, assert).
36pub mod runner;
37/// Declarative TOML-based test scenario types.
38pub mod scenario;
39/// CSS selector sanitization for LLM-generated selectors.
40pub mod selectors;
41
42/// A2A agent server for accepting agent tasks.
43#[cfg(feature = "a2a-server")]
44pub mod a2a_server;
45
46/// `#[browser_test]` macros for `cargo test` integration.
47#[cfg(feature = "macros")]
48pub mod macros;
49
50use std::collections::HashMap;
51use std::time::Duration;
52
53use serde_json::Value;
54
55pub use costs::LlmResponse;
56pub use costs::LlmUsage;
57
58/// Configuration for the LLM client — bundles URL, model, auth, timeouts,
59/// and provider-specific options into a single struct passed everywhere.
60#[derive(Debug, Clone)]
61pub struct LlmConfig {
62    /// OpenAI-compatible API base URL (without trailing `/v1/…`).
63    pub url: String,
64    /// Model name (e.g. `gpt-4o-mini`, `deepseek`).
65    pub model: String,
66    /// API key sent as `Authorization: Bearer <key>`.
67    pub api_key: Option<String>,
68    /// Custom headers appended to every LLM request.
69    pub headers: HashMap<String, String>,
70    /// HTTP timeout.
71    pub timeout: Duration,
72    /// Sampling temperature (0.0–1.0).
73    pub temperature: f64,
74    /// Enable extended thinking / reasoning tokens.
75    /// `None` = don't send any thinking key (provider default).
76    pub thinking: Option<bool>,
77    /// Provider-specific parameters merged into the request body
78    /// (e.g. `effort = "high"` for Anthropic).
79    pub model_params: HashMap<String, Value>,
80}
81
82impl LlmConfig {
83    /// Build a config from environment defaults, falling back to safe
84    /// values when no env vars are set.
85    #[must_use]
86    pub fn from_env() -> Self {
87        Self {
88            url: llm_base_url(),
89            model: llm_model(),
90            api_key: std::env::var("HARNESS_LLM_API_KEY").ok(),
91            headers: parse_headers_env(),
92            timeout: Duration::from_secs(60),
93            temperature: 0.0,
94            thinking: None,
95            model_params: HashMap::new(),
96        }
97    }
98}
99
100/// Parses `HARNESS_LLM_HEADERS` env var (JSON object) into a header map.
101///
102/// Exposed for use by `endpoints.rs` and tests.
103#[must_use]
104pub fn parse_headers_env() -> HashMap<String, String> {
105    let Ok(raw) = std::env::var("HARNESS_LLM_HEADERS") else {
106        return HashMap::new();
107    };
108    let Ok(json) = serde_json::from_str::<Value>(&raw) else {
109        return HashMap::new();
110    };
111    let Some(obj) = json.as_object() else {
112        return HashMap::new();
113    };
114    obj.iter()
115        .filter_map(|(k, v)| v.as_str().map(|s| (k.clone(), s.to_owned())))
116        .collect()
117}
118
119/// Returns the target base URL from `HARNESS_BROWSER_BASE_URL` env,
120/// defaulting to `http://localhost:4200`.
121#[must_use]
122pub fn base_url() -> String {
123    std::env::var("HARNESS_BROWSER_BASE_URL").unwrap_or_else(|_| "http://localhost:4200".to_owned())
124}
125
126/// Returns the LLM server base URL from `HARNESS_LLM_TEST_URL` env,
127/// defaulting to `http://localhost:8080`.
128#[must_use]
129pub fn llm_base_url() -> String {
130    std::env::var("HARNESS_LLM_TEST_URL")
131        .unwrap_or_else(|_| "http://localhost:8080".to_owned())
132        .trim_end_matches('/')
133        .to_owned()
134}
135
136/// Returns the LLM model name from `HARNESS_LLM_TEST_MODEL` env,
137/// defaulting to `deepseek`.
138#[must_use]
139pub fn llm_model() -> String {
140    std::env::var("HARNESS_LLM_TEST_MODEL").unwrap_or_else(|_| "deepseek".to_owned())
141}
142
143/// Returns whether to run the browser in headless mode from
144/// `HARNESS_BROWSER_HEADLESS` env, defaulting to `true`.
145#[must_use]
146pub fn browser_headless() -> bool {
147    std::env::var("HARNESS_BROWSER_HEADLESS")
148        .map_or(true, |v| v != "0" && v.to_lowercase() != "false")
149}
150
151/// Builds a `reqwest::Client` with the given timeout.
152#[must_use]
153pub fn http_client(timeout: Duration) -> reqwest::Client {
154    reqwest::Client::builder()
155        .timeout(timeout)
156        .build()
157        .expect("build reqwest client")
158}
159
160/// Sends a chat completion request to the LLM.
161///
162/// Returns `Some(content)` on success, `None` on any error.
163///
164/// Prefer `llm_chat_with_usage` if you need token counting.
165#[must_use]
166pub async fn llm_chat(llm: &LlmConfig, system: &str, user: &str) -> Option<String> {
167    llm_chat_with_usage(llm, system, user)
168        .await
169        .map(|r| r.content)
170        .ok()
171}
172
173/// Sends a chat completion request to the LLM and returns both the content
174/// and token usage from the API response.
175///
176/// Retries transient failures (network errors, HTTP 429/5xx, invalid
177/// responses) with a short backoff, and returns the last underlying error
178/// instead of collapsing everything into a generic "server down" message.
179/// The error text includes the HTTP status and a truncated response-body
180/// snippet, so a gateway that answers with an HTML error page is
181/// identifiable in CI logs instead of surfacing as a bare JSON decode
182/// error.
183///
184/// # Errors
185///
186/// Returns the last underlying error as a human-readable string when every
187/// attempt fails (transport error, non-success HTTP status, response that is
188/// not valid JSON, or a response missing `choices[0].message.content`).
189pub async fn llm_chat_with_usage(
190    llm: &LlmConfig,
191    system: &str,
192    user: &str,
193) -> Result<LlmResponse, String> {
194    let client = http_client(llm.timeout);
195    let mut last_err = String::from("LLM call failed");
196    let mut attempts: u32 = 0;
197
198    while attempts < LLM_CALL_ATTEMPTS {
199        attempts += 1;
200        match llm_chat_once(&client, llm, system, user).await {
201            Ok(resp) => return Ok(resp),
202            Err(err) => {
203                last_err = err.to_string();
204                if attempts >= LLM_CALL_ATTEMPTS || !err.is_retryable() {
205                    break;
206                }
207                tokio::time::sleep(Duration::from_millis(500 * u64::from(attempts))).await;
208            }
209        }
210    }
211
212    Err(format!(
213        "LLM call failed after {attempts} attempt(s) (endpoint {url}): {last_err}",
214        url = llm.url
215    ))
216}
217
218/// Number of attempts for a single chat completion call.
219const LLM_CALL_ATTEMPTS: u32 = 3;
220
221/// Internal error type for a single LLM request attempt, distinguishing
222/// transient failures (worth retrying) from deterministic configuration
223/// errors (fail immediately).
224enum LlmCallError {
225    /// Transport-level failure (connect, timeout, TLS, …).
226    Transport { message: String },
227    /// Non-success HTTP status with a body snippet.
228    Http { status: u16, body: String },
229    /// Success status but the body is not valid JSON.
230    InvalidJson {
231        status: u16,
232        detail: String,
233        body: String,
234    },
235    /// Valid JSON but missing `choices[0].message.content`.
236    MissingContent { json: String },
237}
238
239impl std::fmt::Display for LlmCallError {
240    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
241        match self {
242            Self::Transport { message } => write!(f, "LLM HTTP request failed: {message}"),
243            Self::Http { status, body } => {
244                write!(
245                    f,
246                    "LLM endpoint returned HTTP {status}: {}",
247                    truncate(body, 300)
248                )
249            }
250            Self::InvalidJson {
251                status,
252                detail,
253                body,
254            } => write!(
255                f,
256                "LLM endpoint returned HTTP {status} with non-JSON body ({detail}): {}",
257                truncate(body, 300)
258            ),
259            Self::MissingContent { json } => write!(
260                f,
261                "LLM response missing choices[0].message.content: {}",
262                truncate(json, 300)
263            ),
264        }
265    }
266}
267
268impl LlmCallError {
269    /// Whether another attempt may succeed. Network errors, rate limits,
270    /// server errors, and 200-with-garbage responses can be transient on
271    /// flaky gateways; auth/not-found errors are deterministic.
272    #[must_use]
273    fn is_retryable(&self) -> bool {
274        match self {
275            Self::Transport { .. } | Self::MissingContent { .. } => true,
276            Self::Http { status, .. } => {
277                *status == 408 || *status == 429 || (500..600).contains(status)
278            }
279            Self::InvalidJson { status, .. } => {
280                *status == 200 || *status == 408 || *status == 429 || (500..600).contains(status)
281            }
282        }
283    }
284}
285
286/// Single LLM chat request attempt; returns the underlying error as text.
287async fn llm_chat_once(
288    client: &reqwest::Client,
289    llm: &LlmConfig,
290    system: &str,
291    user: &str,
292) -> Result<LlmResponse, LlmCallError> {
293    let mut payload = serde_json::json!({
294        "model": llm.model,
295        "messages": [
296            {"role": "system", "content": system},
297            {"role": "user", "content": user}
298        ],
299        "max_tokens": 4096,
300        "temperature": llm.temperature
301    });
302    if let Some(think) = llm.thinking {
303        if think {
304            payload["thinking"] = serde_json::json!({"type": "enabled"});
305        } else {
306            payload["thinking"] = serde_json::json!({"type": "disabled"});
307        }
308    }
309    // Merge provider-specific parameters into the request body.
310    if !llm.model_params.is_empty() {
311        if let Value::Object(ref mut map) = payload {
312            for (key, val) in &llm.model_params {
313                map.insert(key.clone(), val.clone());
314            }
315        }
316    }
317
318    let mut req = client
319        .post(format!("{}/v1/chat/completions", llm.url))
320        .header("Content-Type", "application/json");
321
322    if let Some(ref key) = llm.api_key {
323        req = req.header("Authorization", format!("Bearer {key}"));
324    }
325    for (name, value) in &llm.headers {
326        req = req.header(name.as_str(), value.as_str());
327    }
328
329    let resp = req
330        .json(&payload)
331        .send()
332        .await
333        .map_err(|e| LlmCallError::Transport {
334            message: e.to_string(),
335        })?;
336    let status = resp.status();
337    let status_u16 = status.as_u16();
338    let body = resp.text().await.unwrap_or_default();
339    if !status.is_success() {
340        return Err(LlmCallError::Http {
341            status: status_u16,
342            body,
343        });
344    }
345    let json: Value = match serde_json::from_str(&body) {
346        Ok(v) => v,
347        Err(e) => {
348            return Err(LlmCallError::InvalidJson {
349                status: status_u16,
350                detail: e.to_string(),
351                body,
352            });
353        }
354    };
355    let usage = costs::extract_usage(&json);
356    let content = json["choices"][0]["message"]["content"]
357        .as_str()
358        .map(String::from)
359        .ok_or_else(|| LlmCallError::MissingContent {
360            json: json.to_string(),
361        })?;
362
363    Ok(LlmResponse { content, usage })
364}
365
366/// JavaScript to extract interactive elements from the current page.
367/// Returns a JSON array of objects with tag, selector, and label.
368pub const DOM_EXTRACT_JS: &str = r#"
369(() => {
370  const interactive = 'a, button, input, textarea, select, [role="button"], [onclick], [tabindex], [data-testid], [aria-label]';
371  const els = document.querySelectorAll(interactive);
372  const info = [];
373  const seen = new Set();
374  els.forEach((el, i) => {
375    const rect = el.getBoundingClientRect();
376    if (rect.width === 0 || rect.height === 0) return;
377    const tag = el.tagName.toLowerCase();
378    let selector = '';
379    if (el.id) selector = '#' + CSS.escape(el.id);
380    else if (el.getAttribute('data-testid')) selector = '[data-testid="' + el.getAttribute('data-testid') + '"]';
381    else if (el.name) selector = '[name="' + CSS.escape(el.name) + '"]';
382    else if (el.className && typeof el.className === 'string') {
383      const cls = el.className.trim().split(/\\s+/)[0];
384      if (cls) selector = tag + '.' + CSS.escape(cls);
385    }
386    if (!selector) selector = tag;
387    if (seen.has(selector)) return;
388    seen.add(selector);
389
390    let label = '';
391    const aria = el.getAttribute('aria-label');
392    if (aria) {
393      label = aria;
394    } else if (tag === 'input' || tag === 'textarea' || tag === 'select') {
395      label = el.placeholder || el.name || el.getAttribute('aria-label') || '';
396      if (el.type && !label) label = el.type;
397    } else {
398      label = (el.textContent || '').trim().substring(0, 80);
399    }
400
401    info.push(i + ': ' + selector + ' [' + tag + '] "' + label + '"');
402  });
403  return JSON.stringify(info);
404})()
405"#;
406
407/// Truncates a string to the given maximum length, appending a marker
408/// if truncation occurred.
409#[must_use]
410pub fn truncate(s: &str, max_len: usize) -> String {
411    if s.len() <= max_len {
412        s.to_owned()
413    } else {
414        format!("{}...<truncated>", &s[..max_len])
415    }
416}
417
418#[cfg(test)]
419mod tests {
420    use crate::costs::extract_usage;
421    use crate::truncate;
422    use crate::{llm_base_url, llm_model, parse_headers_env, LlmConfig};
423
424    #[test]
425    fn test_truncate_short() {
426        assert_eq!(truncate("hello", 10), "hello");
427    }
428
429    #[test]
430    fn test_truncate_long() {
431        let result = truncate("hello world", 5);
432        assert!(result.contains("<truncated>"));
433        assert!(result.starts_with("hello"));
434    }
435
436    #[test]
437    fn test_truncate_exact_length() {
438        assert_eq!(truncate("abcde", 5), "abcde");
439    }
440
441    #[test]
442    fn test_truncate_empty() {
443        assert_eq!(truncate("", 5), "");
444    }
445
446    #[test]
447    fn test_parse_headers_env_empty() {
448        std::env::remove_var("HARNESS_LLM_HEADERS");
449        let h = parse_headers_env();
450        assert!(h.is_empty());
451    }
452
453    #[test]
454    fn test_parse_headers_env_valid() {
455        std::env::set_var("HARNESS_LLM_HEADERS", r#"{"X-Org":"acme","X-Version":"1"}"#);
456        let h = parse_headers_env();
457        assert_eq!(h.get("X-Org").map(String::as_str), Some("acme"));
458        assert_eq!(h.get("X-Version").map(String::as_str), Some("1"));
459        std::env::remove_var("HARNESS_LLM_HEADERS");
460    }
461
462    #[test]
463    fn test_parse_headers_env_invalid_json() {
464        std::env::set_var("HARNESS_LLM_HEADERS", "not-json");
465        let h = parse_headers_env();
466        assert!(h.is_empty());
467        std::env::remove_var("HARNESS_LLM_HEADERS");
468    }
469
470    #[test]
471    fn test_llm_config_from_env_defaults() {
472        #[allow(clippy::float_cmp)]
473        {
474            let config = LlmConfig::from_env();
475            assert_eq!(config.temperature, 0.0);
476            assert!(config.thinking.is_none());
477            assert!(config.model_params.is_empty());
478        }
479    }
480
481    #[test]
482    fn test_extract_usage_full() {
483        let json = serde_json::json!({
484            "usage": {
485                "prompt_tokens": 100,
486                "completion_tokens": 200,
487                "total_tokens": 300
488            }
489        });
490        let usage = extract_usage(&json);
491        assert_eq!(usage.prompt_tokens, 100);
492        assert_eq!(usage.completion_tokens, 200);
493        assert_eq!(usage.total_tokens, 300);
494    }
495
496    #[test]
497    fn test_extract_usage_empty() {
498        let json = serde_json::json!({});
499        let usage = extract_usage(&json);
500        assert_eq!(usage.prompt_tokens, 0);
501        assert_eq!(usage.completion_tokens, 0);
502        assert_eq!(usage.total_tokens, 0);
503    }
504
505    #[test]
506    fn test_truncate_unicode() {
507        // truncate uses byte-level slicing, so max_len refers to byte count
508        // 'é' is 2 bytes, so index 3 captures "hé" (h=0, é=bytes 1-2)
509        assert_eq!(truncate("héllo", 3), "hé...<truncated>");
510        // Length 5 captures full string (5 bytes)
511        assert_eq!(truncate("hello", 5), "hello");
512    }
513
514    #[test]
515    fn test_parse_headers_env_non_object() {
516        std::env::set_var("HARNESS_LLM_HEADERS", "[1, 2, 3]");
517        let h = parse_headers_env();
518        assert!(h.is_empty());
519        std::env::remove_var("HARNESS_LLM_HEADERS");
520    }
521
522    #[test]
523    fn test_parse_headers_env_nested_values_filtered() {
524        std::env::set_var(
525            "HARNESS_LLM_HEADERS",
526            r#"{"str":"val","num":42,"bool":true}"#,
527        );
528        let h = parse_headers_env();
529        assert_eq!(h.get("str").map(String::as_str), Some("val"));
530        assert!(!h.contains_key("num"));
531        assert!(!h.contains_key("bool"));
532        std::env::remove_var("HARNESS_LLM_HEADERS");
533    }
534
535    #[test]
536    fn test_llm_config_has_default_model() {
537        let config = LlmConfig::from_env();
538        assert!(!config.model.is_empty());
539    }
540
541    #[test]
542    fn test_llm_base_url_default() {
543        std::env::remove_var("HARNESS_LLM_TEST_URL");
544        let url = llm_base_url();
545        assert_eq!(url, "http://localhost:8080");
546    }
547
548    #[test]
549    fn test_llm_base_url_custom() {
550        std::env::set_var("HARNESS_LLM_TEST_URL", "https://custom.api.com/v1");
551        let url = llm_base_url();
552        assert_eq!(url, "https://custom.api.com/v1");
553        std::env::remove_var("HARNESS_LLM_TEST_URL");
554    }
555
556    #[test]
557    fn test_llm_base_url_trailing_slash() {
558        std::env::set_var("HARNESS_LLM_TEST_URL", "https://api.com/");
559        let url = llm_base_url();
560        assert_eq!(url, "https://api.com");
561        std::env::remove_var("HARNESS_LLM_TEST_URL");
562    }
563
564    #[test]
565    fn test_llm_model_default() {
566        std::env::remove_var("HARNESS_LLM_TEST_MODEL");
567        assert_eq!(llm_model(), "deepseek");
568    }
569
570    #[test]
571    fn test_llm_model_custom() {
572        std::env::set_var("HARNESS_LLM_TEST_MODEL", "gpt-4o");
573        assert_eq!(llm_model(), "gpt-4o");
574        std::env::remove_var("HARNESS_LLM_TEST_MODEL");
575    }
576
577    #[test]
578    fn test_extract_usage_partial() {
579        let json = serde_json::json!({
580            "usage": {
581                "prompt_tokens": 50
582            }
583        });
584        let usage = extract_usage(&json);
585        assert_eq!(usage.prompt_tokens, 50);
586        assert_eq!(usage.completion_tokens, 0);
587        assert_eq!(usage.total_tokens, 0);
588    }
589
590    #[test]
591    fn test_browser_headless_default() {
592        std::env::remove_var("HARNESS_BROWSER_HEADLESS");
593        assert!(crate::browser_headless());
594    }
595}
llm_browser_testkit/lib.rs

llm_browser_testkit/
lib.rs
