use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};
use serde_json::{json, Value};
use tracing::debug;
use crate::tools::page_controller::PageControlTool;
use crate::tools::Tool;
use super::recorder::{ActionOutcome, InteractionRecorder, InteractionTrace, TaskOutcome};
use super::tasks::{ScrollDirection, SuccessCriterion, WebAction, WebTask};
pub fn web_action_to_command(action: &WebAction, screenshot_dir: &Path) -> Value {
match action {
WebAction::Navigate { url } => json!({
"action": "goto",
"url": url,
"wait_until": "load"
}),
WebAction::Click { selector } => json!({
"action": "click",
"selector": selector
}),
WebAction::Fill { selector, value } => json!({
"action": "fill",
"selector": selector,
"text": value
}),
WebAction::Extract { selector, .. } => json!({
"action": "text",
"selector": selector
}),
WebAction::Screenshot { label } => {
let path = screenshot_dir.join(format!("{label}.png"));
json!({
"action": "screenshot",
"path": path.to_string_lossy(),
"full_page": true
})
}
WebAction::WaitFor {
selector,
timeout_ms,
} => json!({
"action": "wait_for",
"selector": selector,
"state": "visible",
"timeout_ms": timeout_ms
}),
WebAction::Scroll { direction, amount } => {
let (dx, dy) = match direction {
ScrollDirection::Up => (0, -amount),
ScrollDirection::Down => (0, *amount),
ScrollDirection::Left => (-*amount, 0),
ScrollDirection::Right => (*amount, 0),
};
json!({
"action": "evaluate",
"expression": format!("window.scrollBy({dx}, {dy})")
})
}
WebAction::Press { key } => json!({
"action": "press",
"key": key
}),
WebAction::Hover { selector } => json!({
"action": "hover",
"selector": selector
}),
}
}
fn result_to_string(v: &Value) -> String {
match v {
Value::String(s) => s.clone(),
Value::Null => String::new(),
Value::Object(_) => {
if let Some(s) = v.get("text").and_then(|t| t.as_str()) {
s.to_string()
} else if let Some(s) = v.get("url").and_then(|t| t.as_str()) {
s.to_string()
} else if let Some(arr) = v.get("texts").and_then(|t| t.as_array()) {
arr.iter()
.filter_map(|e| e.as_str())
.collect::<Vec<_>>()
.join("\n")
} else {
serde_json::to_string(v).unwrap_or_default()
}
}
_ => serde_json::to_string(v).unwrap_or_default(),
}
}
fn result_is_visible(result: &Value) -> bool {
result
.as_bool()
.or_else(|| result.get("visible").and_then(|b| b.as_bool()))
.unwrap_or(false)
}
fn truncate(s: &str, max_len: usize) -> String {
if s.chars().count() > max_len {
let truncated: String = s.chars().take(max_len).collect();
format!("{truncated}...")
} else {
s.to_string()
}
}
pub struct BrowserTaskExecutor {
screenshot_dir: PathBuf,
}
impl BrowserTaskExecutor {
pub fn new(screenshot_dir: PathBuf) -> anyhow::Result<Self> {
std::fs::create_dir_all(&screenshot_dir)?;
Ok(Self { screenshot_dir })
}
pub async fn execute(&self, task: &WebTask) -> InteractionTrace {
let task_screenshot_dir = self.screenshot_dir.join(&task.id);
let _ = std::fs::create_dir_all(&task_screenshot_dir);
let mut recorder = InteractionRecorder::new(&task.id, &task.name, task_screenshot_dir);
let tool = PageControlTool::new();
let mut task_failed = false;
let mut failure_reasons = Vec::new();
let task_start = Instant::now();
let timeout = Duration::from_secs(task.timeout_secs);
for action in &task.actions {
if task_start.elapsed() > timeout {
recorder.record_action(
action.clone(),
ActionOutcome::Timeout,
task_start.elapsed().as_millis() as u64,
None,
None,
);
task_failed = true;
failure_reasons.push("Task timeout exceeded".into());
break;
}
let command = web_action_to_command(action, recorder.screenshot_dir());
let screenshot_path: Option<PathBuf> = command
.get("path")
.and_then(|p| p.as_str())
.map(PathBuf::from);
let remaining = timeout.saturating_sub(task_start.elapsed());
let action_start = Instant::now();
let outcome = match tokio::time::timeout(remaining, tool.execute(command)).await {
Err(_elapsed) => ActionOutcome::Timeout,
Ok(Err(e)) => ActionOutcome::Failed {
error: e.to_string(),
},
Ok(Ok(v)) => {
let success = v.get("success").and_then(|s| s.as_bool());
match success {
Some(true) => {
if let WebAction::Extract { expected, .. } = action {
let result_text = result_to_string(&v["result"]);
if result_text
.to_lowercase()
.contains(&expected.to_lowercase())
{
ActionOutcome::Success {
output: truncate(&result_text, 200),
}
} else {
ActionOutcome::Failed {
error: format!("expected '{expected}' not found"),
}
}
} else {
ActionOutcome::Success {
output: truncate(&result_to_string(&v["result"]), 200),
}
}
}
Some(false) | None => ActionOutcome::Failed {
error: v
.get("error")
.and_then(|e| e.as_str())
.unwrap_or("browser action failed")
.to_string(),
},
}
}
};
let screenshot_after = match (action, &outcome) {
(WebAction::Screenshot { label }, ActionOutcome::Success { .. }) => {
if let Some(ref p) = screenshot_path {
let dims = image::image_dimensions(p).unwrap_or((0, 0));
recorder.record_screenshot(label, p.clone(), dims);
}
screenshot_path.clone()
}
_ => None,
};
let should_break = match &outcome {
ActionOutcome::Timeout => {
task_failed = true;
failure_reasons.push("Task timeout exceeded".into());
true
}
ActionOutcome::Failed { error } => {
debug!(
task_id = %task.id,
action = ?action,
error,
"Browser action failed"
);
task_failed = true;
failure_reasons.push(format!("{action:?} failed: {error}"));
true
}
ActionOutcome::Success { .. } => false,
};
let duration_ms = action_start.elapsed().as_millis() as u64;
recorder.record_action(action.clone(), outcome, duration_ms, None, screenshot_after);
if should_break {
break;
}
}
if !task_failed {
for criterion in &task.success_criteria {
if matches!(criterion, SuccessCriterion::VisualSimilarity { .. }) {
task_failed = true;
failure_reasons
.push("VisualSimilarity not supported in browser executor".into());
continue;
}
let remaining = timeout.saturating_sub(task_start.elapsed());
if remaining.is_zero() {
task_failed = true;
failure_reasons.push("Task timeout exceeded during criteria evaluation".into());
break;
}
let met = match tokio::time::timeout(
remaining,
evaluate_criterion_browser(criterion, &tool),
)
.await
{
Ok(met) => met,
Err(_) => {
task_failed = true;
failure_reasons.push("Criterion evaluation timeout".into());
false
}
};
if !met {
task_failed = true;
failure_reasons.push(format!("Criterion not met: {criterion:?}"));
}
}
}
let final_outcome = if task_failed {
if failure_reasons.iter().any(|r| r.contains("timeout")) {
TaskOutcome::Timeout
} else {
TaskOutcome::Failed {
reasons: failure_reasons,
}
}
} else {
TaskOutcome::Passed
};
let _ = tool.shutdown().await;
recorder.finish(final_outcome)
}
pub async fn execute_all(
&self,
tasks: &[WebTask],
concurrency: usize,
) -> Vec<InteractionTrace> {
use futures::stream::StreamExt;
futures::stream::iter(tasks.iter().map(|task| self.execute(task)))
.buffered(concurrency.max(1))
.collect()
.await
}
}
async fn evaluate_criterion_browser(criterion: &SuccessCriterion, tool: &PageControlTool) -> bool {
match criterion {
SuccessCriterion::UrlContains(s) => match tool.execute(json!({"action": "url"})).await {
Ok(v) => {
let text = result_to_string(&v["result"]);
text.to_lowercase().contains(&s.to_lowercase())
}
Err(_) => false,
},
SuccessCriterion::PageContains(s) => {
match tool
.execute(json!({"action": "text", "selector": "body"}))
.await
{
Ok(v) => {
let text = result_to_string(&v["result"]);
text.to_lowercase().contains(&s.to_lowercase())
}
Err(_) => false,
}
}
SuccessCriterion::ElementVisible(sel) => {
match tool
.execute(json!({"action": "visible", "selector": sel}))
.await
{
Ok(v) => {
let success = v.get("success").and_then(|s| s.as_bool()).unwrap_or(false);
success && result_is_visible(&v["result"])
}
Err(_) => false,
}
}
SuccessCriterion::ExtractedDataMatches { expected, .. } => {
match tool
.execute(json!({"action": "text", "selector": "body"}))
.await
{
Ok(v) => {
let text = result_to_string(&v["result"]);
text.to_lowercase().contains(&expected.to_lowercase())
}
Err(_) => false,
}
}
SuccessCriterion::VisualSimilarity { .. } => false,
}
}
#[cfg(test)]
#[path = "../../../tests/unit/bench_harness/computer_control/browser_executor/browser_executor_test.rs"]
mod tests;