use std::collections::HashMap;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, RwLock};
use async_trait::async_trait;
use car_browser::backend::{BrowserBackend, BrowserError};
use car_browser::models::{A11yNode, Bounds, Modifier, Viewport, WaitCondition};
use car_browser::perception::pipeline::BasicPerceptionPipeline;
use car_browser::perception::PerceptionPipeline;
use car_browser::BrowserToolExecutor;
use car_engine::{Runtime, ToolExecutor};
use car_ir::{Action, ActionProposal, ActionStatus, ActionType, FailureBehavior};
use serde_json::json;
struct MockBackend {
url: RwLock<String>,
title: RwLock<String>,
click_count: AtomicU64,
typed_text: RwLock<Vec<(String, String)>>, scroll_total: AtomicU64,
keypresses: RwLock<Vec<(String, Vec<Modifier>)>>, }
impl MockBackend {
fn new() -> Self {
Self {
url: RwLock::new("https://example.com".into()),
title: RwLock::new("Example Domain".into()),
click_count: AtomicU64::new(0),
typed_text: RwLock::new(Vec::new()),
scroll_total: AtomicU64::new(0),
keypresses: RwLock::new(Vec::new()),
}
}
fn click_count(&self) -> u64 {
self.click_count.load(Ordering::SeqCst)
}
fn typed_entries(&self) -> Vec<(String, String)> {
self.typed_text.read().unwrap().clone()
}
fn keypress_entries(&self) -> Vec<(String, Vec<Modifier>)> {
self.keypresses.read().unwrap().clone()
}
}
fn mock_a11y_tree() -> Vec<A11yNode> {
vec![
A11yNode {
node_id: "ax_root".into(),
role: "document".into(),
name: Some("Example Page".into()),
value: None,
bounds: Bounds::new(0.0, 0.0, 1280.0, 720.0),
children: vec!["ax_heading".into(), "ax_search".into(), "ax_submit".into()],
focusable: false,
focused: false,
disabled: false,
},
A11yNode {
node_id: "ax_heading".into(),
role: "heading".into(),
name: Some("Welcome to Example".into()),
value: None,
bounds: Bounds::new(100.0, 20.0, 400.0, 40.0),
children: vec![],
focusable: false,
focused: false,
disabled: false,
},
A11yNode {
node_id: "ax_search".into(),
role: "textfield".into(),
name: Some("Search".into()),
value: Some("".into()),
bounds: Bounds::new(100.0, 100.0, 300.0, 30.0),
children: vec![],
focusable: true,
focused: false,
disabled: false,
},
A11yNode {
node_id: "ax_submit".into(),
role: "button".into(),
name: Some("Submit".into()),
value: None,
bounds: Bounds::new(420.0, 100.0, 80.0, 30.0),
children: vec![],
focusable: true,
focused: false,
disabled: false,
},
A11yNode {
node_id: "ax_disabled_btn".into(),
role: "button".into(),
name: Some("Disabled Action".into()),
value: None,
bounds: Bounds::new(520.0, 100.0, 100.0, 30.0),
children: vec![],
focusable: false,
focused: false,
disabled: true,
},
]
}
#[async_trait]
impl BrowserBackend for MockBackend {
async fn capture_screenshot(&self) -> Result<Vec<u8>, BrowserError> {
Ok(vec![
0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D, 0x49, 0x48,
0x44, 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x02, 0x00, 0x00,
0x00, 0x90, 0x77, 0x53, 0xDE, 0x00, 0x00, 0x00, 0x0C, 0x49, 0x44, 0x41, 0x54, 0x08,
0xD7, 0x63, 0xF8, 0xFF, 0xFF, 0xFF, 0x00, 0x05, 0xFE, 0x02, 0xFE, 0xDC, 0xCC, 0x59,
0xE7, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4E, 0x44, 0xAE, 0x42, 0x60, 0x82,
])
}
async fn get_accessibility_tree(&self) -> Result<Vec<A11yNode>, BrowserError> {
Ok(mock_a11y_tree())
}
fn get_viewport(&self) -> Result<Viewport, BrowserError> {
Ok(Viewport {
width: 1280,
height: 720,
device_pixel_ratio: 2.0,
})
}
fn get_current_url(&self) -> Result<String, BrowserError> {
Ok(self.url.read().unwrap().clone())
}
async fn get_page_title(&self) -> Result<String, BrowserError> {
Ok(self.title.read().unwrap().clone())
}
async fn navigate(&self, url: &str) -> Result<(), BrowserError> {
*self.url.write().unwrap() = url.to_string();
*self.title.write().unwrap() = format!("Page: {}", url);
Ok(())
}
async fn inject_click(&self, _x: f64, _y: f64) -> Result<(), BrowserError> {
self.click_count.fetch_add(1, Ordering::SeqCst);
Ok(())
}
async fn inject_text(&self, _text: &str) -> Result<(), BrowserError> {
Ok(())
}
async fn inject_keypress(&self, key: &str, modifiers: &[Modifier]) -> Result<(), BrowserError> {
self.keypresses
.write()
.unwrap()
.push((key.to_string(), modifiers.to_vec()));
Ok(())
}
async fn inject_scroll(&self, delta_y: i32) -> Result<(), BrowserError> {
self.scroll_total
.fetch_add(delta_y.unsigned_abs() as u64, Ordering::SeqCst);
Ok(())
}
async fn click_element(&self, node_id: &str) -> Result<(), BrowserError> {
if node_id.starts_with("ax_") {
self.click_count.fetch_add(1, Ordering::SeqCst);
Ok(())
} else {
Err(BrowserError::ElementNotFound(format!(
"Expected AX node ID (ax_*), got: {}",
node_id
)))
}
}
async fn type_into_element(&self, node_id: &str, text: &str) -> Result<(), BrowserError> {
if node_id.starts_with("ax_") {
self.typed_text
.write()
.unwrap()
.push((node_id.to_string(), text.to_string()));
Ok(())
} else {
Err(BrowserError::ElementNotFound(format!(
"Expected AX node ID, got: {}",
node_id
)))
}
}
async fn focus_element(&self, _node_id: &str) -> Result<(), BrowserError> {
Ok(())
}
async fn is_page_loaded(&self) -> Result<bool, BrowserError> {
Ok(true)
}
async fn wait_until(&self, _cond: &WaitCondition, _timeout: u64) -> Result<bool, BrowserError> {
Ok(true)
}
async fn element_exists_a11y(
&self,
name: &str,
_role: Option<&str>,
) -> Result<bool, BrowserError> {
Ok(mock_a11y_tree()
.iter()
.any(|n| n.name.as_deref().map(|s| s.contains(name)).unwrap_or(false)))
}
}
fn make_action(tool: &str, params: serde_json::Value) -> Action {
Action {
id: format!("a_{}", tool),
action_type: ActionType::ToolCall,
tool: Some(tool.to_string()),
parameters: params
.as_object()
.map(|m| m.iter().map(|(k, v)| (k.clone(), v.clone())).collect())
.unwrap_or_default(),
preconditions: vec![],
expected_effects: HashMap::new(),
state_dependencies: vec![],
idempotent: false,
max_retries: 1,
failure_behavior: FailureBehavior::Abort,
timeout_ms: Some(10_000),
metadata: HashMap::new(),
}
}
fn make_proposal(actions: Vec<Action>) -> ActionProposal {
ActionProposal {
id: "test_proposal".into(),
source: "integration_test".into(),
actions,
timestamp: chrono::Utc::now(),
context: HashMap::new(),
}
}
async fn setup_runtime() -> (Runtime, Arc<MockBackend>) {
let backend = Arc::new(MockBackend::new());
let pipeline: Arc<dyn PerceptionPipeline> = Arc::new(BasicPerceptionPipeline::new());
let executor = Arc::new(BrowserToolExecutor::new(
backend.clone() as Arc<dyn BrowserBackend>,
pipeline,
));
let rt = Runtime::new().with_executor(executor as Arc<dyn ToolExecutor>);
for schema in BrowserToolExecutor::tool_schemas() {
rt.register_tool_schema(schema).await;
}
(rt, backend)
}
#[tokio::test]
async fn test_browse_observe_produces_ui_map() {
let (rt, _backend) = setup_runtime().await;
let proposal = make_proposal(vec![make_action("browse_observe", json!({}))]);
let result = rt.execute(&proposal).await;
assert_eq!(result.results.len(), 1);
assert_eq!(result.results[0].status, ActionStatus::Succeeded);
let output = result.results[0].output.as_ref().unwrap();
assert_eq!(output["url"], "https://example.com");
assert!(output["ui_map"].as_str().unwrap().contains("Submit"));
assert!(output["ui_map"].as_str().unwrap().contains("Search"));
assert!(output["element_count"].as_u64().unwrap() >= 3);
}
#[tokio::test]
async fn test_browse_click_resolves_el_id_to_ax_ref() {
let (rt, backend) = setup_runtime().await;
let observe = make_proposal(vec![make_action("browse_observe", json!({}))]);
let obs_result = rt.execute(&observe).await;
assert_eq!(obs_result.results[0].status, ActionStatus::Succeeded);
let ui_map_text = obs_result.results[0].output.as_ref().unwrap()["ui_map"]
.as_str()
.unwrap();
let submit_el = ui_map_text
.lines()
.find(|l| l.starts_with("[el_") && l.contains("Submit"))
.and_then(|l| l.split(']').next())
.map(|s| s.trim_start_matches('[').trim())
.unwrap_or("el_0");
let click = make_proposal(vec![make_action(
"browse_click",
json!({"element_id": submit_el}),
)]);
let click_result = rt.execute(&click).await;
assert_eq!(
click_result.results[0].status,
ActionStatus::Succeeded,
"Click should succeed. Error: {:?}",
click_result.results[0].error
);
assert_eq!(backend.click_count(), 1);
let output = click_result.results[0].output.as_ref().unwrap();
assert!(
output["resolved_id"].as_str().unwrap().starts_with("ax_"),
"Should resolve to AX node ID, got: {}",
output["resolved_id"]
);
}
#[tokio::test]
async fn test_browse_type_resolves_and_types() {
let (rt, backend) = setup_runtime().await;
let observe = make_proposal(vec![make_action("browse_observe", json!({}))]);
rt.execute(&observe).await;
let obs_result = rt
.execute(&make_proposal(vec![make_action(
"browse_observe",
json!({}),
)]))
.await;
let ui_map_text = obs_result.results[0].output.as_ref().unwrap()["ui_map"]
.as_str()
.unwrap();
let search_el = ui_map_text
.lines()
.find(|l| l.starts_with("[el_") && l.contains("Search"))
.and_then(|l| l.split(']').next())
.map(|s| s.trim_start_matches('[').trim())
.unwrap_or("el_0");
let type_action = make_proposal(vec![make_action(
"browse_type",
json!({"element_id": search_el, "text": "hello world"}),
)]);
let result = rt.execute(&type_action).await;
assert_eq!(result.results[0].status, ActionStatus::Succeeded);
let entries = backend.typed_entries();
assert_eq!(entries.len(), 1);
assert_eq!(entries[0].0, "ax_search");
assert_eq!(entries[0].1, "hello world");
}
#[tokio::test]
async fn test_browse_navigate() {
let (rt, backend) = setup_runtime().await;
let nav = make_proposal(vec![make_action(
"browse_navigate",
json!({"url": "https://google.com"}),
)]);
let result = rt.execute(&nav).await;
assert_eq!(result.results[0].status, ActionStatus::Succeeded);
assert_eq!(backend.url.read().unwrap().as_str(), "https://google.com");
}
#[tokio::test]
async fn test_browse_scroll() {
let (rt, backend) = setup_runtime().await;
let scroll = make_proposal(vec![make_action("browse_scroll", json!({"delta_y": 300}))]);
let result = rt.execute(&scroll).await;
assert_eq!(result.results[0].status, ActionStatus::Succeeded);
assert_eq!(backend.scroll_total.load(Ordering::SeqCst), 300);
}
#[tokio::test]
async fn test_full_mission_cycle() {
let (rt, backend) = setup_runtime().await;
let r = rt
.execute(&make_proposal(vec![make_action(
"browse_navigate",
json!({"url": "https://example.com/search"}),
)]))
.await;
assert_eq!(r.results[0].status, ActionStatus::Succeeded);
let r = rt
.execute(&make_proposal(vec![make_action(
"browse_observe",
json!({}),
)]))
.await;
assert_eq!(r.results[0].status, ActionStatus::Succeeded);
let ui_map = r.results[0].output.as_ref().unwrap()["ui_map"]
.as_str()
.unwrap()
.to_string();
let search_el = ui_map
.lines()
.find(|l| l.starts_with("[el_") && l.contains("Search"))
.and_then(|l| l.split(']').next())
.map(|s| s.trim_start_matches('[').trim().to_string())
.unwrap();
let r = rt
.execute(&make_proposal(vec![make_action(
"browse_type",
json!({"element_id": search_el, "text": "rust programming"}),
)]))
.await;
assert_eq!(r.results[0].status, ActionStatus::Succeeded);
let submit_el = ui_map
.lines()
.find(|l| l.starts_with("[el_") && l.contains("Submit"))
.and_then(|l| l.split(']').next())
.map(|s| s.trim_start_matches('[').trim().to_string())
.unwrap();
let r = rt
.execute(&make_proposal(vec![make_action(
"browse_click",
json!({"element_id": submit_el}),
)]))
.await;
assert_eq!(r.results[0].status, ActionStatus::Succeeded);
let r = rt
.execute(&make_proposal(vec![make_action(
"browse_observe",
json!({}),
)]))
.await;
assert_eq!(r.results[0].status, ActionStatus::Succeeded);
assert_eq!(backend.click_count(), 1);
let typed = backend.typed_entries();
assert_eq!(typed.len(), 1);
assert_eq!(typed[0].1, "rust programming");
assert_eq!(
backend.url.read().unwrap().as_str(),
"https://example.com/search"
);
}
#[tokio::test]
async fn test_click_without_observe_falls_back() {
let (rt, _backend) = setup_runtime().await;
let click = make_proposal(vec![make_action(
"browse_click",
json!({"element_id": "el_0"}),
)]);
let result = rt.execute(&click).await;
assert_eq!(result.results[0].status, ActionStatus::Failed);
assert!(result.results[0]
.error
.as_ref()
.unwrap()
.contains("Expected AX node ID"));
}
#[tokio::test]
async fn test_browse_keypress() {
let (rt, backend) = setup_runtime().await;
let observe = make_proposal(vec![make_action("browse_observe", json!({}))]);
let obs_result = rt.execute(&observe).await;
assert_eq!(obs_result.results[0].status, ActionStatus::Succeeded);
let keypress = make_proposal(vec![make_action(
"browse_keypress",
json!({"key": "Enter"}),
)]);
let result = rt.execute(&keypress).await;
assert_eq!(result.results[0].status, ActionStatus::Succeeded);
let output = result.results[0].output.as_ref().unwrap();
assert_eq!(output["key"], "Enter");
assert_eq!(output["status"], "pressed");
let entries = backend.keypress_entries();
assert_eq!(entries.len(), 1);
assert_eq!(entries[0].0, "Enter");
assert!(entries[0].1.is_empty());
}
#[tokio::test]
async fn test_browse_wait() {
let (rt, _backend) = setup_runtime().await;
let wait = make_proposal(vec![make_action(
"browse_wait",
json!({"condition": "page_loaded"}),
)]);
let result = rt.execute(&wait).await;
assert_eq!(result.results[0].status, ActionStatus::Succeeded);
let output = result.results[0].output.as_ref().unwrap();
assert_eq!(output["condition"], "page_loaded");
assert_eq!(output["met"], true);
}
#[tokio::test]
async fn test_unknown_tool_rejected() {
let (rt, _backend) = setup_runtime().await;
let bad = make_proposal(vec![make_action("browse_nonexistent", json!({}))]);
let result = rt.execute(&bad).await;
assert_eq!(result.results[0].status, ActionStatus::Rejected);
}