car_browser/
tools.rs

1//! Browser tool registration and execution for car-engine.
2//!
3//! Registers `browse_*` tools with the CAR runtime and dispatches tool calls
4//! to the appropriate `BrowserBackend` methods.
5
6use std::sync::Arc;
7
8use async_trait::async_trait;
9use car_engine::ToolExecutor;
10use car_ir::ToolSchema;
11use serde_json::{json, Value};
12use tokio::sync::RwLock;
13
14use crate::backend::BrowserBackend;
15use crate::models::{Modifier, WaitCondition};
16use crate::perception::pipeline::PerceptionPipeline;
17use crate::perception::ui_map::UiMap;
18
19/// Tool executor that dispatches `browse_*` tool calls to a `BrowserBackend`.
20///
21/// Maintains a reference to the last UiMap from `browse_observe` to resolve
22/// `el_N` IDs to AX node IDs before passing to the backend.
23pub struct BrowserToolExecutor {
24    backend: Arc<dyn BrowserBackend>,
25    pipeline: Arc<dyn PerceptionPipeline>,
26    /// Last observed UiMap — used to resolve el_N → ax_ref for click/type/focus.
27    last_ui_map: Arc<RwLock<Option<UiMap>>>,
28}
29
30impl BrowserToolExecutor {
31    /// Create a new browser tool executor.
32    pub fn new(backend: Arc<dyn BrowserBackend>, pipeline: Arc<dyn PerceptionPipeline>) -> Self {
33        Self {
34            backend,
35            pipeline,
36            last_ui_map: Arc::new(RwLock::new(None)),
37        }
38    }
39
40    /// Resolve an element ID (el_N) to the backend's AX node ID.
41    ///
42    /// Falls back to the raw ID if no UiMap is available or element not found.
43    async fn resolve_element_id(&self, element_id: &str) -> String {
44        let guard = self.last_ui_map.read().await;
45        if let Some(ui_map) = guard.as_ref() {
46            if let Some(element) = ui_map.get_element(element_id) {
47                if let Some(ref ax_ref) = element.ax_ref {
48                    return ax_ref.clone();
49                }
50            }
51        }
52        // Fallback: pass through as-is (may be a raw AX node ID already)
53        element_id.to_string()
54    }
55
56    /// Get the tool schemas for all browser tools.
57    pub fn tool_schemas() -> Vec<ToolSchema> {
58        vec![
59            ToolSchema {
60                name: "browse_navigate".to_string(),
61                source: car_ir::ToolSourceKind::Builtin,
62                description: "Navigate the browser to a URL".to_string(),
63                parameters: json!({
64                    "type": "object",
65                    "properties": {
66                        "url": { "type": "string", "description": "URL to navigate to" }
67                    },
68                    "required": ["url"]
69                }),
70                returns: Some(json!({"type": "object", "properties": {"url": {"type": "string"}}})),
71                idempotent: false,
72                cache_ttl_secs: None,
73                rate_limit: None,
74            },
75            ToolSchema {
76                name: "browse_click".to_string(),
77                source: car_ir::ToolSourceKind::Builtin,
78                description: "Click on a UI element by accessibility node ID".to_string(),
79                parameters: json!({
80                    "type": "object",
81                    "properties": {
82                        "element_id": { "type": "string", "description": "Accessibility node ID (e.g. 'el_5')" }
83                    },
84                    "required": ["element_id"]
85                }),
86                returns: Some(json!({"type": "object"})),
87                idempotent: false,
88                cache_ttl_secs: None,
89                rate_limit: None,
90            },
91            ToolSchema {
92                name: "browse_type".to_string(),
93                source: car_ir::ToolSourceKind::Builtin,
94                description: "Type text into a UI element by accessibility node ID".to_string(),
95                parameters: json!({
96                    "type": "object",
97                    "properties": {
98                        "element_id": { "type": "string", "description": "Accessibility node ID of a text field" },
99                        "text": { "type": "string", "description": "Text to enter" }
100                    },
101                    "required": ["element_id", "text"]
102                }),
103                returns: Some(json!({"type": "object"})),
104                idempotent: false,
105                cache_ttl_secs: None,
106                rate_limit: None,
107            },
108            ToolSchema {
109                name: "browse_scroll".to_string(),
110                source: car_ir::ToolSourceKind::Builtin,
111                description: "Scroll the browser page".to_string(),
112                parameters: json!({
113                    "type": "object",
114                    "properties": {
115                        "delta_y": { "type": "integer", "description": "Scroll amount (positive = down, negative = up)" }
116                    },
117                    "required": ["delta_y"]
118                }),
119                returns: Some(json!({"type": "object"})),
120                idempotent: false,
121                cache_ttl_secs: None,
122                rate_limit: None,
123            },
124            ToolSchema {
125                name: "browse_keypress".to_string(),
126                source: car_ir::ToolSourceKind::Builtin,
127                description: "Press a key with optional modifiers".to_string(),
128                parameters: json!({
129                    "type": "object",
130                    "properties": {
131                        "key": { "type": "string", "description": "Key to press (e.g. 'Enter', 'a', 'Tab')" },
132                        "modifiers": {
133                            "type": "array",
134                            "items": { "type": "string", "enum": ["shift", "control", "alt", "meta"] },
135                            "description": "Optional modifier keys to hold during keypress"
136                        }
137                    },
138                    "required": ["key"]
139                }),
140                returns: Some(json!({"type": "object", "properties": {"key": {"type": "string"}, "status": {"type": "string"}}})),
141                idempotent: false,
142                cache_ttl_secs: None,
143                rate_limit: None,
144            },
145            ToolSchema {
146                name: "browse_wait".to_string(),
147                source: car_ir::ToolSourceKind::Builtin,
148                description: "Wait for a browser condition to be met".to_string(),
149                parameters: json!({
150                    "type": "object",
151                    "properties": {
152                        "condition": { "type": "string", "description": "Condition to wait for: 'page_loaded' or 'url_changed'" },
153                        "timeout_ms": { "type": "number", "description": "Timeout in milliseconds (default: 5000)" }
154                    },
155                    "required": ["condition"]
156                }),
157                returns: Some(json!({"type": "object", "properties": {"condition": {"type": "string"}, "met": {"type": "boolean"}}})),
158                idempotent: true,
159                cache_ttl_secs: None,
160                rate_limit: None,
161            },
162            ToolSchema {
163                name: "browse_observe".to_string(),
164                source: car_ir::ToolSourceKind::Builtin,
165                description: "Observe the current browser state: take screenshot, extract accessibility tree, produce UiMap".to_string(),
166                parameters: json!({
167                    "type": "object",
168                    "properties": {
169                        "include_screenshot": { "type": "boolean", "description": "Include base64 screenshot inline (default: false, returns file path instead)" },
170                        "ocr": { "type": "boolean", "description": "Fuse OCR over the screenshot: recover labels for nameless controls and surface canvas/image text the accessibility tree misses (default: false). Only takes effect with a vision-capable pipeline and an available OCR backend." }
171                    }
172                }),
173                returns: Some(json!({
174                    "type": "object",
175                    "properties": {
176                        "url": {"type": "string"},
177                        "title": {"type": "string"},
178                        "ui_map": {"type": "string"},
179                        "screenshot_path": {"type": "string"},
180                        "screenshot_base64": {"type": "string", "description": "Only present if include_screenshot=true"}
181                    }
182                })),
183                idempotent: true,
184                cache_ttl_secs: None,
185                rate_limit: None,
186            },
187        ]
188    }
189
190    async fn handle_navigate(&self, params: &Value) -> Result<Value, String> {
191        let url = params
192            .get("url")
193            .and_then(|v| v.as_str())
194            .ok_or("Missing required parameter: url")?;
195        self.backend
196            .navigate(url)
197            .await
198            .map_err(|e| e.to_string())?;
199        Ok(json!({"url": url, "status": "navigated"}))
200    }
201
202    async fn handle_click(&self, params: &Value) -> Result<Value, String> {
203        let element_id = params
204            .get("element_id")
205            .and_then(|v| v.as_str())
206            .ok_or("Missing required parameter: element_id")?;
207        let resolved_id = self.resolve_element_id(element_id).await;
208        self.backend
209            .click_element(&resolved_id)
210            .await
211            .map_err(|e| e.to_string())?;
212        Ok(json!({"element_id": element_id, "resolved_id": resolved_id, "status": "clicked"}))
213    }
214
215    async fn handle_type(&self, params: &Value) -> Result<Value, String> {
216        let element_id = params
217            .get("element_id")
218            .and_then(|v| v.as_str())
219            .ok_or("Missing required parameter: element_id")?;
220        let text = params
221            .get("text")
222            .and_then(|v| v.as_str())
223            .ok_or("Missing required parameter: text")?;
224        let resolved_id = self.resolve_element_id(element_id).await;
225        self.backend
226            .type_into_element(&resolved_id, text)
227            .await
228            .map_err(|e| e.to_string())?;
229        Ok(
230            json!({"element_id": element_id, "resolved_id": resolved_id, "text": text, "status": "typed"}),
231        )
232    }
233
234    async fn handle_scroll(&self, params: &Value) -> Result<Value, String> {
235        let delta_y = params
236            .get("delta_y")
237            .and_then(|v| v.as_i64())
238            .ok_or("Missing required parameter: delta_y")? as i32;
239        self.backend
240            .inject_scroll(delta_y)
241            .await
242            .map_err(|e| e.to_string())?;
243        Ok(json!({"delta_y": delta_y, "status": "scrolled"}))
244    }
245
246    async fn handle_keypress(&self, params: &Value) -> Result<Value, String> {
247        let key = params
248            .get("key")
249            .and_then(|v| v.as_str())
250            .ok_or("Missing required parameter: key")?;
251        let modifiers: Vec<Modifier> = params
252            .get("modifiers")
253            .and_then(|v| v.as_array())
254            .map(|arr| {
255                arr.iter()
256                    .filter_map(|m| match m.as_str()? {
257                        "shift" => Some(Modifier::Shift),
258                        "control" => Some(Modifier::Control),
259                        "alt" => Some(Modifier::Alt),
260                        "meta" => Some(Modifier::Meta),
261                        _ => None,
262                    })
263                    .collect()
264            })
265            .unwrap_or_default();
266        self.backend
267            .inject_keypress(key, &modifiers)
268            .await
269            .map_err(|e| e.to_string())?;
270        Ok(json!({"key": key, "status": "pressed"}))
271    }
272
273    async fn handle_wait(&self, params: &Value) -> Result<Value, String> {
274        let condition_str = params
275            .get("condition")
276            .and_then(|v| v.as_str())
277            .ok_or("Missing required parameter: condition")?;
278        let timeout_ms = params
279            .get("timeout_ms")
280            .and_then(|v| v.as_u64())
281            .unwrap_or(5000);
282        // Accepted shapes:
283        //   "page_loaded" | "url_changed"
284        //   "a11y_contains_text:<text>"
285        //   "element_with_name:<name>"
286        //   "element_with_name:<name>@<role>"
287        let condition = match condition_str {
288            "page_loaded" => WaitCondition::PageLoaded,
289            "url_changed" => WaitCondition::UrlChanged,
290            s if s.starts_with("a11y_contains_text:") => WaitCondition::A11yContainsText {
291                text: s["a11y_contains_text:".len()..].to_string(),
292            },
293            s if s.starts_with("element_with_name:") => {
294                let rest = &s["element_with_name:".len()..];
295                let (name_contains, role) = match rest.split_once('@') {
296                    Some((n, r)) => (n.to_string(), Some(r.to_string())),
297                    None => (rest.to_string(), None),
298                };
299                WaitCondition::ElementWithName {
300                    name_contains,
301                    role,
302                }
303            }
304            other => return Err(format!("Unknown wait condition: {other}")),
305        };
306        let met = self
307            .backend
308            .wait_until(&condition, timeout_ms)
309            .await
310            .map_err(|e| e.to_string())?;
311        Ok(json!({"condition": condition_str, "met": met}))
312    }
313
314    async fn handle_observe(&self, params: &Value) -> Result<Value, String> {
315        let ocr = params.get("ocr").and_then(|v| v.as_bool()).unwrap_or(false);
316        let screenshot = self
317            .backend
318            .capture_screenshot()
319            .await
320            .map_err(|e| e.to_string())?;
321        let a11y_nodes = self
322            .backend
323            .get_accessibility_tree()
324            .await
325            .map_err(|e| e.to_string())?;
326        let url = self.backend.get_current_url().map_err(|e| e.to_string())?;
327        let title = self
328            .backend
329            .get_page_title()
330            .await
331            .map_err(|e| e.to_string())?;
332        let viewport = self.backend.get_viewport().map_err(|e| e.to_string())?;
333
334        // The screenshot is always captured (saved to screenshot_path below),
335        // but only fed to OCR when the caller opts in — an empty slice makes a
336        // vision pipeline take the AX-only path.
337        let perceive_image: &[u8] = if ocr { &screenshot } else { &[] };
338        let ui_map = self
339            .pipeline
340            .perceive(perceive_image, &a11y_nodes, &url, viewport)
341            .await
342            .map_err(|e| e.to_string())?;
343
344        // Store UiMap for element ID resolution in subsequent click/type calls
345        {
346            let mut guard = self.last_ui_map.write().await;
347            *guard = Some(ui_map.clone());
348        }
349
350        let ui_map_text = ui_map.format_summary();
351
352        // Save screenshot to temp file instead of inline base64 (saves ~3-5MB per observe)
353        let screenshot_path = {
354            let dir = std::env::temp_dir().join("car-browser-screenshots");
355            let _ = std::fs::create_dir_all(&dir);
356            let path = dir.join(format!("{}.png", uuid::Uuid::new_v4()));
357            std::fs::write(&path, &screenshot).map_err(|e| e.to_string())?;
358            path.to_string_lossy().to_string()
359        };
360
361        // Screenshot is always saved to disk — never inline base64 (saves ~80KB per observe).
362        // Vision models can read the file via screenshot_path if needed.
363        let result = json!({
364            "url": url,
365            "title": title,
366            "ui_map": ui_map_text,
367            "screenshot_path": screenshot_path,
368            "element_count": ui_map.elements.len(),
369            "viewport": {
370                "width": viewport.width,
371                "height": viewport.height,
372            }
373        });
374
375        Ok(result)
376    }
377}
378
379#[async_trait]
380impl ToolExecutor for BrowserToolExecutor {
381    async fn execute(&self, tool: &str, params: &Value) -> Result<Value, String> {
382        match tool {
383            "browse_navigate" => self.handle_navigate(params).await,
384            "browse_click" => self.handle_click(params).await,
385            "browse_type" => self.handle_type(params).await,
386            "browse_scroll" => self.handle_scroll(params).await,
387            "browse_keypress" => self.handle_keypress(params).await,
388            "browse_wait" => self.handle_wait(params).await,
389            "browse_observe" => self.handle_observe(params).await,
390            _ => Err(format!("Unknown browser tool: {tool}")),
391        }
392    }
393}
car_browser/tools.rs

car_browser/
tools.rs
