lc_tools/extended/computer/
actions.rs

1//! Action types and helpers for the computer-use tool.
2//!
3//! Defines the input/output structures and the operating mode enum used by
4//! `ComputerUseTool`, plus the action-building and
5//! validation helpers that translate user input into Anthropic API payloads.
6
7use serde_json::Value;
8
9#[cfg(feature = "native-computer")]
10use std::time::Duration;
11
12use lc_core::tools::ToolError;
13
14use super::screen::ComputerUseTool;
15
16// ---------------------------------------------------------------------------
17// Mode enum
18// ---------------------------------------------------------------------------
19
20/// Operating mode for `ComputerUseTool`.
21#[derive(Debug, Clone)]
22pub enum ComputerMode {
23    /// Forward actions to the Anthropic computer-use API.
24    AnthropicApi,
25    /// Local screenshot + input simulation (behind `native-computer` feature gate).
26    #[cfg(feature = "native-computer")]
27    Native,
28}
29
30// ---------------------------------------------------------------------------
31// Input / Output types
32// ---------------------------------------------------------------------------
33
34/// Typed input for the computer-use tool (used for schema generation).
35#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
36pub struct ComputerUseInput {
37    /// Action to perform.
38    pub action: String,
39
40    /// Coordinate as `[x, y]` for actions that need a position.
41    #[serde(default, skip_serializing_if = "Option::is_none")]
42    pub coordinate: Option<Vec<i32>>,
43
44    /// Text to type (for `type` action).
45    #[serde(default, skip_serializing_if = "Option::is_none")]
46    pub text: Option<String>,
47
48    /// Key names to press simultaneously (for `key_press` action).
49    #[serde(default, skip_serializing_if = "Option::is_none")]
50    pub keys: Option<Vec<String>>,
51
52    /// Scroll direction: `up` or `down` (for `scroll` action).
53    #[serde(default, skip_serializing_if = "Option::is_none")]
54    pub direction: Option<String>,
55
56    /// Scroll amount in clicks (for `scroll` action).
57    #[serde(default, skip_serializing_if = "Option::is_none")]
58    pub amount: Option<i32>,
59
60    /// Wait duration in milliseconds (for `wait` action).
61    #[serde(default, skip_serializing_if = "Option::is_none")]
62    pub duration_ms: Option<u64>,
63}
64
65/// Typed output from the computer-use tool.
66#[derive(Debug, serde::Serialize)]
67pub struct ComputerUseOutput {
68    /// The action that was executed.
69    pub action: String,
70    /// Human-readable result message.
71    pub result: String,
72    /// Optional base64-encoded screenshot data (for `screenshot` action).
73    #[serde(skip_serializing_if = "Option::is_none")]
74    pub screenshot_base64: Option<String>,
75}
76
77// ---------------------------------------------------------------------------
78// Action helpers (AnthropicApi mode)
79// ---------------------------------------------------------------------------
80
81impl ComputerUseTool {
82    /// Build the Anthropic messages API request body for a computer-use action.
83    pub(super) fn build_anthropic_request(
84        &self,
85        action_input: &ComputerUseInput,
86    ) -> Result<Value, ToolError> {
87        let tool_input = self.build_tool_input(action_input)?;
88        let tool_input_str = serde_json::to_string(&tool_input).unwrap_or_default();
89
90        Ok(serde_json::json!({
91            "model": "claude-sonnet-4-20250514",
92            "max_tokens": 4096,
93            "tools": [
94                {
95                    "type": "computer_20250124",
96                    "name": "computer",
97                    "display_width_px": self.display_width,
98                    "display_height_px": self.display_height,
99                }
100            ],
101            "messages": [
102                {
103                    "role": "user",
104                    "content": format!(
105                        "Use the computer tool with these parameters: {}",
106                        tool_input_str
107                    )
108                }
109            ],
110            "tool_choice": {
111                "type": "tool",
112                "name": "computer"
113            }
114        }))
115    }
116
117    /// Build the tool-specific input dict that Anthropic's computer-use API
118    /// expects for each action type.
119    pub(super) fn build_tool_input(&self, input: &ComputerUseInput) -> Result<Value, ToolError> {
120        match input.action.as_str() {
121            "screenshot" => Ok(serde_json::json!({
122                "action": "screenshot"
123            })),
124            "click" => {
125                let coord = input.coordinate.as_deref().unwrap_or(&[0, 0]);
126                let button = if input
127                    .text
128                    .as_deref()
129                    .unwrap_or("left")
130                    .eq_ignore_ascii_case("right")
131                {
132                    "right"
133                } else {
134                    "left"
135                };
136                Ok(serde_json::json!({
137                    "action": "mouse_click",
138                    "coordinate": coord,
139                    "text": button
140                }))
141            }
142            "type" => Ok(serde_json::json!({
143                "action": "type",
144                "text": input.text.as_deref().unwrap_or("")
145            })),
146            "scroll" => {
147                let coord = input.coordinate.as_deref().unwrap_or(&[0, 0]);
148                let direction = input.direction.as_deref().unwrap_or("down");
149                let scroll_direction = if direction.eq_ignore_ascii_case("up") {
150                    "up"
151                } else if direction.eq_ignore_ascii_case("left") {
152                    "left"
153                } else if direction.eq_ignore_ascii_case("right") {
154                    "right"
155                } else {
156                    "down"
157                };
158                Ok(serde_json::json!({
159                    "action": "scroll",
160                    "coordinate": coord,
161                    "direction": scroll_direction,
162                    "amount": input.amount.unwrap_or(3)
163                }))
164            }
165            "key_press" => Ok(serde_json::json!({
166                "action": "key",
167                "text": input.keys.as_deref().unwrap_or(&[]).join("+")
168            })),
169            "wait" => Ok(serde_json::json!({
170                "action": "wait",
171                "duration_ms": input.duration_ms.unwrap_or(1000)
172            })),
173            _ => Err(ToolError::InvalidInput(format!(
174                "Unknown action: '{}'. Valid actions: screenshot, click, type, scroll, key_press, wait",
175                input.action
176            ))),
177        }
178    }
179
180    /// Send a request to the Anthropic messages API and return the response.
181    pub(super) async fn call_anthropic_api(&self, body: &Value) -> Result<String, ToolError> {
182        let url = format!("{}/v1/messages", self.base_url.trim_end_matches('/'));
183
184        let resp = self
185            .client
186            .post(&url)
187            .header("x-api-key", &self.api_key)
188            .header("anthropic-version", "2023-06-01")
189            .header("anthropic-beta", "computer-use-2025-01-24")
190            .header("content-type", "application/json")
191            .json(body)
192            .send()
193            .await
194            .map_err(|e| {
195                ToolError::ExecutionFailed(format!("Anthropic API request failed: {}", e))
196            })?;
197
198        let status = resp.status();
199        let text = resp.text().await.map_err(|e| {
200            ToolError::ExecutionFailed(format!("Failed to read response body: {}", e))
201        })?;
202
203        if !status.is_success() {
204            return Err(ToolError::ExecutionFailed(format!(
205                "Anthropic API returned status {}: {}",
206                status, text
207            )));
208        }
209
210        Ok(text)
211    }
212
213    /// Execute an action in AnthropicApi mode.
214    pub(super) async fn execute_anthropic(
215        &self,
216        input: &ComputerUseInput,
217    ) -> Result<ComputerUseOutput, ToolError> {
218        if self.api_key.is_empty() {
219            return Err(ToolError::InvalidInput(
220                "API key is required for AnthropicApi mode".to_string(),
221            ));
222        }
223
224        self.validate_input(input)?;
225
226        let body = self.build_anthropic_request(input)?;
227        let response_text = self.call_anthropic_api(&body).await?;
228
229        let screenshot_base64 = if input.action == "screenshot" {
230            serde_json::from_str::<Value>(&response_text)
231                .ok()
232                .and_then(|v| {
233                    v.get("content")?.as_array()?.iter().find_map(|block| {
234                        if block.get("type")?.as_str()? == "image" {
235                            block.get("source")?.get("data")?.as_str().map(String::from)
236                        } else {
237                            None
238                        }
239                    })
240                })
241        } else {
242            None
243        };
244
245        Ok(ComputerUseOutput {
246            action: input.action.clone(),
247            result: response_text,
248            screenshot_base64,
249        })
250    }
251
252    /// Execute an action in Native mode (placeholder).
253    #[cfg(feature = "native-computer")]
254    pub(super) async fn execute_native(
255        &self,
256        input: &ComputerUseInput,
257    ) -> Result<ComputerUseOutput, ToolError> {
258        self.validate_input(input)?;
259
260        match input.action.as_str() {
261            "screenshot" => Ok(ComputerUseOutput {
262                action: "screenshot".to_string(),
263                result: "Screenshot captured (native mode)".to_string(),
264                screenshot_base64: None,
265            }),
266            "click" => {
267                let coord = input.coordinate.as_deref().unwrap_or(&[0, 0]);
268                Ok(ComputerUseOutput {
269                    action: "click".to_string(),
270                    result: format!("Clicked at ({}, {}) (native mode)", coord[0], coord[1]),
271                    screenshot_base64: None,
272                })
273            }
274            "type" => Ok(ComputerUseOutput {
275                action: "type".to_string(),
276                result: format!(
277                    "Typed '{}' (native mode)",
278                    input.text.as_deref().unwrap_or("")
279                ),
280                screenshot_base64: None,
281            }),
282            "scroll" => {
283                let coord = input.coordinate.as_deref().unwrap_or(&[0, 0]);
284                Ok(ComputerUseOutput {
285                    action: "scroll".to_string(),
286                    result: format!(
287                        "Scrolled {} by {} at ({}, {}) (native mode)",
288                        input.direction.as_deref().unwrap_or("down"),
289                        input.amount.unwrap_or(3),
290                        coord[0],
291                        coord[1]
292                    ),
293                    screenshot_base64: None,
294                })
295            }
296            "key_press" => Ok(ComputerUseOutput {
297                action: "key_press".to_string(),
298                result: format!(
299                    "Pressed keys: {} (native mode)",
300                    input.keys.as_deref().unwrap_or(&[]).join("+")
301                ),
302                screenshot_base64: None,
303            }),
304            "wait" => {
305                let ms = input.duration_ms.unwrap_or(1000);
306                tokio::time::sleep(Duration::from_millis(ms)).await;
307                Ok(ComputerUseOutput {
308                    action: "wait".to_string(),
309                    result: format!("Waited {}ms (native mode)", ms),
310                    screenshot_base64: None,
311                })
312            }
313            other => Err(ToolError::InvalidInput(format!(
314                "Unknown action: {}",
315                other
316            ))),
317        }
318    }
319
320    /// Validate the input for the given action.
321    pub(super) fn validate_input(&self, input: &ComputerUseInput) -> Result<(), ToolError> {
322        let valid_actions = ["screenshot", "click", "type", "scroll", "key_press", "wait"];
323
324        if !valid_actions.contains(&input.action.as_str()) {
325            return Err(ToolError::InvalidInput(format!(
326                "Unknown action: '{}'. Valid actions: {:?}",
327                input.action, valid_actions
328            )));
329        }
330
331        match input.action.as_str() {
332            "click" => {
333                if input.coordinate.is_none() {
334                    return Err(ToolError::InvalidInput(
335                        "'click' action requires 'coordinate' field as [x, y]".to_string(),
336                    ));
337                }
338            }
339            "scroll" => {
340                if input.coordinate.is_none() {
341                    return Err(ToolError::InvalidInput(
342                        "'scroll' action requires 'coordinate' field as [x, y]".to_string(),
343                    ));
344                }
345                if input.direction.is_none() {
346                    return Err(ToolError::InvalidInput(
347                        "'scroll' action requires 'direction' field (up/down/left/right)"
348                            .to_string(),
349                    ));
350                }
351            }
352            "type" => {
353                if input.text.is_none() {
354                    return Err(ToolError::InvalidInput(
355                        "'type' action requires 'text' field".to_string(),
356                    ));
357                }
358            }
359            "key_press" => {
360                if input.keys.is_none() {
361                    return Err(ToolError::InvalidInput(
362                        "'key_press' action requires 'keys' field".to_string(),
363                    ));
364                }
365            }
366            _ => {}
367        }
368
369        if let Some(ref coord) = input.coordinate {
370            if coord.len() != 2 {
371                return Err(ToolError::InvalidInput(
372                    "coordinate must be exactly [x, y]".to_string(),
373                ));
374            }
375            if coord[0] < 0 || coord[0] as u32 > self.display_width {
376                return Err(ToolError::InvalidInput(format!(
377                    "x coordinate {} out of bounds (0-{})",
378                    coord[0], self.display_width
379                )));
380            }
381            if coord[1] < 0 || coord[1] as u32 > self.display_height {
382                return Err(ToolError::InvalidInput(format!(
383                    "y coordinate {} out of bounds (0-{})",
384                    coord[1], self.display_height
385                )));
386            }
387        }
388
389        Ok(())
390    }
391}
lc_tools/extended/computer/actions.rs

lc_tools/extended/computer/
actions.rs
