lc_tools/extended/computer/
actions.rs

1//! Action types and helpers for the computer-use tool.
2//!
3//! Defines the input/output structures and the operating mode enum used by
4//! `ComputerUseTool`, plus the action-building and
5//! validation helpers that translate user input into Anthropic API payloads.
6
7use serde_json::Value;
8
9use lc_core::tools::ToolError;
10
11use super::screen::ComputerUseTool;
12
13// ---------------------------------------------------------------------------
14// Mode enum
15// ---------------------------------------------------------------------------
16
17/// Operating mode for `ComputerUseTool`.
18#[derive(Debug, Clone)]
19pub enum ComputerMode {
20    /// Forward actions to the Anthropic computer-use API.
21    AnthropicApi,
22}
23
24// ---------------------------------------------------------------------------
25// Input / Output types
26// ---------------------------------------------------------------------------
27
28/// Typed input for the computer-use tool (used for schema generation).
29#[derive(Debug, serde::Deserialize, schemars::JsonSchema)]
30pub struct ComputerUseInput {
31    /// Action to perform.
32    pub action: String,
33
34    /// Coordinate as `[x, y]` for actions that need a position.
35    #[serde(default, skip_serializing_if = "Option::is_none")]
36    pub coordinate: Option<Vec<i32>>,
37
38    /// Text to type (for `type` action).
39    #[serde(default, skip_serializing_if = "Option::is_none")]
40    pub text: Option<String>,
41
42    /// Key names to press simultaneously (for `key_press` action).
43    #[serde(default, skip_serializing_if = "Option::is_none")]
44    pub keys: Option<Vec<String>>,
45
46    /// Scroll direction: `up` or `down` (for `scroll` action).
47    #[serde(default, skip_serializing_if = "Option::is_none")]
48    pub direction: Option<String>,
49
50    /// Scroll amount in clicks (for `scroll` action).
51    #[serde(default, skip_serializing_if = "Option::is_none")]
52    pub amount: Option<i32>,
53
54    /// Wait duration in milliseconds (for `wait` action).
55    #[serde(default, skip_serializing_if = "Option::is_none")]
56    pub duration_ms: Option<u64>,
57}
58
59/// Typed output from the computer-use tool.
60#[derive(Debug, serde::Serialize)]
61pub struct ComputerUseOutput {
62    /// The action that was executed.
63    pub action: String,
64    /// Human-readable result message.
65    pub result: String,
66    /// Optional base64-encoded screenshot data (for `screenshot` action).
67    #[serde(skip_serializing_if = "Option::is_none")]
68    pub screenshot_base64: Option<String>,
69}
70
71// ---------------------------------------------------------------------------
72// Action helpers (AnthropicApi mode)
73// ---------------------------------------------------------------------------
74
75impl ComputerUseTool {
76    /// Build the Anthropic messages API request body for a computer-use action.
77    pub(super) fn build_anthropic_request(
78        &self,
79        action_input: &ComputerUseInput,
80    ) -> Result<Value, ToolError> {
81        let tool_input = self.build_tool_input(action_input)?;
82        let tool_input_str = serde_json::to_string(&tool_input).unwrap_or_default();
83
84        Ok(serde_json::json!({
85            "model": "claude-sonnet-4-20250514",
86            "max_tokens": 4096,
87            "tools": [
88                {
89                    "type": "computer_20250124",
90                    "name": "computer",
91                    "display_width_px": self.display_width,
92                    "display_height_px": self.display_height,
93                }
94            ],
95            "messages": [
96                {
97                    "role": "user",
98                    "content": format!(
99                        "Use the computer tool with these parameters: {}",
100                        tool_input_str
101                    )
102                }
103            ],
104            "tool_choice": {
105                "type": "tool",
106                "name": "computer"
107            }
108        }))
109    }
110
111    /// Build the tool-specific input dict that Anthropic's computer-use API
112    /// expects for each action type.
113    pub(super) fn build_tool_input(&self, input: &ComputerUseInput) -> Result<Value, ToolError> {
114        match input.action.as_str() {
115            "screenshot" => Ok(serde_json::json!({
116                "action": "screenshot"
117            })),
118            "click" => {
119                let coord = input.coordinate.as_deref().unwrap_or(&[0, 0]);
120                let button = if input
121                    .text
122                    .as_deref()
123                    .unwrap_or("left")
124                    .eq_ignore_ascii_case("right")
125                {
126                    "right"
127                } else {
128                    "left"
129                };
130                Ok(serde_json::json!({
131                    "action": "mouse_click",
132                    "coordinate": coord,
133                    "text": button
134                }))
135            }
136            "type" => Ok(serde_json::json!({
137                "action": "type",
138                "text": input.text.as_deref().unwrap_or("")
139            })),
140            "scroll" => {
141                let coord = input.coordinate.as_deref().unwrap_or(&[0, 0]);
142                let direction = input.direction.as_deref().unwrap_or("down");
143                let scroll_direction = if direction.eq_ignore_ascii_case("up") {
144                    "up"
145                } else if direction.eq_ignore_ascii_case("left") {
146                    "left"
147                } else if direction.eq_ignore_ascii_case("right") {
148                    "right"
149                } else {
150                    "down"
151                };
152                Ok(serde_json::json!({
153                    "action": "scroll",
154                    "coordinate": coord,
155                    "direction": scroll_direction,
156                    "amount": input.amount.unwrap_or(3)
157                }))
158            }
159            "key_press" => Ok(serde_json::json!({
160                "action": "key",
161                "text": input.keys.as_deref().unwrap_or(&[]).join("+")
162            })),
163            "wait" => Ok(serde_json::json!({
164                "action": "wait",
165                "duration_ms": input.duration_ms.unwrap_or(1000)
166            })),
167            _ => Err(ToolError::InvalidInput(format!(
168                "Unknown action: '{}'. Valid actions: screenshot, click, type, scroll, key_press, wait",
169                input.action
170            ))),
171        }
172    }
173
174    /// Send a request to the Anthropic messages API and return the response.
175    pub(super) async fn call_anthropic_api(&self, body: &Value) -> Result<String, ToolError> {
176        let url = format!("{}/v1/messages", self.base_url.trim_end_matches('/'));
177
178        let resp = self
179            .client
180            .post(&url)
181            .header("x-api-key", &self.api_key)
182            .header("anthropic-version", "2023-06-01")
183            .header("anthropic-beta", "computer-use-2025-01-24")
184            .header("content-type", "application/json")
185            .json(body)
186            .send()
187            .await
188            .map_err(|e| {
189                ToolError::ExecutionFailed(format!("Anthropic API request failed: {}", e))
190            })?;
191
192        let status = resp.status();
193        let text = resp.text().await.map_err(|e| {
194            ToolError::ExecutionFailed(format!("Failed to read response body: {}", e))
195        })?;
196
197        if !status.is_success() {
198            return Err(ToolError::ExecutionFailed(format!(
199                "Anthropic API returned status {}: {}",
200                status, text
201            )));
202        }
203
204        Ok(text)
205    }
206
207    /// Execute an action in AnthropicApi mode.
208    pub(super) async fn execute_anthropic(
209        &self,
210        input: &ComputerUseInput,
211    ) -> Result<ComputerUseOutput, ToolError> {
212        if self.api_key.is_empty() {
213            return Err(ToolError::InvalidInput(
214                "API key is required for AnthropicApi mode".to_string(),
215            ));
216        }
217
218        self.validate_input(input)?;
219
220        let body = self.build_anthropic_request(input)?;
221        let response_text = self.call_anthropic_api(&body).await?;
222
223        let screenshot_base64 = if input.action == "screenshot" {
224            serde_json::from_str::<Value>(&response_text)
225                .ok()
226                .and_then(|v| {
227                    v.get("content")?.as_array()?.iter().find_map(|block| {
228                        if block.get("type")?.as_str()? == "image" {
229                            block.get("source")?.get("data")?.as_str().map(String::from)
230                        } else {
231                            None
232                        }
233                    })
234                })
235        } else {
236            None
237        };
238
239        Ok(ComputerUseOutput {
240            action: input.action.clone(),
241            result: response_text,
242            screenshot_base64,
243        })
244    }
245
246    /// Validate the input for the given action.
247    pub(super) fn validate_input(&self, input: &ComputerUseInput) -> Result<(), ToolError> {
248        let valid_actions = ["screenshot", "click", "type", "scroll", "key_press", "wait"];
249
250        if !valid_actions.contains(&input.action.as_str()) {
251            return Err(ToolError::InvalidInput(format!(
252                "Unknown action: '{}'. Valid actions: {:?}",
253                input.action, valid_actions
254            )));
255        }
256
257        match input.action.as_str() {
258            "click" => {
259                if input.coordinate.is_none() {
260                    return Err(ToolError::InvalidInput(
261                        "'click' action requires 'coordinate' field as [x, y]".to_string(),
262                    ));
263                }
264            }
265            "scroll" => {
266                if input.coordinate.is_none() {
267                    return Err(ToolError::InvalidInput(
268                        "'scroll' action requires 'coordinate' field as [x, y]".to_string(),
269                    ));
270                }
271                if input.direction.is_none() {
272                    return Err(ToolError::InvalidInput(
273                        "'scroll' action requires 'direction' field (up/down/left/right)"
274                            .to_string(),
275                    ));
276                }
277            }
278            "type" => {
279                if input.text.is_none() {
280                    return Err(ToolError::InvalidInput(
281                        "'type' action requires 'text' field".to_string(),
282                    ));
283                }
284            }
285            "key_press" => {
286                if input.keys.is_none() {
287                    return Err(ToolError::InvalidInput(
288                        "'key_press' action requires 'keys' field".to_string(),
289                    ));
290                }
291            }
292            _ => {}
293        }
294
295        if let Some(ref coord) = input.coordinate {
296            if coord.len() != 2 {
297                return Err(ToolError::InvalidInput(
298                    "coordinate must be exactly [x, y]".to_string(),
299                ));
300            }
301            if coord[0] < 0 || coord[0] as u32 > self.display_width {
302                return Err(ToolError::InvalidInput(format!(
303                    "x coordinate {} out of bounds (0-{})",
304                    coord[0], self.display_width
305                )));
306            }
307            if coord[1] < 0 || coord[1] as u32 > self.display_height {
308                return Err(ToolError::InvalidInput(format!(
309                    "y coordinate {} out of bounds (0-{})",
310                    coord[1], self.display_height
311                )));
312            }
313        }
314
315        Ok(())
316    }
317}
lc_tools/extended/computer/actions.rs

lc_tools/extended/computer/
actions.rs
