oxi_agent/tools/
computer_tool.rs

1/// Computer tool — computer control using Vision AI capabilities.
2///
3/// Provides a simplified interface for computer interaction through Vision
4/// models: screenshot capture, mouse/keyboard actions. Requires a
5/// Vision-capable model and appropriate OS-level access (screen capture,
6/// input simulation). The omp implementation uses the full `pi-natives`
7/// desktop automation layer; this is a practical stub.
8use super::{AgentTool, AgentToolResult, ToolContext, ToolError, ToolExecutionMode, ToolTier};
9use async_trait::async_trait;
10use serde_json::{Value, json};
11use tokio::sync::oneshot;
12
13/// ComputerTool — basic computer control interface.
14pub struct ComputerTool;
15
16#[async_trait]
17impl AgentTool for ComputerTool {
18    fn name(&self) -> &str {
19        "computer"
20    }
21
22    fn label(&self) -> &str {
23        "Computer"
24    }
25
26    fn description(&self) -> &str {
27        concat!(
28            "Interact with the computer using Vision AI: capture screenshots, ",
29            "move mouse, click, type text, press keys. ",
30            "Actions: screenshot (capture screen), mouse_move (x, y), ",
31            "left_click, right_click, type (text), key (key name), ",
32            "scroll (x, y). Requires OS-level access permissions."
33        )
34    }
35
36    fn essential(&self) -> bool {
37        false
38    }
39
40    fn parameters_schema(&self) -> Value {
41        json!({
42            "type": "object",
43            "properties": {
44                "action": {
45                    "type": "string",
46                    "enum": ["screenshot", "mouse_move", "left_click", "right_click", "double_click", "type", "key", "scroll"],
47                    "description": "Computer action to perform."
48                },
49                "x": {
50                    "type": "integer",
51                    "description": "X coordinate (for mouse_move, scroll)."
52                },
53                "y": {
54                    "type": "integer",
55                    "description": "Y coordinate (for mouse_move, scroll)."
56                },
57                "text": {
58                    "type": "string",
59                    "description": "Text to type (for type action)."
60                },
61                "key": {
62                    "type": "string",
63                    "description": "Key name to press (for key action: Enter, Tab, Escape, etc.)."
64                }
65            },
66            "required": ["action"]
67        })
68    }
69
70    fn intent(&self) -> Option<&str> {
71        Some("Control the computer via Vision AI")
72    }
73
74    fn execution_mode(&self) -> ToolExecutionMode {
75        ToolExecutionMode::SequentialOnly
76    }
77
78    fn tool_tier(&self) -> ToolTier {
79        ToolTier::Exec
80    }
81
82    async fn execute(
83        &self,
84        _tool_call_id: &str,
85        params: Value,
86        _signal: Option<oneshot::Receiver<()>>,
87        _ctx: &ToolContext,
88    ) -> Result<AgentToolResult, ToolError> {
89        let action = params
90            .get("action")
91            .and_then(|v| v.as_str())
92            .ok_or_else(|| "Missing required parameter: action".to_string())?;
93
94        // Full implementation would use OS-level APIs (Core Graphics on macOS,
95        // X11/Wayland on Linux, Win32 on Windows).
96        // For now, provide actionable guidance.
97
98        let response = match action {
99            "screenshot" => concat!(
100                "Screenshot capture requested.\n",
101                "To capture a screenshot, use:\n",
102                "  macOS: `screencapture -x /tmp/screenshot.png`\n",
103                "  Linux: `import -window root /tmp/screenshot.png`\n",
104                "  Then use `read` to view the image.\n\n",
105                "Full vision-integrated screenshot requires native desktop ",
106                "access (Core Graphics / X11 / Win32) which is not yet implemented."
107            )
108            .to_string(),
109            "mouse_move" | "left_click" | "right_click" | "double_click" | "scroll" => {
110                let x = params.get("x").and_then(|v| v.as_i64()).unwrap_or(0);
111                let y = params.get("y").and_then(|v| v.as_i64()).unwrap_or(0);
112                format!("Action '{}' at ({}, {}) requested.\n", action, x, y)
113                    + "Computer control requires OS-level input simulation not yet implemented.\n"
114                    + "Use the `bash` tool with platform-specific commands:\n"
115                    + "  macOS: `osascript -e 'tell app \"System Events\" to click at {x, y}'`\n"
116                    + "  Linux: `xdotool mousemove x y`"
117            }
118            "type" => {
119                let text = params.get("text").and_then(|v| v.as_str()).unwrap_or("");
120                format!("Type action requested: '{}'\n", text)
121                    + "Keyboard input simulation requires OS-level APIs.\n"
122                    + "Use `bash` with platform-specific tools:\n"
123                    + "  macOS: `osascript -e 'tell app \"System Events\" to keystroke \"...\"'`\n"
124                    + "  Linux: `xdotool type '...'`"
125            }
126            "key" => {
127                let key = params.get("key").and_then(|v| v.as_str()).unwrap_or("");
128                format!("Key press action requested: '{}'\n", key)
129                    + "Key simulation requires OS-level APIs.\n"
130                    + "Use `bash` with platform-specific tools:\n"
131                    + "  macOS: `osascript -e 'tell app \"System Events\" to key code 36'`\n"
132                    + "  Linux: `xdotool key Return`"
133            }
134            _ => format!("Unknown computer action: {}", action),
135        };
136
137        Ok(AgentToolResult::success(response))
138    }
139}
140
141#[cfg(test)]
142mod tests {
143    use super::*;
144
145    #[tokio::test]
146    async fn test_computer_screenshot() {
147        let tool = ComputerTool;
148        let params = json!({"action": "screenshot"});
149        let result = tool
150            .execute("id", params, None, &ToolContext::default())
151            .await
152            .unwrap();
153        assert!(result.success);
154        assert!(result.output.contains("Screenshot capture requested"));
155    }
156
157    #[tokio::test]
158    async fn test_computer_mouse_move() {
159        let tool = ComputerTool;
160        let params = json!({"action": "mouse_move", "x": 100, "y": 200});
161        let result = tool
162            .execute("id", params, None, &ToolContext::default())
163            .await
164            .unwrap();
165        assert!(result.success);
166        assert!(result.output.contains("mouse_move"));
167    }
168}
oxi_agent/tools/computer_tool.rs

oxi_agent/tools/
computer_tool.rs
