use anyhow::{anyhow, Result};
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use std::collections::HashMap;
use std::sync::Arc;
#[cfg(target_os = "macos")]
mod macos;
#[cfg(target_os = "linux")]
mod linux;
#[cfg(target_os = "windows")]
mod windows;
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "snake_case")]
pub enum UiAction {
Click,
DoubleClick,
RightClick,
MiddleClick,
Move,
MoveRelative,
Drag,
DragRelative,
Scroll,
Type,
Write,
Press,
KeyDown,
KeyUp,
Hotkey,
Screenshot,
ScreenshotRegion,
GetActiveWindow,
ListWindows,
FocusWindow,
GetScreens,
ScreenSize,
Position,
Sleep,
SetPause,
SetFailsafe,
Batch,
Info,
}
impl Default for UiAction {
fn default() -> Self {
Self::Info
}
}
impl std::str::FromStr for UiAction {
type Err = anyhow::Error;
fn from_str(s: &str) -> Result<Self> {
match s.to_lowercase().as_str() {
"click" => Ok(Self::Click),
"double_click" | "doubleclick" => Ok(Self::DoubleClick),
"right_click" | "rightclick" => Ok(Self::RightClick),
"middle_click" | "middleclick" => Ok(Self::MiddleClick),
"move" => Ok(Self::Move),
"move_relative" | "moverelative" => Ok(Self::MoveRelative),
"drag" => Ok(Self::Drag),
"drag_relative" | "dragrelative" => Ok(Self::DragRelative),
"scroll" => Ok(Self::Scroll),
"type" => Ok(Self::Type),
"write" => Ok(Self::Write),
"press" => Ok(Self::Press),
"key_down" | "keydown" => Ok(Self::KeyDown),
"key_up" | "keyup" => Ok(Self::KeyUp),
"hotkey" => Ok(Self::Hotkey),
"screenshot" => Ok(Self::Screenshot),
"screenshot_region" | "screenshotregion" => Ok(Self::ScreenshotRegion),
"get_active_window" | "getactivewindow" => Ok(Self::GetActiveWindow),
"list_windows" | "listwindows" => Ok(Self::ListWindows),
"focus_window" | "focuswindow" => Ok(Self::FocusWindow),
"get_screens" | "getscreens" => Ok(Self::GetScreens),
"screen_size" | "screensize" => Ok(Self::ScreenSize),
"position" => Ok(Self::Position),
"sleep" => Ok(Self::Sleep),
"set_pause" | "setpause" => Ok(Self::SetPause),
"set_failsafe" | "setfailsafe" => Ok(Self::SetFailsafe),
"batch" => Ok(Self::Batch),
"info" => Ok(Self::Info),
_ => Err(anyhow!("Unknown action: {}", s)),
}
}
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct ComputerToolArgs {
#[serde(default)]
pub action: String,
pub x: Option<i32>,
pub y: Option<i32>,
pub dx: Option<i32>,
pub dy: Option<i32>,
pub end_x: Option<i32>,
pub end_y: Option<i32>,
pub text: Option<String>,
pub key: Option<String>,
pub keys: Option<Vec<String>>,
#[serde(default = "default_button")]
pub button: String,
pub amount: Option<i32>,
#[serde(default = "default_duration")]
pub duration: f64,
#[serde(default = "default_interval")]
pub interval: f64,
pub region: Option<Vec<i32>>,
#[serde(default)]
pub clear: bool,
pub title: Option<String>,
pub name: Option<String>,
pub width: Option<i32>,
pub height: Option<i32>,
pub value: Option<f64>,
pub actions: Option<Vec<Value>>,
}
fn default_button() -> String {
"left".to_string()
}
fn default_duration() -> f64 {
0.25
}
fn default_interval() -> f64 {
0.02
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WindowInfo {
pub app: Option<String>,
pub title: String,
pub x: i32,
pub y: i32,
pub width: i32,
pub height: i32,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PlatformInfo {
pub platform: String,
pub native_available: bool,
pub backends: HashMap<String, bool>,
}
pub trait NativeControl: Send + Sync {
fn platform_info(&self) -> PlatformInfo;
fn mouse_position(&self) -> Result<(i32, i32)>;
fn screen_size(&self) -> Result<(i32, i32)>;
fn click(&self, x: i32, y: i32, button: &str) -> Result<()>;
fn double_click(&self, x: i32, y: i32) -> Result<()>;
fn move_to(&self, x: i32, y: i32) -> Result<()>;
fn drag(&self, start_x: i32, start_y: i32, end_x: i32, end_y: i32, button: &str) -> Result<()>;
fn scroll(&self, amount: i32, x: Option<i32>, y: Option<i32>) -> Result<()>;
fn key_down(&self, key: &str) -> Result<()>;
fn key_up(&self, key: &str) -> Result<()>;
fn press(&self, key: &str) -> Result<()>;
fn hotkey(&self, keys: &[String]) -> Result<()>;
fn type_char(&self, c: char) -> Result<()>;
fn type_text(&self, text: &str, interval: f64) -> Result<()>;
fn screenshot(&self, region: Option<&[i32]>) -> Result<Vec<u8>>;
fn get_pixel(&self, x: i32, y: i32) -> Result<(u8, u8, u8)>;
fn get_active_window(&self) -> Result<WindowInfo>;
fn list_windows(&self) -> Result<Vec<WindowInfo>>;
fn focus_window(&self, title: &str) -> Result<bool>;
fn minimize_window(&self, title: &str) -> Result<bool>;
fn maximize_window(&self, title: &str) -> Result<bool>;
fn resize_window(&self, title: &str, width: i32, height: i32) -> Result<bool>;
fn move_window(&self, title: &str, x: i32, y: i32) -> Result<bool>;
fn close_window(&self, title: &str) -> Result<bool>;
}
fn get_native_control() -> Box<dyn NativeControl> {
#[cfg(target_os = "macos")]
{
Box::new(macos::MacOSControl::new())
}
#[cfg(target_os = "linux")]
{
Box::new(linux::LinuxControl::new())
}
#[cfg(target_os = "windows")]
{
Box::new(windows::WindowsControl::new())
}
#[cfg(not(any(target_os = "macos", target_os = "linux", target_os = "windows")))]
{
panic!("Unsupported platform");
}
}
pub struct ComputerTool {
control: Arc<dyn NativeControl>,
defined_regions: HashMap<String, (i32, i32, i32, i32)>,
pause: f64,
failsafe: bool,
}
impl ComputerTool {
pub fn new() -> Self {
Self {
control: Arc::from(get_native_control()),
defined_regions: HashMap::new(),
pause: 0.1,
failsafe: true,
}
}
pub async fn execute(&mut self, args: ComputerToolArgs) -> Result<String> {
let action: UiAction = if args.action.is_empty() {
UiAction::Info
} else {
args.action.parse()?
};
let ctrl = Arc::clone(&self.control);
let result = match action {
UiAction::Click => {
let x = args.x.ok_or_else(|| anyhow!("x required"))?;
let y = args.y.ok_or_else(|| anyhow!("y required"))?;
let button = args.button.clone();
tokio::task::spawn_blocking(move || ctrl.click(x, y, &button)).await??;
json!({"success": true, "clicked": [x, y], "button": args.button})
}
UiAction::DoubleClick => {
let x = args.x.ok_or_else(|| anyhow!("x required"))?;
let y = args.y.ok_or_else(|| anyhow!("y required"))?;
tokio::task::spawn_blocking(move || ctrl.double_click(x, y)).await??;
json!({"success": true, "double_clicked": [x, y]})
}
UiAction::RightClick => {
let x = args.x.ok_or_else(|| anyhow!("x required"))?;
let y = args.y.ok_or_else(|| anyhow!("y required"))?;
tokio::task::spawn_blocking(move || ctrl.click(x, y, "right")).await??;
json!({"success": true, "right_clicked": [x, y]})
}
UiAction::MiddleClick => {
let x = args.x.ok_or_else(|| anyhow!("x required"))?;
let y = args.y.ok_or_else(|| anyhow!("y required"))?;
tokio::task::spawn_blocking(move || ctrl.click(x, y, "middle")).await??;
json!({"success": true, "middle_clicked": [x, y]})
}
UiAction::Move => {
let x = args.x.ok_or_else(|| anyhow!("x required"))?;
let y = args.y.ok_or_else(|| anyhow!("y required"))?;
tokio::task::spawn_blocking(move || ctrl.move_to(x, y)).await??;
json!({"success": true, "moved_to": [x, y]})
}
UiAction::MoveRelative => {
let dx = args.dx.ok_or_else(|| anyhow!("dx required"))?;
let dy = args.dy.ok_or_else(|| anyhow!("dy required"))?;
let (cx, cy) = tokio::task::spawn_blocking({
let ctrl = Arc::clone(&ctrl);
move || ctrl.mouse_position()
}).await??;
tokio::task::spawn_blocking(move || ctrl.move_to(cx + dx, cy + dy)).await??;
json!({"success": true, "moved_by": [dx, dy]})
}
UiAction::Drag => {
let x = args.x.ok_or_else(|| anyhow!("x required"))?;
let y = args.y.ok_or_else(|| anyhow!("y required"))?;
let (start_x, start_y) = tokio::task::spawn_blocking({
let ctrl = Arc::clone(&ctrl);
move || ctrl.mouse_position()
}).await??;
let end_x = args.end_x.unwrap_or(x);
let end_y = args.end_y.unwrap_or(y);
let button = args.button.clone();
tokio::task::spawn_blocking(move || {
ctrl.drag(start_x, start_y, end_x, end_y, &button)
}).await??;
json!({"success": true, "dragged_to": [end_x, end_y]})
}
UiAction::DragRelative => {
let dx = args.dx.ok_or_else(|| anyhow!("dx required"))?;
let dy = args.dy.ok_or_else(|| anyhow!("dy required"))?;
let (cx, cy) = tokio::task::spawn_blocking({
let ctrl = Arc::clone(&ctrl);
move || ctrl.mouse_position()
}).await??;
let button = args.button.clone();
tokio::task::spawn_blocking(move || {
ctrl.drag(cx, cy, cx + dx, cy + dy, &button)
}).await??;
json!({"success": true, "dragged_by": [dx, dy]})
}
UiAction::Scroll => {
let amount = args.amount.ok_or_else(|| anyhow!("amount required"))?;
let x = args.x;
let y = args.y;
tokio::task::spawn_blocking(move || ctrl.scroll(amount, x, y)).await??;
json!({"success": true, "scrolled": amount})
}
UiAction::Type => {
let text = args.text.ok_or_else(|| anyhow!("text required"))?;
let len = text.len();
let interval = args.interval;
tokio::task::spawn_blocking(move || ctrl.type_text(&text, interval)).await??;
json!({"success": true, "typed": len})
}
UiAction::Write => {
let text = args.text.ok_or_else(|| anyhow!("text required"))?;
let len = text.len();
if args.clear {
#[cfg(target_os = "macos")]
let keys = vec!["command".to_string(), "a".to_string()];
#[cfg(not(target_os = "macos"))]
let keys = vec!["ctrl".to_string(), "a".to_string()];
tokio::task::spawn_blocking({
let ctrl = Arc::clone(&ctrl);
move || ctrl.hotkey(&keys)
}).await??;
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
}
let interval = args.interval;
tokio::task::spawn_blocking(move || ctrl.type_text(&text, interval)).await??;
json!({"success": true, "wrote": len, "cleared": args.clear})
}
UiAction::Press => {
let key = args.key.ok_or_else(|| anyhow!("key required"))?;
let key_clone = key.clone();
tokio::task::spawn_blocking(move || ctrl.press(&key_clone)).await??;
json!({"success": true, "pressed": key})
}
UiAction::KeyDown => {
let key = args.key.ok_or_else(|| anyhow!("key required"))?;
let key_clone = key.clone();
tokio::task::spawn_blocking(move || ctrl.key_down(&key_clone)).await??;
json!({"success": true, "key_down": key})
}
UiAction::KeyUp => {
let key = args.key.ok_or_else(|| anyhow!("key required"))?;
let key_clone = key.clone();
tokio::task::spawn_blocking(move || ctrl.key_up(&key_clone)).await??;
json!({"success": true, "key_up": key})
}
UiAction::Hotkey => {
let keys = args.keys.ok_or_else(|| anyhow!("keys required"))?;
let combo = keys.join("+");
tokio::task::spawn_blocking(move || ctrl.hotkey(&keys)).await??;
json!({"success": true, "hotkey": combo})
}
UiAction::Screenshot | UiAction::ScreenshotRegion => {
let region: Option<Vec<i32>> = args.region.clone();
let data = tokio::task::spawn_blocking(move || {
ctrl.screenshot(region.as_deref())
}).await??;
if let Some(name) = args.name {
let path = if name.starts_with('/') || name.starts_with('~') {
shellexpand::tilde(&name).to_string()
} else {
format!("{}/{}", std::env::temp_dir().display(), name)
};
let path = if !path.ends_with(".png") {
format!("{}.png", path)
} else {
path
};
tokio::fs::write(&path, &data).await?;
json!({
"success": true,
"format": "png",
"size": data.len(),
"path": path
})
} else {
use base64::{Engine, engine::general_purpose::STANDARD};
let b64 = STANDARD.encode(&data);
json!({
"success": true,
"format": "png",
"size": data.len(),
"base64": b64
})
}
}
UiAction::GetActiveWindow => {
let info = tokio::task::spawn_blocking(move || {
ctrl.get_active_window()
}).await??;
json!(info)
}
UiAction::ListWindows => {
let windows = tokio::task::spawn_blocking(move || {
ctrl.list_windows()
}).await??;
json!({"windows": windows, "count": windows.len()})
}
UiAction::FocusWindow => {
let title = args.title.or(args.text).ok_or_else(|| anyhow!("title required"))?;
let title_clone = title.clone();
let success = tokio::task::spawn_blocking(move || {
ctrl.focus_window(&title_clone)
}).await??;
json!({"success": success, "focused": title})
}
UiAction::GetScreens => {
let (w, h) = tokio::task::spawn_blocking(move || {
ctrl.screen_size()
}).await??;
json!([{"name": "Primary", "resolution": format!("{}x{}", w, h), "main": true}])
}
UiAction::ScreenSize => {
let (w, h) = tokio::task::spawn_blocking(move || {
ctrl.screen_size()
}).await??;
json!({"width": w, "height": h})
}
UiAction::Position => {
let (x, y) = tokio::task::spawn_blocking(move || {
ctrl.mouse_position()
}).await??;
json!({"x": x, "y": y})
}
UiAction::Sleep => {
let secs = args.value.ok_or_else(|| anyhow!("value required"))?;
tokio::time::sleep(std::time::Duration::from_secs_f64(secs)).await;
json!({"success": true, "slept": secs})
}
UiAction::SetPause => {
let val = args.value.ok_or_else(|| anyhow!("value required"))?;
self.pause = val;
json!({"success": true, "pause": self.pause})
}
UiAction::SetFailsafe => {
let val = args.value.ok_or_else(|| anyhow!("value required"))?;
self.failsafe = val != 0.0;
json!({"success": true, "failsafe": self.failsafe})
}
UiAction::Batch => {
let actions = args.actions.ok_or_else(|| anyhow!("actions required"))?;
let start = std::time::Instant::now();
let mut results = Vec::new();
for (i, action_val) in actions.iter().enumerate() {
let action_args: ComputerToolArgs = serde_json::from_value(action_val.clone())
.unwrap_or_default();
match Box::pin(self.execute(action_args)).await {
Ok(_) => {
results.push(json!({"index": i, "success": true}));
}
Err(e) => {
results.push(json!({"index": i, "error": e.to_string()}));
}
}
}
let elapsed = start.elapsed().as_millis();
json!({
"success": true,
"count": results.len(),
"elapsed_ms": elapsed,
"results": results
})
}
UiAction::Info => {
let ctrl2 = Arc::clone(&ctrl);
let (mx, my) = tokio::task::spawn_blocking(move || {
ctrl.mouse_position()
}).await?.unwrap_or((0, 0));
let ctrl3 = Arc::clone(&ctrl2);
let (sw, sh) = tokio::task::spawn_blocking(move || {
ctrl2.screen_size()
}).await?.unwrap_or((0, 0));
let platform_info = ctrl3.platform_info();
json!({
"screen": {"width": sw, "height": sh},
"mouse": {"x": mx, "y": my},
"platform": platform_info,
"pause": self.pause,
"failsafe": self.failsafe,
"regions": self.defined_regions.keys().collect::<Vec<_>>()
})
}
};
Ok(serde_json::to_string(&result)?)
}
}
#[derive(Debug, Serialize, Deserialize)]
pub struct ComputerToolDefinition {
pub name: String,
pub description: String,
pub input_schema: Value,
}
impl ComputerToolDefinition {
pub fn new() -> Self {
let platform = std::env::consts::OS;
let backend = match platform {
"macos" => "quartz",
"linux" => "x11",
"windows" => "win32",
_ => "unknown",
};
Self {
name: "computer".to_string(),
description: format!(
r#"Control local computer with native API acceleration.
PLATFORM: {}
BACKENDS: {}
MOUSE (< 5ms native):
- click(x, y) / double_click / right_click / middle_click
- move(x, y) / move_relative(dx, dy)
- drag(x, y) / drag_relative(dx, dy)
- scroll(amount, x, y)
KEYBOARD (< 2ms native):
- type(text, interval): Type text
- write(text, clear): Type with optional clear
- press(key): Press and release key
- key_down(key) / key_up(key): Hold/release
- hotkey(keys): Key combination ["command", "c"]
SCREEN (< 50ms native):
- screenshot() / screenshot_region(region)
- get_screens(): List displays
- screen_size() / position()
WINDOWS:
- get_active_window(): Frontmost window info
- list_windows(): All windows with bounds
- focus_window(title): Activate window
BATCH:
- batch(actions): Execute multiple actions
INFO:
- info()
Examples:
ui(action="click", x=100, y=200)
ui(action="type", text="Hello")
ui(action="hotkey", keys=["command", "c"])
ui(action="screenshot")
ui(action="batch", actions=[
{{"action": "click", "x": 100, "y": 200}},
{{"action": "type", "text": "test"}}
])"#,
platform, backend
),
input_schema: json!({
"type": "object",
"properties": {
"action": {
"type": "string",
"description": "Action to perform",
"default": "info"
},
"x": {"type": "integer", "description": "X coordinate"},
"y": {"type": "integer", "description": "Y coordinate"},
"dx": {"type": "integer", "description": "Delta X"},
"dy": {"type": "integer", "description": "Delta Y"},
"end_x": {"type": "integer", "description": "End X for drag"},
"end_y": {"type": "integer", "description": "End Y for drag"},
"text": {"type": "string", "description": "Text to type"},
"key": {"type": "string", "description": "Key to press"},
"keys": {
"type": "array",
"items": {"type": "string"},
"description": "Keys for hotkey"
},
"button": {
"type": "string",
"description": "Mouse button",
"default": "left"
},
"amount": {"type": "integer", "description": "Scroll amount"},
"duration": {"type": "number", "description": "Duration", "default": 0.25},
"interval": {"type": "number", "description": "Type interval", "default": 0.02},
"region": {
"type": "array",
"items": {"type": "integer"},
"description": "Region [x,y,w,h]"
},
"clear": {"type": "boolean", "description": "Clear before write", "default": false},
"title": {"type": "string", "description": "Window title"},
"name": {"type": "string", "description": "Screenshot filename"},
"value": {"type": "number", "description": "Value for settings"},
"actions": {
"type": "array",
"items": {"type": "object"},
"description": "Batch actions"
}
}
}),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn test_info_action() {
let mut tool = ComputerTool::new();
let args = ComputerToolArgs {
action: "info".to_string(),
..Default::default()
};
let result = tool.execute(args).await;
assert!(result.is_ok());
let output = result.unwrap();
assert!(output.contains("screen"));
assert!(output.contains("mouse"));
}
#[tokio::test]
async fn test_position_action() {
let mut tool = ComputerTool::new();
let args = ComputerToolArgs {
action: "position".to_string(),
..Default::default()
};
let result = tool.execute(args).await;
assert!(result.is_ok());
let output = result.unwrap();
assert!(output.contains("x"));
assert!(output.contains("y"));
}
#[tokio::test]
async fn test_screen_size_action() {
let mut tool = ComputerTool::new();
let args = ComputerToolArgs {
action: "screen_size".to_string(),
..Default::default()
};
let result = tool.execute(args).await;
assert!(result.is_ok());
let output = result.unwrap();
assert!(output.contains("width"));
assert!(output.contains("height"));
}
}