1use anyhow::{anyhow, Result};
14use serde::{Deserialize, Serialize};
15use serde_json::{json, Value};
16use std::collections::HashMap;
17use std::sync::Arc;
18
19#[cfg(target_os = "macos")]
20mod macos;
21
22#[cfg(target_os = "linux")]
23mod linux;
24
25#[cfg(target_os = "windows")]
26mod windows;
27
28#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
30#[serde(rename_all = "snake_case")]
31pub enum UiAction {
32 Click,
34 DoubleClick,
35 RightClick,
36 MiddleClick,
37 Move,
38 MoveRelative,
39 Drag,
40 DragRelative,
41 Scroll,
42 Type,
44 Write,
45 Press,
46 KeyDown,
47 KeyUp,
48 Hotkey,
49 Screenshot,
51 ScreenshotRegion,
52 GetActiveWindow,
54 ListWindows,
55 FocusWindow,
56 GetScreens,
58 ScreenSize,
59 Position,
60 Sleep,
62 SetPause,
63 SetFailsafe,
64 Batch,
66 Info,
68}
69
70impl Default for UiAction {
71 fn default() -> Self {
72 Self::Info
73 }
74}
75
76impl std::str::FromStr for UiAction {
77 type Err = anyhow::Error;
78
79 fn from_str(s: &str) -> Result<Self> {
80 match s.to_lowercase().as_str() {
81 "click" => Ok(Self::Click),
82 "double_click" | "doubleclick" => Ok(Self::DoubleClick),
83 "right_click" | "rightclick" => Ok(Self::RightClick),
84 "middle_click" | "middleclick" => Ok(Self::MiddleClick),
85 "move" => Ok(Self::Move),
86 "move_relative" | "moverelative" => Ok(Self::MoveRelative),
87 "drag" => Ok(Self::Drag),
88 "drag_relative" | "dragrelative" => Ok(Self::DragRelative),
89 "scroll" => Ok(Self::Scroll),
90 "type" => Ok(Self::Type),
91 "write" => Ok(Self::Write),
92 "press" => Ok(Self::Press),
93 "key_down" | "keydown" => Ok(Self::KeyDown),
94 "key_up" | "keyup" => Ok(Self::KeyUp),
95 "hotkey" => Ok(Self::Hotkey),
96 "screenshot" => Ok(Self::Screenshot),
97 "screenshot_region" | "screenshotregion" => Ok(Self::ScreenshotRegion),
98 "get_active_window" | "getactivewindow" => Ok(Self::GetActiveWindow),
99 "list_windows" | "listwindows" => Ok(Self::ListWindows),
100 "focus_window" | "focuswindow" => Ok(Self::FocusWindow),
101 "get_screens" | "getscreens" => Ok(Self::GetScreens),
102 "screen_size" | "screensize" => Ok(Self::ScreenSize),
103 "position" => Ok(Self::Position),
104 "sleep" => Ok(Self::Sleep),
105 "set_pause" | "setpause" => Ok(Self::SetPause),
106 "set_failsafe" | "setfailsafe" => Ok(Self::SetFailsafe),
107 "batch" => Ok(Self::Batch),
108 "info" => Ok(Self::Info),
109 _ => Err(anyhow!("Unknown action: {}", s)),
110 }
111 }
112}
113
114#[derive(Debug, Clone, Default, Serialize, Deserialize)]
116pub struct ComputerToolArgs {
117 #[serde(default)]
118 pub action: String,
119 pub x: Option<i32>,
121 pub y: Option<i32>,
122 pub dx: Option<i32>,
123 pub dy: Option<i32>,
124 pub end_x: Option<i32>,
125 pub end_y: Option<i32>,
126 pub text: Option<String>,
128 pub key: Option<String>,
129 pub keys: Option<Vec<String>>,
130 #[serde(default = "default_button")]
132 pub button: String,
133 pub amount: Option<i32>,
134 #[serde(default = "default_duration")]
135 pub duration: f64,
136 #[serde(default = "default_interval")]
137 pub interval: f64,
138 pub region: Option<Vec<i32>>,
139 #[serde(default)]
140 pub clear: bool,
141 pub title: Option<String>,
143 pub name: Option<String>,
145 pub width: Option<i32>,
147 pub height: Option<i32>,
148 pub value: Option<f64>,
150 pub actions: Option<Vec<Value>>,
152}
153
154fn default_button() -> String {
155 "left".to_string()
156}
157
158fn default_duration() -> f64 {
159 0.25
160}
161
162fn default_interval() -> f64 {
163 0.02
164}
165
166#[derive(Debug, Clone, Serialize, Deserialize)]
168pub struct WindowInfo {
169 pub app: Option<String>,
170 pub title: String,
171 pub x: i32,
172 pub y: i32,
173 pub width: i32,
174 pub height: i32,
175}
176
177#[derive(Debug, Clone, Serialize, Deserialize)]
179pub struct PlatformInfo {
180 pub platform: String,
181 pub native_available: bool,
182 pub backends: HashMap<String, bool>,
183}
184
185pub trait NativeControl: Send + Sync {
188 fn platform_info(&self) -> PlatformInfo;
191
192 fn mouse_position(&self) -> Result<(i32, i32)>;
195
196 fn screen_size(&self) -> Result<(i32, i32)>;
198
199 fn click(&self, x: i32, y: i32, button: &str) -> Result<()>;
202
203 fn double_click(&self, x: i32, y: i32) -> Result<()>;
205
206 fn move_to(&self, x: i32, y: i32) -> Result<()>;
208
209 fn drag(&self, start_x: i32, start_y: i32, end_x: i32, end_y: i32, button: &str) -> Result<()>;
211
212 fn scroll(&self, amount: i32, x: Option<i32>, y: Option<i32>) -> Result<()>;
214
215 fn key_down(&self, key: &str) -> Result<()>;
218
219 fn key_up(&self, key: &str) -> Result<()>;
221
222 fn press(&self, key: &str) -> Result<()>;
224
225 fn hotkey(&self, keys: &[String]) -> Result<()>;
227
228 fn type_char(&self, c: char) -> Result<()>;
230
231 fn type_text(&self, text: &str, interval: f64) -> Result<()>;
233
234 fn screenshot(&self, region: Option<&[i32]>) -> Result<Vec<u8>>;
237
238 fn get_pixel(&self, x: i32, y: i32) -> Result<(u8, u8, u8)>;
240
241 fn get_active_window(&self) -> Result<WindowInfo>;
244
245 fn list_windows(&self) -> Result<Vec<WindowInfo>>;
247
248 fn focus_window(&self, title: &str) -> Result<bool>;
250
251 fn minimize_window(&self, title: &str) -> Result<bool>;
253
254 fn maximize_window(&self, title: &str) -> Result<bool>;
256
257 fn resize_window(&self, title: &str, width: i32, height: i32) -> Result<bool>;
259
260 fn move_window(&self, title: &str, x: i32, y: i32) -> Result<bool>;
262
263 fn close_window(&self, title: &str) -> Result<bool>;
265}
266
267fn get_native_control() -> Box<dyn NativeControl> {
269 #[cfg(target_os = "macos")]
270 {
271 Box::new(macos::MacOSControl::new())
272 }
273
274 #[cfg(target_os = "linux")]
275 {
276 Box::new(linux::LinuxControl::new())
277 }
278
279 #[cfg(target_os = "windows")]
280 {
281 Box::new(windows::WindowsControl::new())
282 }
283
284 #[cfg(not(any(target_os = "macos", target_os = "linux", target_os = "windows")))]
285 {
286 panic!("Unsupported platform");
287 }
288}
289
290pub struct ComputerTool {
292 control: Arc<dyn NativeControl>,
293 defined_regions: HashMap<String, (i32, i32, i32, i32)>,
294 pause: f64,
295 failsafe: bool,
296}
297
298impl ComputerTool {
299 pub fn new() -> Self {
300 Self {
301 control: Arc::from(get_native_control()),
302 defined_regions: HashMap::new(),
303 pause: 0.1,
304 failsafe: true,
305 }
306 }
307
308 pub async fn execute(&mut self, args: ComputerToolArgs) -> Result<String> {
309 let action: UiAction = if args.action.is_empty() {
310 UiAction::Info
311 } else {
312 args.action.parse()?
313 };
314
315 let ctrl = Arc::clone(&self.control);
317
318 let result = match action {
319 UiAction::Click => {
321 let x = args.x.ok_or_else(|| anyhow!("x required"))?;
322 let y = args.y.ok_or_else(|| anyhow!("y required"))?;
323 let button = args.button.clone();
324 tokio::task::spawn_blocking(move || ctrl.click(x, y, &button)).await??;
325 json!({"success": true, "clicked": [x, y], "button": args.button})
326 }
327
328 UiAction::DoubleClick => {
329 let x = args.x.ok_or_else(|| anyhow!("x required"))?;
330 let y = args.y.ok_or_else(|| anyhow!("y required"))?;
331 tokio::task::spawn_blocking(move || ctrl.double_click(x, y)).await??;
333 json!({"success": true, "double_clicked": [x, y]})
334 }
335
336 UiAction::RightClick => {
337 let x = args.x.ok_or_else(|| anyhow!("x required"))?;
338 let y = args.y.ok_or_else(|| anyhow!("y required"))?;
339 tokio::task::spawn_blocking(move || ctrl.click(x, y, "right")).await??;
340 json!({"success": true, "right_clicked": [x, y]})
341 }
342
343 UiAction::MiddleClick => {
344 let x = args.x.ok_or_else(|| anyhow!("x required"))?;
345 let y = args.y.ok_or_else(|| anyhow!("y required"))?;
346 tokio::task::spawn_blocking(move || ctrl.click(x, y, "middle")).await??;
347 json!({"success": true, "middle_clicked": [x, y]})
348 }
349
350 UiAction::Move => {
351 let x = args.x.ok_or_else(|| anyhow!("x required"))?;
352 let y = args.y.ok_or_else(|| anyhow!("y required"))?;
353 tokio::task::spawn_blocking(move || ctrl.move_to(x, y)).await??;
354 json!({"success": true, "moved_to": [x, y]})
355 }
356
357 UiAction::MoveRelative => {
358 let dx = args.dx.ok_or_else(|| anyhow!("dx required"))?;
359 let dy = args.dy.ok_or_else(|| anyhow!("dy required"))?;
360 let (cx, cy) = tokio::task::spawn_blocking({
362 let ctrl = Arc::clone(&ctrl);
363 move || ctrl.mouse_position()
364 }).await??;
365 tokio::task::spawn_blocking(move || ctrl.move_to(cx + dx, cy + dy)).await??;
366 json!({"success": true, "moved_by": [dx, dy]})
367 }
368
369 UiAction::Drag => {
370 let x = args.x.ok_or_else(|| anyhow!("x required"))?;
371 let y = args.y.ok_or_else(|| anyhow!("y required"))?;
372 let (start_x, start_y) = tokio::task::spawn_blocking({
373 let ctrl = Arc::clone(&ctrl);
374 move || ctrl.mouse_position()
375 }).await??;
376 let end_x = args.end_x.unwrap_or(x);
377 let end_y = args.end_y.unwrap_or(y);
378 let button = args.button.clone();
379 tokio::task::spawn_blocking(move || {
381 ctrl.drag(start_x, start_y, end_x, end_y, &button)
382 }).await??;
383 json!({"success": true, "dragged_to": [end_x, end_y]})
384 }
385
386 UiAction::DragRelative => {
387 let dx = args.dx.ok_or_else(|| anyhow!("dx required"))?;
388 let dy = args.dy.ok_or_else(|| anyhow!("dy required"))?;
389 let (cx, cy) = tokio::task::spawn_blocking({
390 let ctrl = Arc::clone(&ctrl);
391 move || ctrl.mouse_position()
392 }).await??;
393 let button = args.button.clone();
394 tokio::task::spawn_blocking(move || {
395 ctrl.drag(cx, cy, cx + dx, cy + dy, &button)
396 }).await??;
397 json!({"success": true, "dragged_by": [dx, dy]})
398 }
399
400 UiAction::Scroll => {
401 let amount = args.amount.ok_or_else(|| anyhow!("amount required"))?;
402 let x = args.x;
403 let y = args.y;
404 tokio::task::spawn_blocking(move || ctrl.scroll(amount, x, y)).await??;
405 json!({"success": true, "scrolled": amount})
406 }
407
408 UiAction::Type => {
409 let text = args.text.ok_or_else(|| anyhow!("text required"))?;
410 let len = text.len();
411 let interval = args.interval;
412 tokio::task::spawn_blocking(move || ctrl.type_text(&text, interval)).await??;
414 json!({"success": true, "typed": len})
415 }
416
417 UiAction::Write => {
418 let text = args.text.ok_or_else(|| anyhow!("text required"))?;
419 let len = text.len();
420 if args.clear {
421 #[cfg(target_os = "macos")]
423 let keys = vec!["command".to_string(), "a".to_string()];
424 #[cfg(not(target_os = "macos"))]
425 let keys = vec!["ctrl".to_string(), "a".to_string()];
426 tokio::task::spawn_blocking({
427 let ctrl = Arc::clone(&ctrl);
428 move || ctrl.hotkey(&keys)
429 }).await??;
430 tokio::time::sleep(std::time::Duration::from_millis(50)).await;
431 }
432 let interval = args.interval;
433 tokio::task::spawn_blocking(move || ctrl.type_text(&text, interval)).await??;
434 json!({"success": true, "wrote": len, "cleared": args.clear})
435 }
436
437 UiAction::Press => {
438 let key = args.key.ok_or_else(|| anyhow!("key required"))?;
439 let key_clone = key.clone();
440 tokio::task::spawn_blocking(move || ctrl.press(&key_clone)).await??;
441 json!({"success": true, "pressed": key})
442 }
443
444 UiAction::KeyDown => {
445 let key = args.key.ok_or_else(|| anyhow!("key required"))?;
446 let key_clone = key.clone();
447 tokio::task::spawn_blocking(move || ctrl.key_down(&key_clone)).await??;
448 json!({"success": true, "key_down": key})
449 }
450
451 UiAction::KeyUp => {
452 let key = args.key.ok_or_else(|| anyhow!("key required"))?;
453 let key_clone = key.clone();
454 tokio::task::spawn_blocking(move || ctrl.key_up(&key_clone)).await??;
455 json!({"success": true, "key_up": key})
456 }
457
458 UiAction::Hotkey => {
459 let keys = args.keys.ok_or_else(|| anyhow!("keys required"))?;
460 let combo = keys.join("+");
461 tokio::task::spawn_blocking(move || ctrl.hotkey(&keys)).await??;
462 json!({"success": true, "hotkey": combo})
463 }
464
465 UiAction::Screenshot | UiAction::ScreenshotRegion => {
466 let region: Option<Vec<i32>> = args.region.clone();
467 let data = tokio::task::spawn_blocking(move || {
469 ctrl.screenshot(region.as_deref())
470 }).await??;
471
472 if let Some(name) = args.name {
474 let path = if name.starts_with('/') || name.starts_with('~') {
475 shellexpand::tilde(&name).to_string()
476 } else {
477 format!("{}/{}", std::env::temp_dir().display(), name)
478 };
479 let path = if !path.ends_with(".png") {
480 format!("{}.png", path)
481 } else {
482 path
483 };
484 tokio::fs::write(&path, &data).await?;
486 json!({
487 "success": true,
488 "format": "png",
489 "size": data.len(),
490 "path": path
491 })
492 } else {
493 use base64::{Engine, engine::general_purpose::STANDARD};
494 let b64 = STANDARD.encode(&data);
495 json!({
496 "success": true,
497 "format": "png",
498 "size": data.len(),
499 "base64": b64
500 })
501 }
502 }
503
504 UiAction::GetActiveWindow => {
505 let info = tokio::task::spawn_blocking(move || {
507 ctrl.get_active_window()
508 }).await??;
509 json!(info)
510 }
511
512 UiAction::ListWindows => {
513 let windows = tokio::task::spawn_blocking(move || {
515 ctrl.list_windows()
516 }).await??;
517 json!({"windows": windows, "count": windows.len()})
518 }
519
520 UiAction::FocusWindow => {
521 let title = args.title.or(args.text).ok_or_else(|| anyhow!("title required"))?;
522 let title_clone = title.clone();
523 let success = tokio::task::spawn_blocking(move || {
525 ctrl.focus_window(&title_clone)
526 }).await??;
527 json!({"success": success, "focused": title})
528 }
529
530 UiAction::GetScreens => {
531 let (w, h) = tokio::task::spawn_blocking(move || {
533 ctrl.screen_size()
534 }).await??;
535 json!([{"name": "Primary", "resolution": format!("{}x{}", w, h), "main": true}])
536 }
537
538 UiAction::ScreenSize => {
539 let (w, h) = tokio::task::spawn_blocking(move || {
540 ctrl.screen_size()
541 }).await??;
542 json!({"width": w, "height": h})
543 }
544
545 UiAction::Position => {
546 let (x, y) = tokio::task::spawn_blocking(move || {
548 ctrl.mouse_position()
549 }).await??;
550 json!({"x": x, "y": y})
551 }
552
553 UiAction::Sleep => {
554 let secs = args.value.ok_or_else(|| anyhow!("value required"))?;
555 tokio::time::sleep(std::time::Duration::from_secs_f64(secs)).await;
557 json!({"success": true, "slept": secs})
558 }
559
560 UiAction::SetPause => {
561 let val = args.value.ok_or_else(|| anyhow!("value required"))?;
562 self.pause = val;
563 json!({"success": true, "pause": self.pause})
564 }
565
566 UiAction::SetFailsafe => {
567 let val = args.value.ok_or_else(|| anyhow!("value required"))?;
568 self.failsafe = val != 0.0;
569 json!({"success": true, "failsafe": self.failsafe})
570 }
571
572 UiAction::Batch => {
573 let actions = args.actions.ok_or_else(|| anyhow!("actions required"))?;
574 let start = std::time::Instant::now();
575 let mut results = Vec::new();
576
577 for (i, action_val) in actions.iter().enumerate() {
578 let action_args: ComputerToolArgs = serde_json::from_value(action_val.clone())
579 .unwrap_or_default();
580
581 match Box::pin(self.execute(action_args)).await {
582 Ok(_) => {
583 results.push(json!({"index": i, "success": true}));
584 }
585 Err(e) => {
586 results.push(json!({"index": i, "error": e.to_string()}));
587 }
588 }
589 }
590
591 let elapsed = start.elapsed().as_millis();
592 json!({
593 "success": true,
594 "count": results.len(),
595 "elapsed_ms": elapsed,
596 "results": results
597 })
598 }
599
600 UiAction::Info => {
601 let ctrl2 = Arc::clone(&ctrl);
603 let (mx, my) = tokio::task::spawn_blocking(move || {
604 ctrl.mouse_position()
605 }).await?.unwrap_or((0, 0));
606 let ctrl3 = Arc::clone(&ctrl2);
607 let (sw, sh) = tokio::task::spawn_blocking(move || {
608 ctrl2.screen_size()
609 }).await?.unwrap_or((0, 0));
610 let platform_info = ctrl3.platform_info();
611
612 json!({
613 "screen": {"width": sw, "height": sh},
614 "mouse": {"x": mx, "y": my},
615 "platform": platform_info,
616 "pause": self.pause,
617 "failsafe": self.failsafe,
618 "regions": self.defined_regions.keys().collect::<Vec<_>>()
619 })
620 }
621 };
622
623 Ok(serde_json::to_string(&result)?)
624 }
625}
626
627#[derive(Debug, Serialize, Deserialize)]
629pub struct ComputerToolDefinition {
630 pub name: String,
631 pub description: String,
632 pub input_schema: Value,
633}
634
635impl ComputerToolDefinition {
636 pub fn new() -> Self {
637 let platform = std::env::consts::OS;
638 let backend = match platform {
639 "macos" => "quartz",
640 "linux" => "x11",
641 "windows" => "win32",
642 _ => "unknown",
643 };
644
645 Self {
646 name: "computer".to_string(),
647 description: format!(
648 r#"Control local computer with native API acceleration.
649
650PLATFORM: {}
651BACKENDS: {}
652
653MOUSE (< 5ms native):
654- click(x, y) / double_click / right_click / middle_click
655- move(x, y) / move_relative(dx, dy)
656- drag(x, y) / drag_relative(dx, dy)
657- scroll(amount, x, y)
658
659KEYBOARD (< 2ms native):
660- type(text, interval): Type text
661- write(text, clear): Type with optional clear
662- press(key): Press and release key
663- key_down(key) / key_up(key): Hold/release
664- hotkey(keys): Key combination ["command", "c"]
665
666SCREEN (< 50ms native):
667- screenshot() / screenshot_region(region)
668- get_screens(): List displays
669- screen_size() / position()
670
671WINDOWS:
672- get_active_window(): Frontmost window info
673- list_windows(): All windows with bounds
674- focus_window(title): Activate window
675
676BATCH:
677- batch(actions): Execute multiple actions
678
679INFO:
680- info()
681
682Examples:
683 ui(action="click", x=100, y=200)
684 ui(action="type", text="Hello")
685 ui(action="hotkey", keys=["command", "c"])
686 ui(action="screenshot")
687 ui(action="batch", actions=[
688 {{"action": "click", "x": 100, "y": 200}},
689 {{"action": "type", "text": "test"}}
690 ])"#,
691 platform, backend
692 ),
693 input_schema: json!({
694 "type": "object",
695 "properties": {
696 "action": {
697 "type": "string",
698 "description": "Action to perform",
699 "default": "info"
700 },
701 "x": {"type": "integer", "description": "X coordinate"},
702 "y": {"type": "integer", "description": "Y coordinate"},
703 "dx": {"type": "integer", "description": "Delta X"},
704 "dy": {"type": "integer", "description": "Delta Y"},
705 "end_x": {"type": "integer", "description": "End X for drag"},
706 "end_y": {"type": "integer", "description": "End Y for drag"},
707 "text": {"type": "string", "description": "Text to type"},
708 "key": {"type": "string", "description": "Key to press"},
709 "keys": {
710 "type": "array",
711 "items": {"type": "string"},
712 "description": "Keys for hotkey"
713 },
714 "button": {
715 "type": "string",
716 "description": "Mouse button",
717 "default": "left"
718 },
719 "amount": {"type": "integer", "description": "Scroll amount"},
720 "duration": {"type": "number", "description": "Duration", "default": 0.25},
721 "interval": {"type": "number", "description": "Type interval", "default": 0.02},
722 "region": {
723 "type": "array",
724 "items": {"type": "integer"},
725 "description": "Region [x,y,w,h]"
726 },
727 "clear": {"type": "boolean", "description": "Clear before write", "default": false},
728 "title": {"type": "string", "description": "Window title"},
729 "name": {"type": "string", "description": "Screenshot filename"},
730 "value": {"type": "number", "description": "Value for settings"},
731 "actions": {
732 "type": "array",
733 "items": {"type": "object"},
734 "description": "Batch actions"
735 }
736 }
737 }),
738 }
739 }
740}
741
742#[cfg(test)]
743mod tests {
744 use super::*;
745
746 #[tokio::test]
747 async fn test_info_action() {
748 let mut tool = ComputerTool::new();
749 let args = ComputerToolArgs {
750 action: "info".to_string(),
751 ..Default::default()
752 };
753
754 let result = tool.execute(args).await;
755 assert!(result.is_ok());
756 let output = result.unwrap();
757 assert!(output.contains("screen"));
758 assert!(output.contains("mouse"));
759 }
760
761 #[tokio::test]
762 async fn test_position_action() {
763 let mut tool = ComputerTool::new();
764 let args = ComputerToolArgs {
765 action: "position".to_string(),
766 ..Default::default()
767 };
768
769 let result = tool.execute(args).await;
770 assert!(result.is_ok());
771 let output = result.unwrap();
772 assert!(output.contains("x"));
773 assert!(output.contains("y"));
774 }
775
776 #[tokio::test]
777 async fn test_screen_size_action() {
778 let mut tool = ComputerTool::new();
779 let args = ComputerToolArgs {
780 action: "screen_size".to_string(),
781 ..Default::default()
782 };
783
784 let result = tool.execute(args).await;
785 assert!(result.is_ok());
786 let output = result.unwrap();
787 assert!(output.contains("width"));
788 assert!(output.contains("height"));
789 }
790}