rsclaw-desktop 0.1.0

Desktop crate for RsClaw — internal workspace crate, not for direct use
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125

//! Desktop automation primitives for WASM plugins.
//!
//! Provides cross-platform window management, input synthesis, and
//! screenshot capture.  Backed by `enigo` (input) and `xcap`
//! (screenshot) with platform-specific window queries.

use anyhow::Result;

// ---------------------------------------------------------------------------
// Trait
// ---------------------------------------------------------------------------

/// Cross-platform desktop session.
#[async_trait::async_trait]
pub trait DesktopSession: Send + Sync {
    /// Activate an application by bundle-id (macOS), exe name (Windows), or
    /// WM_CLASS (Linux). Returns "ok" on success.
    async fn activate_app(&self, bundle_id: &str) -> Result<String, String>;

    /// List all windows of the target app. Returns JSON array:
    /// `[{"idx":1,"title":"...","x":0,"y":0,"w":900,"h":600}]`
    async fn list_windows(&self, bundle_id: &str) -> Result<String, String>;

    /// Close a specific window by its index (from list-windows).
    async fn close_window(&self, bundle_id: &str, window_idx: u32) -> Result<String, String>;

    /// Get the main window bounds (x, y, w, h) as JSON.
    /// Prefers title=="Weixin", falls back to largest window.
    async fn get_main_window(&self, bundle_id: &str) -> Result<String, String>;

    /// Screenshot the app's main window. Returns `data:image/png;base64,...`.
    async fn screenshot_window(&self, bundle_id: &str) -> Result<String, String>;

    /// Screenshot a screen region. Returns data URI.
    async fn screenshot_region(&self, x: u32, y: u32, w: u32, h: u32) -> Result<String, String>;

    /// Scan an absolute screen region for pixels near a target RGB colour
    /// (per-channel `tolerance`). Cross-platform (xcap capture + pure pixel
    /// math, no OCR/VLM) — used for red-badge presence detection. Returns JSON
    /// `{"hit":bool,"count":N,"total":T,"ratio":f}`, `hit = count >= min_count`.
    #[allow(clippy::too_many_arguments)]
    async fn region_has_color(
        &self,
        x: u32,
        y: u32,
        w: u32,
        h: u32,
        r: u32,
        g: u32,
        b: u32,
        tolerance: u32,
        min_count: u32,
    ) -> Result<String, String>;

    /// OCR the app's main window with on-device text recognition (macOS
    /// Vision). Returns a JSON array of recognised lines with 0-1000
    /// relative centre coordinates: `[{"text":"...","x":143,"y":699}, ...]`.
    /// Precise enough to click a named row, unlike VLM grounding.
    async fn ocr_window(&self, bundle_id: &str) -> Result<String, String>;

    /// Move the mouse cursor to absolute screen coordinates without clicking.
    async fn mouse_move(&self, x: u32, y: u32) -> Result<String, String>;

    /// Mouse left-click at absolute screen coordinates.
    async fn mouse_click(&self, x: u32, y: u32) -> Result<String, String>;

    /// Mouse double-click at absolute screen coordinates.
    async fn mouse_double_click(&self, x: u32, y: u32) -> Result<String, String>;

    /// Drag from (x1, y1) to (x2, y2).
    async fn mouse_drag(&self, x1: u32, y1: u32, x2: u32, y2: u32) -> Result<String, String>;

    /// Scroll the mouse wheel. Positive clicks = down, negative = up.
    async fn mouse_scroll(&self, clicks: i32) -> Result<String, String>;

    /// Press a key with optional modifiers.
    /// key: key name (e.g. "Return", "Escape", "v", "k").
    /// modifiers: list of "command", "control", "shift", "option"/"alt".
    async fn key_press(&self, key: &str, modifiers: &[String]) -> Result<String, String>;

    /// Set the system clipboard text.
    async fn clipboard_set(&self, text: &str) -> Result<String, String>;

    /// Get the system clipboard text.
    async fn clipboard_get(&self) -> Result<String, String>;

    /// Set the system clipboard to a file reference (for paste-as-file).
    async fn clipboard_set_file(&self, file_path: &str) -> Result<String, String>;

    /// Get image data (PNG) from the system clipboard. Returns data URI.
    async fn clipboard_get_image(&self) -> Result<String, String>;

    /// Mouse right-click at absolute screen coordinates.
    async fn mouse_right_click(&self, x: u32, y: u32) -> Result<String, String>;

    /// Open a native file dialog. Returns the selected file path or error.
    async fn file_dialog_open(&self, title: &str, filters: &[String]) -> Result<String, String>;
}

// ---------------------------------------------------------------------------
// Factory
// ---------------------------------------------------------------------------

/// Create a platform-appropriate desktop session.
pub fn create_session() -> Box<dyn DesktopSession> {
    #[cfg(target_os = "macos")]
    {
        Box::new(native::NativeDesktopSession)
    }
    #[cfg(target_os = "windows")]
    {
        Box::new(native::NativeDesktopSession)
    }
    #[cfg(target_os = "linux")]
    {
        Box::new(native::NativeDesktopSession)
    }
    #[cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))]
    {
        compile_error!("DesktopSession not supported on this platform")
    }
}

mod native;
pub mod macos_perm;