mermaid_cli/providers/tool/computer_use/
mod.rs

1//! Computer-use tools — screenshot capture, mouse + keyboard control.
2//!
3//! Seven tools total (`screenshot`, `click`, `type_text`, `press_key`,
4//! `scroll`, `mouse_move`, `list_windows`) share one `ComputerUseDriver`.
5//! The driver owns the platform-specific subprocess dispatch (scrot/
6//! xdotool on X11, grim/ydotool on Wayland, screencapture/cliclick on
7//! macOS) and the `ScreenshotRegistry` — a small LRU buffer of recent
8//! capture metadata so the model can pass `screenshot_id` on
9//! `click`/`mouse_move` to lock coordinates to a specific capture.
10//!
11//! Registration is gated two ways:
12//! - `TuiMode::Headless` (`mermaid run <prompt>`) never registers any
13//!   computer-use tool regardless of what the display probes say —
14//!   a CI job has no user to watch a screenshot.
15//! - `Backend::probe()` runs an eager capability check at startup
16//!   (env vars + required binaries + `xdpyinfo` smoke test). If the
17//!   result is `Unsupported`, no tools register.
18//!
19//! The driver ALSO exposes `ensure_alive()` which every tool calls at
20//! the top of `execute`. It's a cheap re-probe that catches the
21//! "`DISPLAY=:0` ghost" case: env looks right, binaries exist, but
22//! the X server is actually unreachable (SSH forwarding without an
23//! X server, detached display, laptop lid closed).
24
25pub mod click;
26pub mod driver;
27pub mod list_windows;
28pub mod mouse_move;
29pub mod press_key;
30pub mod screenshot;
31pub mod scroll;
32pub mod type_text;
33
34use mermaid_domain::ProgressEvent;
35use std::process::Command;
36
37use serde_json::Value;
38
39use crate::providers::ctx::ExecContext;
40use mermaid_domain::{ToolMetadata, ToolOutcome, ToolRunMetadata};
41
42pub use click::ClickTool;
43pub use driver::ComputerUseDriver;
44pub use list_windows::ListWindowsTool;
45pub use mouse_move::MouseMoveTool;
46pub use press_key::PressKeyTool;
47pub use screenshot::ScreenshotTool;
48pub use scroll::ScrollTool;
49pub use type_text::TypeTextTool;
50
51/// Platform / display-server the driver dispatches to.
52#[derive(Debug, Clone, Copy, PartialEq, Eq)]
53pub enum Backend {
54    X11,
55    Wayland,
56    MacOS,
57    Windows,
58    Unsupported,
59}
60
61impl Backend {
62    /// Whether the driver has any tools it can run on this backend.
63    #[must_use]
64    pub fn is_usable(self) -> bool {
65        !matches!(self, Self::Unsupported)
66    }
67
68    /// Whether this backend can inject pointer + keyboard events (click,
69    /// `type_text`, `press_key`, scroll, `mouse_move`). X11 (xdotool) and Wayland
70    /// (ydotool/wtype) only; macOS capture works via `screencapture`, but the
71    /// input verbs are unimplemented and `bail!` in the driver, so they must
72    /// not be advertised there (#35). Windows is a stub.
73    #[must_use]
74    pub fn supports_input_injection(self) -> bool {
75        matches!(self, Self::X11 | Self::Wayland)
76    }
77
78    /// Whether this backend can enumerate windows (`list_windows`). X11 only —
79    /// via `xdotool search`; Wayland has no portable primitive and the driver
80    /// `bail!`s (#35).
81    #[must_use]
82    pub fn supports_window_listing(self) -> bool {
83        matches!(self, Self::X11)
84    }
85}
86
87/// Eager probe. Runs at startup to decide registration — does the
88/// right binary exist? Is the display reachable? Returns
89/// `Backend::Unsupported` when mermaid can't drive the display even
90/// though env vars might suggest otherwise (e.g. SSH forwarding).
91#[must_use]
92pub fn probe() -> Backend {
93    if cfg!(target_os = "macos") {
94        if has_command("screencapture") {
95            return Backend::MacOS;
96        }
97        return Backend::Unsupported;
98    }
99    if cfg!(target_os = "windows") {
100        // Windows backend is a v0.6 stub — not wired here. Once a
101        // real impl lands, probe PowerShell / SendInput here.
102        return Backend::Unsupported;
103    }
104
105    // Linux: try Wayland first (prefer if both are set).
106    if std::env::var("WAYLAND_DISPLAY").is_ok()
107        && has_command("grim")
108        && (has_command("ydotool") || has_command("wtype"))
109    {
110        return Backend::Wayland;
111    }
112
113    // Linux: fall back to X11. The xdpyinfo probe catches the ghost
114    // case — DISPLAY is set but no X server responds (common over
115    // SSH without X forwarding, or after a stale SSH reconnect).
116    if std::env::var("DISPLAY").is_ok()
117        && has_command("scrot")
118        && has_command("xdotool")
119        && xdpyinfo_alive()
120    {
121        return Backend::X11;
122    }
123
124    Backend::Unsupported
125}
126
127/// Quick re-probe used by `ComputerUseDriver::ensure_alive`. Cheaper
128/// than the full `probe()` — just checks the display answers — so
129/// every tool call can afford it.
130#[must_use]
131pub fn display_is_reachable(backend: Backend) -> bool {
132    match backend {
133        Backend::X11 => xdpyinfo_alive(),
134        Backend::Wayland => std::env::var("WAYLAND_DISPLAY").is_ok(),
135        Backend::MacOS | Backend::Windows => true,
136        Backend::Unsupported => false,
137    }
138}
139
140pub(super) fn has_command(name: &str) -> bool {
141    // `which` returns 0 iff the binary is on PATH. Cheap and universal
142    // across Linux + macOS; Windows would want `where.exe` but
143    // computer-use on Windows is stubbed out anyway.
144    Command::new("which")
145        .arg(name)
146        .output()
147        .map(|o| o.status.success() && !o.stdout.is_empty())
148        .unwrap_or(false)
149}
150
151/// Exit-0 check on `xdpyinfo` with a 200ms timeout. This is the
152/// difference between "`DISPLAY` is set" and "an X server will
153/// actually answer us."
154fn xdpyinfo_alive() -> bool {
155    if !has_command("xdpyinfo") {
156        // Some minimal X setups don't ship xdpyinfo. Fall back to a
157        // `xdotool getactivewindow` probe (we already require
158        // xdotool for clicks anyway).
159        return Command::new("xdotool")
160            .arg("getactivewindow")
161            .output()
162            .map(|o| o.status.success())
163            .unwrap_or(false);
164    }
165    // Use a timeout wrapper so a wedged display doesn't hang startup.
166    match Command::new("timeout").arg("0.2").arg("xdpyinfo").output() {
167        Ok(o) => o.status.success(),
168        Err(_) => {
169            // `timeout` not available (macOS older versions). Fall
170            // back to a direct call — shouldn't happen on Linux X11.
171            Command::new("xdpyinfo")
172                .output()
173                .map(|o| o.status.success())
174                .unwrap_or(false)
175        },
176    }
177}
178
179pub(super) fn computer_use_success(
180    action: &'static str,
181    params: Value,
182    output: String,
183    duration_secs: f64,
184) -> ToolOutcome {
185    ToolOutcome::success(output, format!("{action} completed"), duration_secs).with_metadata(
186        ToolRunMetadata {
187            detail: ToolMetadata::ComputerUse {
188                action: action.to_string(),
189                params,
190            },
191            ..ToolRunMetadata::default()
192        },
193    )
194}
195
196/// Shared post-action auto-screenshot for click / `type_text` / `press_key`.
197///
198/// When `computer_use.auto_screenshot` is enabled, captures the focused window,
199/// emits an inline `Artifact` preview on the progress channel, and returns
200/// `(summary, base64_png)` for the caller to fold into its outcome. Returns
201/// `None` when the flag is off OR the best-effort capture failed — callers then
202/// build a screenshot-less outcome (#98). Gating lives here so the three tools
203/// share one decision point rather than three byte-identical blocks.
204pub(super) async fn emit_auto_screenshot(
205    driver: &ComputerUseDriver,
206    ctx: &ExecContext,
207    caption: &'static str,
208) -> Option<(String, String)> {
209    if !ctx.config.computer_use.auto_screenshot {
210        return None;
211    }
212    let (summary, base64_png) = driver.capture_focused_for_autoshot(&ctx.token).await?;
213    if let Ok(bytes) =
214        base64::Engine::decode(&base64::engine::general_purpose::STANDARD, &base64_png)
215    {
216        let _ = ctx
217            .progress
218            .send(ProgressEvent::Artifact {
219                mime: "image/png".to_string(),
220                data: bytes,
221                caption: Some(caption.to_string()),
222            })
223            .await;
224    }
225    Some((summary, base64_png))
226}
227
228#[cfg(test)]
229mod tests {
230    use super::*;
231
232    #[test]
233    fn backend_unsupported_is_not_usable() {
234        assert!(!Backend::Unsupported.is_usable());
235        assert!(Backend::X11.is_usable());
236        assert!(Backend::Wayland.is_usable());
237        assert!(Backend::MacOS.is_usable());
238    }
239
240    #[test]
241    fn input_injection_only_on_linux_backends() {
242        assert!(Backend::X11.supports_input_injection());
243        assert!(Backend::Wayland.supports_input_injection());
244        assert!(!Backend::MacOS.supports_input_injection());
245        assert!(!Backend::Windows.supports_input_injection());
246        assert!(!Backend::Unsupported.supports_input_injection());
247    }
248
249    #[test]
250    fn window_listing_only_on_x11() {
251        assert!(Backend::X11.supports_window_listing());
252        assert!(!Backend::Wayland.supports_window_listing());
253        assert!(!Backend::MacOS.supports_window_listing());
254    }
255
256    #[test]
257    fn probe_does_not_panic_on_headless() {
258        // In the test runner (no DISPLAY, no WAYLAND_DISPLAY on most
259        // CI envs), probe() must return Unsupported without panicking.
260        // We don't assert a specific result because dev machines may
261        // have a live display.
262        let _ = probe();
263    }
264
265    #[tokio::test]
266    async fn auto_screenshot_is_noop_when_disabled() {
267        use mermaid_domain::{ToolCallId, TurnId};
268        let mut cfg = mermaid_domain::Config::default();
269        cfg.computer_use.auto_screenshot = false;
270        let (tx, mut rx) = tokio::sync::mpsc::channel::<ProgressEvent>(8);
271        let ctx = ExecContext::new(
272            tokio_util::sync::CancellationToken::new(),
273            tx,
274            ToolCallId(1),
275            TurnId(1),
276            std::path::PathBuf::from("/tmp"),
277            std::sync::Arc::new(cfg),
278            String::new(),
279            None,
280            None,
281            None,
282            mermaid_runtime::SafetyMode::FullAccess,
283            None,
284            None,
285            None,
286            None,
287            None,
288        );
289        // Backend is irrelevant — the flag short-circuits before any capture.
290        let driver = ComputerUseDriver::new(Backend::Unsupported);
291        assert!(emit_auto_screenshot(&driver, &ctx, "test").await.is_none());
292        assert!(rx.try_recv().is_err(), "no artifact emitted when disabled");
293    }
294}
mermaid_cli/providers/tool/computer_use/mod.rs

mermaid_cli/providers/tool/computer_use/
mod.rs
