mermaid_cli/providers/tool/computer_use/
mod.rs

1//! Computer-use tools — screenshot capture, mouse + keyboard control.
2//!
3//! Seven tools total (`screenshot`, `click`, `type_text`, `press_key`,
4//! `scroll`, `mouse_move`, `list_windows`) share one `ComputerUseDriver`.
5//! The driver owns the platform-specific subprocess dispatch (scrot/
6//! xdotool on X11, grim/ydotool on Wayland, screencapture/cliclick on
7//! macOS) and the `ScreenshotRegistry` — a small LRU buffer of recent
8//! capture metadata so the model can pass `screenshot_id` on
9//! `click`/`mouse_move` to lock coordinates to a specific capture.
10//!
11//! Registration is gated two ways:
12//! - `TuiMode::Headless` (`mermaid run <prompt>`) never registers any
13//!   computer-use tool regardless of what the display probes say —
14//!   a CI job has no user to watch a screenshot.
15//! - `Backend::probe()` runs an eager capability check at startup
16//!   (env vars + required binaries + `xdpyinfo` smoke test). If the
17//!   result is `Unsupported`, no tools register.
18//!
19//! The driver ALSO exposes `ensure_alive()` which every tool calls at
20//! the top of `execute`. It's a cheap re-probe that catches the
21//! "`DISPLAY=:0` ghost" case: env looks right, binaries exist, but
22//! the X server is actually unreachable (SSH forwarding without an
23//! X server, detached display, laptop lid closed).
24
25pub mod click;
26pub mod driver;
27pub mod list_windows;
28pub mod mouse_move;
29pub mod press_key;
30pub mod screenshot;
31pub mod scroll;
32pub mod type_text;
33
34use std::path::Path;
35use std::process::Command;
36
37use serde_json::Value;
38
39use crate::domain::{ToolMetadata, ToolOutcome, ToolRunMetadata};
40use crate::providers::ctx::{ExecContext, ProgressEvent};
41
42pub use click::ClickTool;
43pub use driver::ComputerUseDriver;
44pub use list_windows::ListWindowsTool;
45pub use mouse_move::MouseMoveTool;
46pub use press_key::PressKeyTool;
47pub use screenshot::ScreenshotTool;
48pub use scroll::ScrollTool;
49pub use type_text::TypeTextTool;
50
51/// Platform / display-server the driver dispatches to.
52#[derive(Debug, Clone, Copy, PartialEq, Eq)]
53pub enum Backend {
54    X11,
55    Wayland,
56    MacOS,
57    Windows,
58    Unsupported,
59}
60
61impl Backend {
62    /// Whether the driver has any tools it can run on this backend.
63    pub fn is_usable(self) -> bool {
64        !matches!(self, Backend::Unsupported)
65    }
66
67    /// Whether this backend can inject pointer + keyboard events (click,
68    /// type_text, press_key, scroll, mouse_move). X11 (xdotool) and Wayland
69    /// (ydotool/wtype) only; macOS capture works via `screencapture`, but the
70    /// input verbs are unimplemented and `bail!` in the driver, so they must
71    /// not be advertised there (#35). Windows is a stub.
72    pub fn supports_input_injection(self) -> bool {
73        matches!(self, Backend::X11 | Backend::Wayland)
74    }
75
76    /// Whether this backend can enumerate windows (`list_windows`). X11 only —
77    /// via `xdotool search`; Wayland has no portable primitive and the driver
78    /// `bail!`s (#35).
79    pub fn supports_window_listing(self) -> bool {
80        matches!(self, Backend::X11)
81    }
82}
83
84/// Eager probe. Runs at startup to decide registration — does the
85/// right binary exist? Is the display reachable? Returns
86/// `Backend::Unsupported` when mermaid can't drive the display even
87/// though env vars might suggest otherwise (e.g. SSH forwarding).
88pub fn probe() -> Backend {
89    if cfg!(target_os = "macos") {
90        if has_command("screencapture") {
91            return Backend::MacOS;
92        }
93        return Backend::Unsupported;
94    }
95    if cfg!(target_os = "windows") {
96        // Windows backend is a v0.6 stub — not wired here. Once a
97        // real impl lands, probe PowerShell / SendInput here.
98        return Backend::Unsupported;
99    }
100
101    // Linux: try Wayland first (prefer if both are set).
102    if std::env::var("WAYLAND_DISPLAY").is_ok()
103        && has_command("grim")
104        && (has_command("ydotool") || has_command("wtype"))
105    {
106        return Backend::Wayland;
107    }
108
109    // Linux: fall back to X11. The xdpyinfo probe catches the ghost
110    // case — DISPLAY is set but no X server responds (common over
111    // SSH without X forwarding, or after a stale SSH reconnect).
112    if std::env::var("DISPLAY").is_ok()
113        && has_command("scrot")
114        && has_command("xdotool")
115        && xdpyinfo_alive()
116    {
117        return Backend::X11;
118    }
119
120    Backend::Unsupported
121}
122
123/// Quick re-probe used by `ComputerUseDriver::ensure_alive`. Cheaper
124/// than the full `probe()` — just checks the display answers — so
125/// every tool call can afford it.
126pub fn display_is_reachable(backend: Backend) -> bool {
127    match backend {
128        Backend::X11 => xdpyinfo_alive(),
129        Backend::Wayland => std::env::var("WAYLAND_DISPLAY").is_ok(),
130        Backend::MacOS | Backend::Windows => true,
131        Backend::Unsupported => false,
132    }
133}
134
135pub(super) fn has_command(name: &str) -> bool {
136    // `which` returns 0 iff the binary is on PATH. Cheap and universal
137    // across Linux + macOS; Windows would want `where.exe` but
138    // computer-use on Windows is stubbed out anyway.
139    Command::new("which")
140        .arg(name)
141        .output()
142        .map(|o| o.status.success() && !o.stdout.is_empty())
143        .unwrap_or(false)
144}
145
146/// Exit-0 check on `xdpyinfo` with a 200ms timeout. This is the
147/// difference between "`DISPLAY` is set" and "an X server will
148/// actually answer us."
149fn xdpyinfo_alive() -> bool {
150    if !has_command("xdpyinfo") {
151        // Some minimal X setups don't ship xdpyinfo. Fall back to a
152        // `xdotool getactivewindow` probe (we already require
153        // xdotool for clicks anyway).
154        return Command::new("xdotool")
155            .arg("getactivewindow")
156            .output()
157            .map(|o| o.status.success())
158            .unwrap_or(false);
159    }
160    // Use a timeout wrapper so a wedged display doesn't hang startup.
161    match Command::new("timeout").arg("0.2").arg("xdpyinfo").output() {
162        Ok(o) => o.status.success(),
163        Err(_) => {
164            // `timeout` not available (macOS older versions). Fall
165            // back to a direct call — shouldn't happen on Linux X11.
166            Command::new("xdpyinfo")
167                .output()
168                .map(|o| o.status.success())
169                .unwrap_or(false)
170        },
171    }
172}
173
174/// Utility: strip to filename-safe path for temp files.
175#[allow(dead_code)]
176pub(crate) fn path_stem(p: &Path) -> String {
177    p.file_stem()
178        .and_then(|s| s.to_str())
179        .map(|s| s.to_string())
180        .unwrap_or_else(|| "unknown".to_string())
181}
182
183pub(super) fn computer_use_success(
184    action: &'static str,
185    params: Value,
186    output: String,
187    duration_secs: f64,
188) -> ToolOutcome {
189    ToolOutcome::success(output, format!("{} completed", action), duration_secs).with_metadata(
190        ToolRunMetadata {
191            detail: ToolMetadata::ComputerUse {
192                action: action.to_string(),
193                params,
194            },
195            ..ToolRunMetadata::default()
196        },
197    )
198}
199
200/// Shared post-action auto-screenshot for click / type_text / press_key.
201///
202/// When `computer_use.auto_screenshot` is enabled, captures the focused window,
203/// emits an inline `Artifact` preview on the progress channel, and returns
204/// `(summary, base64_png)` for the caller to fold into its outcome. Returns
205/// `None` when the flag is off OR the best-effort capture failed — callers then
206/// build a screenshot-less outcome (#98). Gating lives here so the three tools
207/// share one decision point rather than three byte-identical blocks.
208pub(super) async fn emit_auto_screenshot(
209    driver: &ComputerUseDriver,
210    ctx: &ExecContext,
211    caption: &'static str,
212) -> Option<(String, String)> {
213    if !ctx.config.computer_use.auto_screenshot {
214        return None;
215    }
216    let (summary, base64_png) = driver.capture_focused_for_autoshot(&ctx.token).await?;
217    if let Ok(bytes) =
218        base64::Engine::decode(&base64::engine::general_purpose::STANDARD, &base64_png)
219    {
220        let _ = ctx
221            .progress
222            .send(ProgressEvent::Artifact {
223                mime: "image/png".to_string(),
224                data: bytes,
225                caption: Some(caption.to_string()),
226            })
227            .await;
228    }
229    Some((summary, base64_png))
230}
231
232#[cfg(test)]
233mod tests {
234    use super::*;
235
236    #[test]
237    fn backend_unsupported_is_not_usable() {
238        assert!(!Backend::Unsupported.is_usable());
239        assert!(Backend::X11.is_usable());
240        assert!(Backend::Wayland.is_usable());
241        assert!(Backend::MacOS.is_usable());
242    }
243
244    #[test]
245    fn input_injection_only_on_linux_backends() {
246        assert!(Backend::X11.supports_input_injection());
247        assert!(Backend::Wayland.supports_input_injection());
248        assert!(!Backend::MacOS.supports_input_injection());
249        assert!(!Backend::Windows.supports_input_injection());
250        assert!(!Backend::Unsupported.supports_input_injection());
251    }
252
253    #[test]
254    fn window_listing_only_on_x11() {
255        assert!(Backend::X11.supports_window_listing());
256        assert!(!Backend::Wayland.supports_window_listing());
257        assert!(!Backend::MacOS.supports_window_listing());
258    }
259
260    #[test]
261    fn probe_does_not_panic_on_headless() {
262        // In the test runner (no DISPLAY, no WAYLAND_DISPLAY on most
263        // CI envs), probe() must return Unsupported without panicking.
264        // We don't assert a specific result because dev machines may
265        // have a live display.
266        let _ = probe();
267    }
268
269    #[tokio::test]
270    async fn auto_screenshot_is_noop_when_disabled() {
271        use crate::domain::{ToolCallId, TurnId};
272        let mut cfg = crate::app::Config::default();
273        cfg.computer_use.auto_screenshot = false;
274        let (tx, mut rx) = tokio::sync::mpsc::channel::<ProgressEvent>(8);
275        let ctx = ExecContext::new(
276            tokio_util::sync::CancellationToken::new(),
277            tx,
278            ToolCallId(1),
279            TurnId(1),
280            std::path::PathBuf::from("/tmp"),
281            std::sync::Arc::new(cfg),
282            String::new(),
283            None,
284            None,
285            None,
286            crate::runtime::SafetyMode::FullAccess,
287            None,
288            None,
289            None,
290            None,
291            None,
292        );
293        // Backend is irrelevant — the flag short-circuits before any capture.
294        let driver = ComputerUseDriver::new(Backend::Unsupported);
295        assert!(emit_auto_screenshot(&driver, &ctx, "test").await.is_none());
296        assert!(rx.try_recv().is_err(), "no artifact emitted when disabled");
297    }
298}
mermaid_cli/providers/tool/computer_use/mod.rs

mermaid_cli/providers/tool/computer_use/
mod.rs
