mermaid_cli/providers/tool/computer_use/
click.rs

1//! `click` — mouse click at model-space `(x, y)`. `screenshot_id`
2//! selects which capture the coords refer to (None = latest). After
3//! clicking, auto-captures the focused window so the model can
4//! verify the result inline.
5
6use std::sync::Arc;
7use std::time::Instant;
8
9use async_trait::async_trait;
10use serde_json::Value;
11
12use crate::constants::POST_CLICK_DELAY_MS;
13use crate::domain::{ToolDefinition, ToolOutcome};
14use crate::providers::ctx::ExecContext;
15
16use super::super::ToolExecutor;
17use super::computer_use_success;
18use super::driver::ComputerUseDriver;
19
20pub struct ClickTool {
21    driver: Arc<ComputerUseDriver>,
22}
23
24impl ClickTool {
25    pub fn new(driver: Arc<ComputerUseDriver>) -> Self {
26        Self { driver }
27    }
28}
29
30#[async_trait]
31impl ToolExecutor for ClickTool {
32    fn name(&self) -> &'static str {
33        "click"
34    }
35
36    fn schema(&self) -> ToolDefinition {
37        ToolDefinition {
38            name: "click".to_string(),
39            description:
40                "Click at model-space (x, y). Pass `screenshot_id` to lock coordinates to a \
41                 specific past screenshot; omit for the most recent. Auto-captures the \
42                 focused window afterwards so the result is visible inline."
43                    .to_string(),
44            input_schema: serde_json::json!({
45                "type": "object",
46                "properties": {
47                    "x": { "type": "integer" },
48                    "y": { "type": "integer" },
49                    "button": { "type": "string", "enum": ["left", "middle", "right"], "default": "left" },
50                    "screenshot_id": { "type": "integer" }
51                },
52                "required": ["x", "y"]
53            }),
54        }
55    }
56
57    async fn execute(&self, args: Value, ctx: ExecContext) -> ToolOutcome {
58        let started = Instant::now();
59        if let Err(error) = self.driver.ensure_alive_async().await {
60            return ToolOutcome::error(error, started.elapsed().as_secs_f64());
61        }
62        if let Some(blocked) = super::super::policy_gate::gate_external(
63            &ctx,
64            "click",
65            crate::runtime::ToolCategory::ComputerUse,
66            "computer-use: click".to_string(),
67            &args,
68        )
69        .await
70        {
71            return blocked;
72        }
73
74        let x = args.get("x").and_then(|v| v.as_i64()).map(|n| n as i32);
75        let y = args.get("y").and_then(|v| v.as_i64()).map(|n| n as i32);
76        let (x, y) = match (x, y) {
77            (Some(x), Some(y)) => (x, y),
78            _ => {
79                return ToolOutcome::error(
80                    "click requires integer `x` and `y`",
81                    started.elapsed().as_secs_f64(),
82                );
83            },
84        };
85        let button = args
86            .get("button")
87            .and_then(|v| v.as_str())
88            .unwrap_or("left")
89            .to_string();
90        let screenshot_id = args.get("screenshot_id").and_then(|v| v.as_u64());
91
92        let (sx, sy) = match self.driver.scale_coords(x, y, screenshot_id) {
93            Ok(p) => p,
94            Err(e) => return ToolOutcome::error(e, started.elapsed().as_secs_f64()),
95        };
96
97        let click_res = tokio::select! {
98            biased;
99            _ = ctx.token.cancelled() => return ToolOutcome::cancelled(),
100            r = self.driver.click(sx, sy, &button, &ctx.token) => r,
101        };
102        if let Err(e) = click_res {
103            return ToolOutcome::error(
104                format!("click failed: {}", e),
105                started.elapsed().as_secs_f64(),
106            );
107        }
108
109        // Let the WM process focus change + UI update before the
110        // auto-screenshot captures the result.
111        tokio::time::sleep(std::time::Duration::from_millis(POST_CLICK_DELAY_MS)).await;
112
113        let mut msg = format!(
114            "Clicked {} at ({}, {}) [screen: ({}, {})]",
115            button, x, y, sx, sy
116        );
117        if let Some(warning) = self.driver.check_cursor_landed(sx, sy).await {
118            msg.push('\n');
119            msg.push_str(&warning);
120        }
121
122        let (summary, image) =
123            match super::emit_auto_screenshot(&self.driver, &ctx, "click auto-screenshot").await {
124                Some((s, b64)) => (Some(s), Some(b64)),
125                None => (None, None),
126            };
127
128        let final_output = match &summary {
129            Some(s) => format!("{}\n[auto-screenshot: {}]", msg, s),
130            None => msg,
131        };
132        let mut outcome =
133            computer_use_success("click", args, final_output, started.elapsed().as_secs_f64());
134        if let Some(image) = image {
135            outcome = outcome.with_images(vec![image]);
136        }
137        outcome
138    }
139}
mermaid_cli/providers/tool/computer_use/click.rs

mermaid_cli/providers/tool/computer_use/
click.rs
