mermaid_cli/providers/tool/computer_use/
click.rs

1//! `click` — mouse click at model-space `(x, y)`. `screenshot_id`
2//! selects which capture the coords refer to (None = latest). After
3//! clicking, auto-captures the focused window so the model can
4//! verify the result inline.
5
6use std::sync::Arc;
7use std::time::Instant;
8
9use async_trait::async_trait;
10use serde_json::Value;
11
12use crate::constants::POST_CLICK_DELAY_MS;
13use crate::domain::{ToolDefinition, ToolOutcome};
14use crate::providers::ctx::{ExecContext, ProgressEvent};
15
16use super::super::ToolExecutor;
17use super::computer_use_success;
18use super::driver::ComputerUseDriver;
19
20pub struct ClickTool {
21    driver: Arc<ComputerUseDriver>,
22}
23
24impl ClickTool {
25    pub fn new(driver: Arc<ComputerUseDriver>) -> Self {
26        Self { driver }
27    }
28}
29
30#[async_trait]
31impl ToolExecutor for ClickTool {
32    fn name(&self) -> &'static str {
33        "click"
34    }
35
36    fn schema(&self) -> ToolDefinition {
37        ToolDefinition {
38            name: "click".to_string(),
39            description:
40                "Click at model-space (x, y). Pass `screenshot_id` to lock coordinates to a \
41                 specific past screenshot; omit for the most recent. Auto-captures the \
42                 focused window afterwards so the result is visible inline."
43                    .to_string(),
44            input_schema: serde_json::json!({
45                "type": "object",
46                "properties": {
47                    "x": { "type": "integer" },
48                    "y": { "type": "integer" },
49                    "button": { "type": "string", "enum": ["left", "middle", "right"], "default": "left" },
50                    "screenshot_id": { "type": "integer" }
51                },
52                "required": ["x", "y"]
53            }),
54        }
55    }
56
57    async fn execute(&self, args: Value, ctx: ExecContext) -> ToolOutcome {
58        let started = Instant::now();
59        if let Err(error) = self.driver.ensure_alive() {
60            return ToolOutcome::error(error, started.elapsed().as_secs_f64());
61        }
62        if let Some(blocked) = super::super::policy_gate::gate_external(
63            &ctx,
64            "click",
65            crate::runtime::ToolCategory::ComputerUse,
66            "computer-use: click".to_string(),
67            &args,
68        )
69        .await
70        {
71            return blocked;
72        }
73
74        let x = args.get("x").and_then(|v| v.as_i64()).map(|n| n as i32);
75        let y = args.get("y").and_then(|v| v.as_i64()).map(|n| n as i32);
76        let (x, y) = match (x, y) {
77            (Some(x), Some(y)) => (x, y),
78            _ => {
79                return ToolOutcome::error(
80                    "click requires integer `x` and `y`",
81                    started.elapsed().as_secs_f64(),
82                );
83            },
84        };
85        let button = args
86            .get("button")
87            .and_then(|v| v.as_str())
88            .unwrap_or("left")
89            .to_string();
90        let screenshot_id = args.get("screenshot_id").and_then(|v| v.as_u64());
91
92        let (sx, sy) = match self.driver.scale_coords(x, y, screenshot_id) {
93            Ok(p) => p,
94            Err(e) => return ToolOutcome::error(e, started.elapsed().as_secs_f64()),
95        };
96
97        let click_res = tokio::select! {
98            biased;
99            _ = ctx.token.cancelled() => return ToolOutcome::cancelled(),
100            r = self.driver.click(sx, sy, &button, &ctx.token) => r,
101        };
102        if let Err(e) = click_res {
103            return ToolOutcome::error(
104                format!("click failed: {}", e),
105                started.elapsed().as_secs_f64(),
106            );
107        }
108
109        // Let the WM process focus change + UI update before the
110        // auto-screenshot captures the result.
111        tokio::time::sleep(std::time::Duration::from_millis(POST_CLICK_DELAY_MS)).await;
112
113        let mut msg = format!(
114            "Clicked {} at ({}, {}) [screen: ({}, {})]",
115            button, x, y, sx, sy
116        );
117        if let Some(warning) = self.driver.check_cursor_landed(sx, sy).await {
118            msg.push('\n');
119            msg.push_str(&warning);
120        }
121
122        let (summary, image) = match self.driver.capture_focused_for_autoshot(&ctx.token).await {
123            Some((s, b64)) => (Some(s), Some(b64)),
124            None => (None, None),
125        };
126
127        if let Some(b64) = &image
128            && let Ok(bytes) =
129                base64::Engine::decode(&base64::engine::general_purpose::STANDARD, b64)
130        {
131            let _ = ctx
132                .progress
133                .send(ProgressEvent::Artifact {
134                    mime: "image/png".to_string(),
135                    data: bytes,
136                    caption: Some("click auto-screenshot".to_string()),
137                })
138                .await;
139        }
140
141        let final_output = match &summary {
142            Some(s) => format!("{}\n[auto-screenshot: {}]", msg, s),
143            None => msg,
144        };
145        let mut outcome =
146            computer_use_success("click", args, final_output, started.elapsed().as_secs_f64());
147        if let Some(image) = image {
148            outcome = outcome.with_images(vec![image]);
149        }
150        outcome
151    }
152}
mermaid_cli/providers/tool/computer_use/click.rs

mermaid_cli/providers/tool/computer_use/
click.rs
