mermaid_cli/providers/tool/computer_use/
click.rs

1//! `click` — mouse click at model-space `(x, y)`. `screenshot_id`
2//! selects which capture the coords refer to (None = latest). After
3//! clicking, auto-captures the focused window so the model can
4//! verify the result inline.
5
6use std::sync::Arc;
7use std::time::Instant;
8
9use async_trait::async_trait;
10use serde_json::Value;
11
12use crate::constants::POST_CLICK_DELAY_MS;
13use crate::domain::{ToolDefinition, ToolOutcome};
14use crate::providers::ctx::{ExecContext, ProgressEvent};
15
16use super::super::ToolExecutor;
17use super::computer_use_success;
18use super::driver::ComputerUseDriver;
19
20pub struct ClickTool {
21    driver: Arc<ComputerUseDriver>,
22}
23
24impl ClickTool {
25    pub fn new(driver: Arc<ComputerUseDriver>) -> Self {
26        Self { driver }
27    }
28}
29
30#[async_trait]
31impl ToolExecutor for ClickTool {
32    fn name(&self) -> &'static str {
33        "click"
34    }
35
36    fn schema(&self) -> ToolDefinition {
37        ToolDefinition {
38            name: "click".to_string(),
39            description:
40                "Click at model-space (x, y). Pass `screenshot_id` to lock coordinates to a \
41                 specific past screenshot; omit for the most recent. Auto-captures the \
42                 focused window afterwards so the result is visible inline."
43                    .to_string(),
44            input_schema: serde_json::json!({
45                "type": "object",
46                "properties": {
47                    "x": { "type": "integer" },
48                    "y": { "type": "integer" },
49                    "button": { "type": "string", "enum": ["left", "middle", "right"], "default": "left" },
50                    "screenshot_id": { "type": "integer" }
51                },
52                "required": ["x", "y"]
53            }),
54        }
55    }
56
57    async fn execute(&self, args: Value, ctx: ExecContext) -> ToolOutcome {
58        let started = Instant::now();
59        if let Err(error) = self.driver.ensure_alive() {
60            return ToolOutcome::error(error, started.elapsed().as_secs_f64());
61        }
62        if let Some(blocked) = super::super::policy_gate::gate_external(
63            &ctx,
64            "click",
65            crate::runtime::ToolCategory::ComputerUse,
66            "computer-use: click".to_string(),
67            &args,
68        ) {
69            return blocked;
70        }
71
72        let x = args.get("x").and_then(|v| v.as_i64()).map(|n| n as i32);
73        let y = args.get("y").and_then(|v| v.as_i64()).map(|n| n as i32);
74        let (x, y) = match (x, y) {
75            (Some(x), Some(y)) => (x, y),
76            _ => {
77                return ToolOutcome::error(
78                    "click requires integer `x` and `y`",
79                    started.elapsed().as_secs_f64(),
80                );
81            },
82        };
83        let button = args
84            .get("button")
85            .and_then(|v| v.as_str())
86            .unwrap_or("left")
87            .to_string();
88        let screenshot_id = args.get("screenshot_id").and_then(|v| v.as_u64());
89
90        let (sx, sy) = match self.driver.scale_coords(x, y, screenshot_id) {
91            Ok(p) => p,
92            Err(e) => return ToolOutcome::error(e, started.elapsed().as_secs_f64()),
93        };
94
95        let click_res = tokio::select! {
96            biased;
97            _ = ctx.token.cancelled() => return ToolOutcome::cancelled(),
98            r = self.driver.click(sx, sy, &button, &ctx.token) => r,
99        };
100        if let Err(e) = click_res {
101            return ToolOutcome::error(
102                format!("click failed: {}", e),
103                started.elapsed().as_secs_f64(),
104            );
105        }
106
107        // Let the WM process focus change + UI update before the
108        // auto-screenshot captures the result.
109        tokio::time::sleep(std::time::Duration::from_millis(POST_CLICK_DELAY_MS)).await;
110
111        let mut msg = format!(
112            "Clicked {} at ({}, {}) [screen: ({}, {})]",
113            button, x, y, sx, sy
114        );
115        if let Some(warning) = self.driver.check_cursor_landed(sx, sy).await {
116            msg.push('\n');
117            msg.push_str(&warning);
118        }
119
120        let (summary, image) = match self.driver.capture_focused_for_autoshot(&ctx.token).await {
121            Some((s, b64)) => (Some(s), Some(b64)),
122            None => (None, None),
123        };
124
125        if let Some(b64) = &image
126            && let Ok(bytes) =
127                base64::Engine::decode(&base64::engine::general_purpose::STANDARD, b64)
128        {
129            let _ = ctx
130                .progress
131                .send(ProgressEvent::Artifact {
132                    mime: "image/png".to_string(),
133                    data: bytes,
134                    caption: Some("click auto-screenshot".to_string()),
135                })
136                .await;
137        }
138
139        let final_output = match &summary {
140            Some(s) => format!("{}\n[auto-screenshot: {}]", msg, s),
141            None => msg,
142        };
143        let mut outcome =
144            computer_use_success("click", args, final_output, started.elapsed().as_secs_f64());
145        if let Some(image) = image {
146            outcome = outcome.with_images(vec![image]);
147        }
148        outcome
149    }
150}
mermaid_cli/providers/tool/computer_use/click.rs

mermaid_cli/providers/tool/computer_use/
click.rs
