cosh_tools/computer/
mouse.rs

1//! Pointer (mouse) component — validation and dispatch of pointer steps.
2//!
3//! This is a COMPONENT, not a tool: it knows how to validate and execute
4//! ONE pointer step (click/double_click/right_click/move/down/up/scroll/
5//! drag) against either desktop coordinates or an accessibility-tree
6//! element, and nothing else. The pipeline tools (`computer_control`)
7//! compose it with the keyboard ([`super::keyboard`]) and wait engines;
8//! any future tool that needs pixel-precise mouse input can reuse it
9//! without carrying the pipelines along.
10//!
11//! Targeting comes in two forms, validated to be mutually exclusive: raw
12//! coordinates (`x`/`y`, position-dependent) or the element form
13//! (`app`/`pid`/`surface` + `selector`), whose point resolves from the
14//! element's CURRENT bounds at dispatch time — never stale.
15use std::time::Duration;
16
17use xa11y::{
18    Anchor, App, AppExt, ClickOptions, ClickTarget, DragOptions, MouseButton, Point, ScrollDelta,
19};
20
21use super::keyboard;
22use super::snapshot::DEFAULT_TIMEOUT_MS;
23use super::types::{ComputerControl, PointerAction, PointerAnchor};
24
25/// Minimum duration of a `drag` movement — below this the interpolation
26/// degenerates to a jump, which drops intermediate hover/enter events.
27pub const MIN_DRAG_MS: u64 = 50;
28
29/// Default duration of a `drag` movement, in milliseconds.
30pub const DEFAULT_DRAG_MS: u64 = 150;
31
32/// Pointer-step validation: one target form (coordinates XOR element, or
33/// neither for `up`), action-specific fields, drag endpoint pairing.
34///
35/// `tool` names the calling pipeline in the error messages, so a future
36/// consumer reports its own name instead of a hardcoded one.
37///
38/// # Errors
39///
40/// Returns `Err` for a partial coordinate pair, both target forms at once,
41/// a targetless action other than `up`, a leaked element scope
42/// (`app`/`pid` without `selector` or vice versa), `name` and `pid`
43/// together, a 0-based `nth`/`count`, a stray `anchor`, a `scroll` without
44/// deltas, a malformed drag (mixed endpoint forms, missing end point), a
45/// `count`/`held`/`duration_ms`/`x2`/`y2`/`to_selector` applied to an
46/// action that does not take it, or a `duration_ms` under [`MIN_DRAG_MS`].
47pub fn validate(step: &ComputerControl, tool: &str) -> Result<(), String> {
48    let action = step.action.unwrap_or_default();
49    // Partial coordinates are a mistake, never a "default the other axis" —
50    // a lone `x` would otherwise dispatch at (x, 0).
51    if step.x.is_some() != step.y.is_some() {
52        return Err(format!("{tool}: coordinates require BOTH `x` and `y`"));
53    }
54    let has_point = step.x.is_some() || step.y.is_some();
55    let has_element = step.selector.is_some()
56        || step.app.is_some()
57        || step.pid.is_some()
58        || step.surface.is_some();
59
60    // Exactly one target form (or neither, which only `up` allows).
61    if has_point && has_element {
62        return Err(format!(
63            "{tool}: use coordinates (`x`/`y`) OR the element form \
64             (`app`/`pid`/`surface` + `selector`), not both"
65        ));
66    }
67    if action != PointerAction::Up && !has_point && !has_element {
68        return Err(format!(
69            "{tool}: action `{}` requires a target — coordinates (`x`/`y`) \
70             or the element form (`app`/`pid`/`surface` + `selector`)",
71            action_name(action)
72        ));
73    }
74    // Element form needs a selector AND exactly one root (app scope or a
75    // shell surface — phase-6 field test: a surface+selector step is the
76    // same element form, not a scope leak).
77    if step.selector.is_some() && step.app.is_none() && step.pid.is_none() && step.surface.is_none()
78    {
79        return Err(format!(
80            "{tool}: the element form requires `app`, `pid` or `surface` together with `selector`"
81        ));
82    }
83    if (step.app.is_some() || step.pid.is_some() || step.surface.is_some())
84        && step.selector.is_none()
85    {
86        return Err(format!(
87            "{tool}: `app`/`pid`/`surface` without `selector` would leak the scope — \
88             pass `selector`, or use coordinates (`x`/`y`)"
89        ));
90    }
91    if step.app.is_some() && (step.pid.is_some() || step.surface.is_some()) {
92        return Err(format!(
93            "{tool}: target ONE of `app`/`pid`/`surface` — they cannot share a step"
94        ));
95    }
96    if step.pid.is_some() && step.surface.is_some() {
97        return Err(format!(
98            "{tool}: target ONE of `app`/`pid`/`surface` — they cannot share a step"
99        ));
100    }
101    if step.nth == Some(0) {
102        return Err(format!(
103            "{tool}: `nth` is 1-based; use 1 for the first match"
104        ));
105    }
106    if step.nth.is_some() && step.selector.is_none() {
107        return Err(format!("{tool}: `nth` requires `selector`"));
108    }
109    if step.anchor.is_some() && step.selector.is_none() {
110        return Err(format!(
111            "{tool}: `anchor` applies to the element form and requires `selector`"
112        ));
113    }
114
115    match action {
116        PointerAction::Scroll if step.dx.is_none() && step.dy.is_none() => {
117            return Err(format!("{tool}: action `scroll` requires `dx` and/or `dy`"));
118        }
119        PointerAction::Drag => validate_drag(step, tool)?,
120        // Click actions accept `count` and `held`.
121        PointerAction::Click | PointerAction::DoubleClick | PointerAction::RightClick => {}
122        // `count` and `held` are click/drag concepts — a `move`/`scroll`/
123        // `down`/`up` carrying them is a mistake the model should hear about.
124        _ => {
125            if step.count.is_some() {
126                return Err(format!(
127                    "{tool}: `count` applies to click actions, not `{}`",
128                    action_name(action)
129                ));
130            }
131            if step.held.is_some() {
132                return Err(format!(
133                    "{tool}: `held` applies to click and drag actions, not `{}`",
134                    action_name(action)
135                ));
136            }
137        }
138    }
139    if step.count == Some(0) {
140        return Err(format!(
141            "{tool}: `count` is 1-based; use 1 for a single click"
142        ));
143    }
144    if let Some(duration) = step.duration_ms {
145        if action != PointerAction::Drag {
146            return Err(format!("{tool}: `duration_ms` applies only to `drag`"));
147        }
148        if duration < MIN_DRAG_MS {
149            return Err(format!(
150                "{tool}: `duration_ms` must be >= {MIN_DRAG_MS} (a shorter \
151                 movement degenerates to a jump and drops hover events)"
152            ));
153        }
154    }
155    // Drag endpoints are drag-only — a click carrying `x2`/`y2`/`to_selector`
156    // is a mistake the model should hear about, not a silently ignored field.
157    if action != PointerAction::Drag
158        && (step.x2.is_some() || step.y2.is_some() || step.to_selector.is_some())
159    {
160        return Err(format!(
161            "{tool}: `x2`/`y2`/`to_selector` are drag endpoints and apply \
162             only to `drag`, not `{}`",
163            action_name(action)
164        ));
165    }
166    Ok(())
167}
168
169/// `drag` endpoint validation: both ends in the SAME form, no mixing.
170fn validate_drag(step: &ComputerControl, tool: &str) -> Result<(), String> {
171    let coord_end = step.x2.is_some() || step.y2.is_some();
172    let start_is_coord = step.x.is_some() || step.y.is_some();
173    let start_is_element = step.selector.is_some();
174
175    if coord_end && (step.x2.is_none() || step.y2.is_none()) {
176        return Err(format!(
177            "{tool}: `drag` by coordinates requires both `x2` and `y2`"
178        ));
179    }
180    if coord_end && step.to_selector.is_some() {
181        return Err(format!(
182            "{tool}: `drag` end is either `x2`/`y2` OR `to_selector`, not both"
183        ));
184    }
185    if start_is_element && coord_end {
186        return Err(format!(
187            "{tool}: `drag` endpoints must share a form — an element start pairs \
188             with `to_selector`, a coordinate start with `x2`/`y2`"
189        ));
190    }
191    if !start_is_element && step.to_selector.is_some() {
192        return Err(format!(
193            "{tool}: `to_selector` requires an element start (`app`/`pid`/`surface` + \
194             `selector`)"
195        ));
196    }
197    if start_is_element && step.to_selector.is_none() {
198        return Err(format!(
199            "{tool}: `drag` from an element requires `to_selector` for the end \
200             point (element-to-element); use `x2`/`y2` with a coordinate start"
201        ));
202    }
203    if start_is_coord && !coord_end {
204        return Err(format!(
205            "{tool}: `drag` requires an end point — `x2`/`y2` or `to_selector`"
206        ));
207    }
208    Ok(())
209}
210
211/// Execute ONE validated pointer step. The element form resolves the point
212/// from the element's CURRENT bounds at dispatch time (never stale); the
213/// coordinate form passes through. Returns the step's report fragment.
214pub fn run_step(
215    sim: &xa11y::InputSim,
216    step: &ComputerControl,
217    tool: &str,
218) -> Result<String, String> {
219    let action = step.action.unwrap_or_default();
220    let button = parse_button(step.button.as_deref(), tool)?;
221    let held = keyboard::parse_keys(step.held.as_deref().unwrap_or_default(), tool)?;
222    let count = step.count.unwrap_or(1);
223
224    // Resolve the effective point for actions that need one. `up` and the
225    // button-state half of `down` run at the current cursor position.
226    let point: Option<Point> = if let Some(selector) = &step.selector {
227        // resolve_element already renders through the error component with
228        // the failing selector in context — no extra wrapper here.
229        let element = resolve_element(step, selector, step.nth.unwrap_or(1), tool)?;
230        Some(
231            xa11y::point_for(&element, anchor(step.anchor.unwrap_or_default())).map_err(|e| {
232                super::errors::render(tool, &format!("resolve point for `{selector}`"), &e)
233            })?,
234        )
235    } else {
236        match (step.x, step.y) {
237            (Some(x), Some(y)) => Some(Point::new(x, y)),
238            _ => None,
239        }
240    };
241
242    // The drag's end point is resolved ONCE, before dispatch, and reused
243    // for the report below — re-resolving `to_selector` after the drag ran
244    // would let a UI change (e.g. the drop reordering the tree) turn a
245    // successful drag into a spurious failure.
246    let mut drag_to: Option<Point> = None;
247    let result = match action {
248        PointerAction::Click => sim.mouse().click_with(
249            point_target(point),
250            ClickOptions::new().button(button).count(count).held(held),
251        ),
252        // `count` is honored on EVERY click action: double_click defaults to
253        // 2, right_click defaults to 1 — passing an explicit `count` always
254        // wins, so `right_click` + count 2 is a double right-click.
255        PointerAction::DoubleClick => sim.mouse().click_with(
256            point_target(point),
257            ClickOptions::new()
258                .button(button)
259                .count(step.count.unwrap_or(2))
260                .held(held),
261        ),
262        PointerAction::RightClick => sim.mouse().click_with(
263            point_target(point),
264            ClickOptions::new()
265                .button(MouseButton::Right)
266                .count(count)
267                .held(held),
268        ),
269        PointerAction::Move => sim
270            .mouse()
271            .move_to(point.expect("validated: move requires a target")),
272        // `down` presses at the CURRENT cursor position, so an explicit
273        // target must be moved to FIRST — otherwise a drag would start at
274        // wherever the cursor happens to be while the output still claims
275        // the requested coordinates.
276        PointerAction::Down => {
277            if let Some(p) = point {
278                sim.mouse()
279                    .move_to(p)
280                    .map_err(|e| super::errors::render(tool, "move pointer", &e))?;
281            }
282            sim.mouse().down(button)
283        }
284        PointerAction::Up => sim.mouse().up(button),
285        PointerAction::Scroll => sim.mouse().scroll(
286            point.expect("validated: scroll requires a target"),
287            ScrollDelta {
288                dx: step.dx.unwrap_or(0),
289                dy: step.dy.unwrap_or(0),
290            },
291        ),
292        PointerAction::Drag => {
293            let from = point.expect("validated: drag requires a start target");
294            let to = drag_end_point(step, tool)?;
295            drag_to = Some(to);
296            sim.mouse().drag_with(
297                from,
298                to,
299                DragOptions::new()
300                    .button(button)
301                    .held(held)
302                    .duration(Duration::from_millis(
303                        step.duration_ms.unwrap_or(DEFAULT_DRAG_MS),
304                    )),
305            )
306        }
307    };
308    result.map_err(|e| super::errors::render(tool, &action_name(action).to_lowercase(), &e))?;
309
310    // Human-facing report with the EFFECTIVE point — for the element form
311    // this is what the bounds resolved to at dispatch time, which makes a
312    // mis-aimed click debuggable.
313    Ok(match action {
314        PointerAction::Click | PointerAction::DoubleClick | PointerAction::RightClick => {
315            let verb = match action {
316                PointerAction::Click => "clicked",
317                PointerAction::DoubleClick => "double-clicked",
318                _ => "right-clicked",
319            };
320            let extra = match action {
321                PointerAction::Click if count > 1 => format!(" x{count}"),
322                _ => String::new(),
323            };
324            format!("{verb} {}{extra}", where_at(step, point))
325        }
326        PointerAction::Move => format!("moved to {}", where_at(step, point)),
327        PointerAction::Down => format!("button down {}", where_at(step, point)),
328        PointerAction::Up => format!(
329            "released {}",
330            step.button
331                .as_deref()
332                .map(str::trim)
333                .filter(|s| !s.is_empty())
334                .unwrap_or("left")
335        ),
336        PointerAction::Scroll => format!(
337            "scrolled ({},{}) {}",
338            step.dx.unwrap_or(0),
339            step.dy.unwrap_or(0),
340            where_at(step, point)
341        ),
342        PointerAction::Drag => {
343            // Reuse the end point resolved BEFORE dispatch (finding from
344            // review): re-resolving `to_selector` here could fail after the
345            // drag already ran and report a success as an error.
346            let to = drag_to.expect("drag arm resolved its end point before dispatch");
347            format!("dragged {} → ({},{})", where_at(step, point), to.x, to.y)
348        }
349    })
350}
351
352/// Human-facing location of a pointer step's effect: the selector with its
353/// resolved point, the bare point, or the cursor for targetless steps.
354fn where_at(step: &ComputerControl, point: Option<Point>) -> String {
355    match (step.selector.as_deref(), point) {
356        (Some(sel), Some(p)) => format!("`{sel}` @{},{}", p.x, p.y),
357        (Some(sel), None) => format!("`{sel}`"),
358        (None, Some(p)) => format!("@{},{}", p.x, p.y),
359        (None, None) => "at the current position".into(),
360    }
361}
362
363/// Resolve ONE element under the step's root — a shell surface when
364/// `surface` is set, the application otherwise (validated: exactly one of
365/// app/pid/surface). Shared by the click/move/scroll point resolution and
366/// the drag's `to_selector` end, so a future root addition lands in one
367/// place.
368fn resolve_element(
369    step: &ComputerControl,
370    selector: &str,
371    nth: usize,
372    tool: &str,
373) -> Result<xa11y::Element, String> {
374    let timeout = Duration::from_millis(DEFAULT_TIMEOUT_MS);
375    if let Some(surface_kind) = step.surface {
376        super::surface::resolve(surface_kind, timeout)
377            .map_err(|e| format!("{tool}: {e}"))?
378            .locator(selector)
379            .nth(nth)
380            .element()
381            .map_err(|e| super::errors::render(tool, "resolve element", &e))
382    } else {
383        resolve_app(step, tool)?
384            .locator(selector)
385            .nth(nth)
386            .element()
387            .map_err(|e| super::errors::render(tool, "resolve element", &e))
388    }
389}
390
391/// The drag's end point in the SAME form as the start (validated):
392/// element → element resolves `to_selector` on the same root at dispatch
393/// time; coordinates → coordinates pass through.
394fn drag_end_point(step: &ComputerControl, tool: &str) -> Result<Point, String> {
395    if let Some(to_selector) = &step.to_selector {
396        // resolve_element already renders through the error component —
397        // no extra wrapper here.
398        let element = resolve_element(step, to_selector, 1, tool)?;
399        xa11y::point_for(&element, anchor(step.anchor.unwrap_or_default()))
400            .map_err(|e| super::errors::render(tool, "resolve drag end point", &e))
401    } else {
402        Ok(Point::new(
403            step.x2.expect("validated: drag end coordinates"),
404            step.y2.expect("validated: drag end coordinates"),
405        ))
406    }
407}
408
409/// Wrap the resolved point into a [`ClickTarget`] (both forms land as a
410/// point by the time the click runs — the element's bounds were just read).
411fn point_target(point: Option<Point>) -> ClickTarget<'static> {
412    ClickTarget::Point(point.unwrap_or(Point::new(0, 0)))
413}
414
415/// Resolve the target application (validating `app`/`pid` exclusivity).
416fn resolve_app(step: &ComputerControl, tool: &str) -> Result<App, String> {
417    let timeout = Duration::from_millis(DEFAULT_TIMEOUT_MS);
418    let name = step.app.as_deref().map(str::trim).filter(|s| !s.is_empty());
419    match (name, step.pid) {
420        (Some(name), None) => {
421            App::by_name(name, timeout).map_err(|e| super::errors::render_app_miss(tool, &e))
422        }
423        (None, Some(pid)) => {
424            App::by_pid(pid, timeout).map_err(|e| super::errors::render_app_miss(tool, &e))
425        }
426        (None, None) => Err(format!("{tool}: the element form requires `app` or `pid`")),
427        (Some(_), Some(_)) => Err(format!("{tool}: provide `app` or `pid`, not both")),
428    }
429}
430
431/// Parse a button name (`left`/`right`/`middle`).
432pub fn parse_button(name: Option<&str>, tool: &str) -> Result<MouseButton, String> {
433    match name.map(str::trim).map(str::to_ascii_lowercase).as_deref() {
434        None | Some("left") | Some("") => Ok(MouseButton::Left),
435        Some("right") => Ok(MouseButton::Right),
436        Some("middle") => Ok(MouseButton::Middle),
437        Some(other) => Err(format!(
438            "{tool}: unknown button `{other}` (expected left, right or middle)"
439        )),
440    }
441}
442
443/// Map the wire anchor to xa11y's [`Anchor`].
444pub fn anchor(value: PointerAnchor) -> Anchor {
445    match value {
446        PointerAnchor::Center => Anchor::Center,
447        PointerAnchor::TopLeft => Anchor::TopLeft,
448        PointerAnchor::TopRight => Anchor::TopRight,
449        PointerAnchor::BottomLeft => Anchor::BottomLeft,
450        PointerAnchor::BottomRight => Anchor::BottomRight,
451    }
452}
453
454/// Wire-facing name of a pointer action.
455pub fn action_name(action: PointerAction) -> &'static str {
456    match action {
457        PointerAction::Click => "click",
458        PointerAction::DoubleClick => "double_click",
459        PointerAction::RightClick => "right_click",
460        PointerAction::Move => "move",
461        PointerAction::Down => "down",
462        PointerAction::Up => "up",
463        PointerAction::Scroll => "scroll",
464        PointerAction::Drag => "drag",
465    }
466}
cosh_tools/computer/mouse.rs

cosh_tools/computer/
mouse.rs
