computer_use/backend/
input.rs

1//! Pointer and keyboard input through enigo.
2//!
3//! Adapted from the Bezalel macOS desktop backend. The same enigo calls run
4//! on macOS, Windows, and Linux (X11 via libxdo).
5
6use enigo::{Axis, Button, Coordinate, Direction, Keyboard, Mouse};
7
8use crate::error::ComputerError;
9use crate::types::{Modifier, MouseButton};
10
11pub(crate) fn move_mouse(x: i32, y: i32) -> Result<(), ComputerError> {
12    let mut session = session()?;
13    session
14        .move_mouse(x, y, Coordinate::Abs)
15        .map_err(|error| ComputerError::ExecutionFailed(format!("mouse move failed: {error}")))
16}
17
18pub(crate) fn click(x: i32, y: i32, button: MouseButton) -> Result<(), ComputerError> {
19    let mapped = map_button(button)?;
20    let mut session = session()?;
21    session
22        .move_mouse(x, y, Coordinate::Abs)
23        .map_err(|error| ComputerError::ExecutionFailed(format!("mouse move failed: {error}")))?;
24    session
25        .button(mapped, Direction::Click)
26        .map_err(|error| ComputerError::ExecutionFailed(format!("click failed: {error}")))
27}
28
29/// Press, move through every point, then release. A failed move releases the button.
30#[cfg_attr(test, allow(dead_code))]
31pub(crate) fn drag(points: &[(i32, i32)], button: MouseButton) -> Result<(), ComputerError> {
32    if points.len() < 2 {
33        return Err(ComputerError::InvalidArguments(
34            "drag needs at least 2 points".into(),
35        ));
36    }
37    let mapped = map_button(button)?;
38    let mut session = session()?;
39    session
40        .move_mouse(points[0].0, points[0].1, Coordinate::Abs)
41        .map_err(|error| ComputerError::ExecutionFailed(format!("mouse move failed: {error}")))?;
42    session
43        .button(mapped, Direction::Press)
44        .map_err(|error| ComputerError::ExecutionFailed(format!("button press failed: {error}")))?;
45    let mut released = false;
46    let moved = (|| {
47        for point in &points[1..] {
48            session
49                .move_mouse(point.0, point.1, Coordinate::Abs)
50                .map_err(|error| {
51                    ComputerError::ExecutionFailed(format!("mouse move failed: {error}"))
52                })?;
53        }
54        session
55            .button(mapped, Direction::Release)
56            .map_err(|error| {
57                ComputerError::ExecutionFailed(format!("button release failed: {error}"))
58            })?;
59        released = true;
60        Ok(())
61    })();
62    if !released {
63        let _ = session.button(mapped, Direction::Release);
64    }
65    moved
66}
67
68/// Button down. The process registry releases it if no later up runs.
69#[cfg_attr(test, allow(dead_code))]
70pub(crate) fn button_down(x: i32, y: i32, button: MouseButton) -> Result<(), ComputerError> {
71    let mapped = map_button(button)?;
72    let mut session = session()?;
73    session
74        .move_mouse(x, y, Coordinate::Abs)
75        .map_err(|error| ComputerError::ExecutionFailed(format!("mouse move failed: {error}")))?;
76    session
77        .button(mapped, Direction::Press)
78        .map_err(|error| ComputerError::ExecutionFailed(format!("button press failed: {error}")))?;
79    super::held::note_button(super::held::HeldButton {
80        button,
81        x,
82        y,
83        where_: super::held::HeldWhere::Activate,
84    });
85    Ok(())
86}
87
88/// Button up. Clears the registry after a successful release.
89#[cfg_attr(test, allow(dead_code))]
90pub(crate) fn button_up(x: i32, y: i32, button: MouseButton) -> Result<(), ComputerError> {
91    let mapped = map_button(button)?;
92    let mut session = session()?;
93    session
94        .move_mouse(x, y, Coordinate::Abs)
95        .map_err(|error| ComputerError::ExecutionFailed(format!("mouse move failed: {error}")))?;
96    session
97        .button(mapped, Direction::Release)
98        .map_err(|error| {
99            ComputerError::ExecutionFailed(format!("button release failed: {error}"))
100        })?;
101    super::held::clear_button(button);
102    Ok(())
103}
104
105pub(crate) fn type_text(text: &str) -> Result<(), ComputerError> {
106    if text.is_empty() {
107        return Err(ComputerError::InvalidArguments(
108            "text must not be empty".into(),
109        ));
110    }
111    if text.contains('\0') {
112        return Err(ComputerError::InvalidArguments(
113            "text must not contain a null byte".into(),
114        ));
115    }
116    let mut session = session()?;
117    session
118        .text(text)
119        .map_err(|error| ComputerError::ExecutionFailed(format!("type failed: {error}")))
120}
121
122pub(crate) fn press_key(modifiers: &[Modifier], key: &str) -> Result<(), ComputerError> {
123    let enigo_key = map_key(key)?;
124    let mut session = session()?;
125    let mapped: Vec<enigo::Key> = modifiers.iter().copied().map(map_modifier).collect();
126    for modifier in &mapped {
127        session.key(*modifier, Direction::Press).map_err(|error| {
128            ComputerError::ExecutionFailed(format!("modifier press failed: {error}"))
129        })?;
130    }
131    let typed = session.key(enigo_key, Direction::Click);
132    for modifier in mapped.iter().rev() {
133        let _ = session.key(*modifier, Direction::Release);
134    }
135    typed.map_err(|error| ComputerError::ExecutionFailed(format!("key press failed: {error}")))
136}
137
138/// Key down, sleep `ms`, key up. The registry keeps the key if the up fails.
139#[cfg_attr(test, allow(dead_code))]
140pub(crate) fn hold_key(modifiers: &[Modifier], key: &str, ms: u64) -> Result<(), ComputerError> {
141    if ms > crate::tier::MAX_HOLD_MS {
142        return Err(ComputerError::InvalidArguments(format!(
143            "hold_key ms must be at most {}",
144            crate::tier::MAX_HOLD_MS
145        )));
146    }
147    key_down(modifiers, key)?;
148    if ms > 0 {
149        std::thread::sleep(std::time::Duration::from_millis(ms));
150    }
151    key_up(modifiers, key)
152}
153
154#[cfg_attr(test, allow(dead_code))]
155pub(crate) fn key_down(modifiers: &[Modifier], key: &str) -> Result<(), ComputerError> {
156    let enigo_key = map_key(key)?;
157    let mut session = session()?;
158    let mapped: Vec<enigo::Key> = modifiers.iter().copied().map(map_modifier).collect();
159    for modifier in &mapped {
160        session.key(*modifier, Direction::Press).map_err(|error| {
161            ComputerError::ExecutionFailed(format!("modifier press failed: {error}"))
162        })?;
163    }
164    session
165        .key(enigo_key, Direction::Press)
166        .map_err(|error| ComputerError::ExecutionFailed(format!("key press failed: {error}")))?;
167    super::held::note_key(super::held::HeldKey {
168        modifiers: modifiers.to_vec(),
169        key: key.to_string(),
170        where_: super::held::HeldWhere::Activate,
171    });
172    Ok(())
173}
174
175#[cfg_attr(test, allow(dead_code))]
176pub(crate) fn key_up(modifiers: &[Modifier], key: &str) -> Result<(), ComputerError> {
177    let enigo_key = map_key(key)?;
178    let mut session = session()?;
179    let released = session.key(enigo_key, Direction::Release);
180    for modifier in modifiers.iter().rev() {
181        let _ = session.key(map_modifier(*modifier), Direction::Release);
182    }
183    released
184        .map_err(|error| ComputerError::ExecutionFailed(format!("key release failed: {error}")))?;
185    super::held::clear_keys();
186    Ok(())
187}
188
189pub(crate) fn scroll(dx: i32, dy: i32) -> Result<(), ComputerError> {
190    if dx == 0 && dy == 0 {
191        return Ok(());
192    }
193    let mut session = session()?;
194    if dy != 0 {
195        session.scroll(dy, Axis::Vertical).map_err(|error| {
196            ComputerError::ExecutionFailed(format!("vertical scroll failed: {error}"))
197        })?;
198    }
199    if dx != 0 {
200        session.scroll(dx, Axis::Horizontal).map_err(|error| {
201            ComputerError::ExecutionFailed(format!("horizontal scroll failed: {error}"))
202        })?;
203    }
204    Ok(())
205}
206
207pub(crate) fn cursor_position() -> Result<(i32, i32), ComputerError> {
208    let session = session()?;
209    session
210        .location()
211        .map_err(|error| ComputerError::ExecutionFailed(format!("cursor position failed: {error}")))
212}
213
214/// Return an error when `key` is not a known name or a single character.
215pub fn validate_key(key: &str) -> Result<(), ComputerError> {
216    crate::keys::macos_function_gap(key)?;
217    map_key(key).map(|_| ())
218}
219
220fn session() -> Result<enigo::Enigo, ComputerError> {
221    // Agents should fail with an error instead of blocking on a permission dialog.
222    let settings = enigo::Settings {
223        open_prompt_to_get_permissions: false,
224        release_keys_when_dropped: true,
225        ..enigo::Settings::default()
226    };
227    enigo::Enigo::new(&settings).map_err(|error| {
228        ComputerError::ExecutionFailed(format!("failed to initialize input: {error}"))
229    })
230}
231
232fn map_button(button: MouseButton) -> Result<Button, ComputerError> {
233    match button {
234        MouseButton::Left => Ok(Button::Left),
235        MouseButton::Right => Ok(Button::Right),
236        MouseButton::Middle => Ok(Button::Middle),
237        #[cfg(any(target_os = "windows", all(unix, not(target_os = "macos"))))]
238        MouseButton::Back => Ok(Button::Back),
239        #[cfg(any(target_os = "windows", all(unix, not(target_os = "macos"))))]
240        MouseButton::Forward => Ok(Button::Forward),
241        #[cfg(target_os = "macos")]
242        MouseButton::Back | MouseButton::Forward => Err(ComputerError::Unsupported(
243            "enigo on macOS has no back or forward button; use tier event".into(),
244        )),
245    }
246}
247
248fn map_modifier(modifier: Modifier) -> enigo::Key {
249    match modifier {
250        Modifier::Command | Modifier::Meta => enigo::Key::Meta,
251        Modifier::Control => enigo::Key::Control,
252        Modifier::Shift => enigo::Key::Shift,
253        Modifier::Alt => enigo::Key::Alt,
254    }
255}
256
257fn map_key(key: &str) -> Result<enigo::Key, ComputerError> {
258    if key.chars().count() != 1 {
259        return map_named(key).or_else(|_| map_named(&key.to_ascii_lowercase()));
260    }
261    // A single character is typed as itself, including digits and letters.
262    // Named keys (enter, f1, left, ...) are more than one character.
263    match key.chars().next() {
264        Some(ch) => Ok(enigo::Key::Unicode(ch)),
265        None => Err(ComputerError::InvalidArguments("unknown key \"\"".into())),
266    }
267}
268
269fn map_named(key: &str) -> Result<enigo::Key, ComputerError> {
270    let mapped = match key {
271        "space" => enigo::Key::Space,
272        "enter" | "return" => enigo::Key::Return,
273        "tab" => enigo::Key::Tab,
274        "esc" | "escape" => enigo::Key::Escape,
275        "backspace" => enigo::Key::Backspace,
276        "delete" => enigo::Key::Delete,
277        "left" => enigo::Key::LeftArrow,
278        "right" => enigo::Key::RightArrow,
279        "up" => enigo::Key::UpArrow,
280        "down" => enigo::Key::DownArrow,
281        "home" => enigo::Key::Home,
282        "end" => enigo::Key::End,
283        "pageup" => enigo::Key::PageUp,
284        "pagedown" => enigo::Key::PageDown,
285        // enigo's macOS `Key` enum names this Help. It is the Insert key.
286        #[cfg(target_os = "macos")]
287        "insert" => enigo::Key::Help,
288        #[cfg(not(target_os = "macos"))]
289        "insert" => enigo::Key::Insert,
290        "plus" => enigo::Key::Unicode('+'),
291        "minus" => enigo::Key::Unicode('-'),
292        "f1" => enigo::Key::F1,
293        "f2" => enigo::Key::F2,
294        "f3" => enigo::Key::F3,
295        "f4" => enigo::Key::F4,
296        "f5" => enigo::Key::F5,
297        "f6" => enigo::Key::F6,
298        "f7" => enigo::Key::F7,
299        "f8" => enigo::Key::F8,
300        "f9" => enigo::Key::F9,
301        "f10" => enigo::Key::F10,
302        "f11" => enigo::Key::F11,
303        "f12" => enigo::Key::F12,
304        "f13" => enigo::Key::F13,
305        "f14" => enigo::Key::F14,
306        "f15" => enigo::Key::F15,
307        "f16" => enigo::Key::F16,
308        "f17" => enigo::Key::F17,
309        "f18" => enigo::Key::F18,
310        "f19" => enigo::Key::F19,
311        "f20" => enigo::Key::F20,
312        // enigo omits F21–F24 on macOS. The event tier has no virtual keycode either.
313        #[cfg(not(target_os = "macos"))]
314        "f21" => enigo::Key::F21,
315        #[cfg(not(target_os = "macos"))]
316        "f22" => enigo::Key::F22,
317        #[cfg(not(target_os = "macos"))]
318        "f23" => enigo::Key::F23,
319        #[cfg(not(target_os = "macos"))]
320        "f24" => enigo::Key::F24,
321        #[cfg(target_os = "macos")]
322        "f21" | "f22" | "f23" | "f24" => {
323            return Err(ComputerError::Unsupported(format!(
324                "{key} has no macOS virtual keycode; F13 through F20 are available"
325            )));
326        }
327        "shift" => enigo::Key::Shift,
328        "control" | "ctrl" => enigo::Key::Control,
329        "alt" | "option" => enigo::Key::Alt,
330        "meta" | "cmd" | "command" | "super" => enigo::Key::Meta,
331        _ => {
332            return Err(ComputerError::InvalidArguments(format!(
333                "unknown key {key:?}"
334            )))
335        }
336    };
337    Ok(mapped)
338}
339
340#[cfg(test)]
341mod tests {
342    use super::*;
343
344    #[test]
345    fn maps_named_function_and_arrow_keys() {
346        assert!(matches!(map_key("space").unwrap(), enigo::Key::Space));
347        assert!(matches!(map_key("enter").unwrap(), enigo::Key::Return));
348        assert!(matches!(map_key("tab").unwrap(), enigo::Key::Tab));
349        assert!(matches!(map_key("esc").unwrap(), enigo::Key::Escape));
350        assert!(matches!(
351            map_key("backspace").unwrap(),
352            enigo::Key::Backspace
353        ));
354        assert!(matches!(map_key("delete").unwrap(), enigo::Key::Delete));
355        assert!(matches!(map_key("left").unwrap(), enigo::Key::LeftArrow));
356        assert!(matches!(map_key("right").unwrap(), enigo::Key::RightArrow));
357        assert!(matches!(map_key("up").unwrap(), enigo::Key::UpArrow));
358        assert!(matches!(map_key("down").unwrap(), enigo::Key::DownArrow));
359        assert!(matches!(map_key("home").unwrap(), enigo::Key::Home));
360        assert!(matches!(map_key("end").unwrap(), enigo::Key::End));
361        assert!(matches!(map_key("pageup").unwrap(), enigo::Key::PageUp));
362        assert!(matches!(map_key("pagedown").unwrap(), enigo::Key::PageDown));
363        assert!(matches!(map_key("f1").unwrap(), enigo::Key::F1));
364        assert!(matches!(map_key("F12").unwrap(), enigo::Key::F12));
365        assert!(matches!(map_key("meta").unwrap(), enigo::Key::Meta));
366        assert!(matches!(map_key("shift").unwrap(), enigo::Key::Shift));
367    }
368
369    #[test]
370    fn maps_single_characters_and_rejects_unknown_names() {
371        match map_key("a").unwrap() {
372            enigo::Key::Unicode(ch) => assert_eq!(ch, 'a'),
373            other => panic!("expected Unicode a, got {other:?}"),
374        }
375        match map_key("Z").unwrap() {
376            enigo::Key::Unicode(ch) => assert_eq!(ch, 'Z'),
377            other => panic!("expected Unicode Z, got {other:?}"),
378        }
379        match map_key("4").unwrap() {
380            enigo::Key::Unicode(ch) => assert_eq!(ch, '4'),
381            other => panic!("expected Unicode 4, got {other:?}"),
382        }
383        assert!(map_key("not-a-key").is_err());
384        assert!(map_key("").is_err());
385        assert!(validate_key("enter").is_ok());
386        assert!(validate_key("nope").is_err());
387    }
388
389    #[test]
390    fn modifiers_use_meta_for_command_and_super() {
391        assert!(matches!(map_modifier(Modifier::Command), enigo::Key::Meta));
392        assert!(matches!(map_modifier(Modifier::Meta), enigo::Key::Meta));
393        assert!(matches!(
394            map_modifier(Modifier::Control),
395            enigo::Key::Control
396        ));
397        assert!(matches!(map_modifier(Modifier::Shift), enigo::Key::Shift));
398        assert!(matches!(map_modifier(Modifier::Alt), enigo::Key::Alt));
399        assert!(matches!(
400            map_button(MouseButton::Left).unwrap(),
401            Button::Left
402        ));
403        assert!(matches!(
404            map_button(MouseButton::Right).unwrap(),
405            Button::Right
406        ));
407        assert!(matches!(
408            map_button(MouseButton::Middle).unwrap(),
409            Button::Middle
410        ));
411        #[cfg(target_os = "macos")]
412        {
413            assert_eq!(
414                map_button(MouseButton::Back).unwrap_err().code(),
415                "unsupported"
416            );
417            assert_eq!(validate_key("f24").unwrap_err().code(), "unsupported");
418        }
419        #[cfg(not(target_os = "macos"))]
420        {
421            assert!(matches!(
422                map_button(MouseButton::Back).unwrap(),
423                Button::Back
424            ));
425            assert!(matches!(
426                map_button(MouseButton::Forward).unwrap(),
427                Button::Forward
428            ));
429            assert!(validate_key("f24").is_ok());
430        }
431        assert!(matches!(map_key("plus").unwrap(), enigo::Key::Unicode('+')));
432        #[cfg(target_os = "macos")]
433        assert!(matches!(map_key("insert").unwrap(), enigo::Key::Help));
434        #[cfg(not(target_os = "macos"))]
435        assert!(matches!(map_key("insert").unwrap(), enigo::Key::Insert));
436        assert!(validate_key("f13").is_ok());
437    }
438}
computer_use/backend/input.rs

computer_use/backend/
input.rs
