computer_use/platform/macos/
mod.rs

1mod apps;
2mod ax;
3mod input;
4mod keys;
5mod menu;
6mod screenshot;
7mod skylight;
8mod tree;
9mod window;
10
11use std::ops::Range;
12
13use objc2_application_services::{AXIsProcessTrusted, AXIsProcessTrustedWithOptions, kAXTrustedCheckOptionPrompt};
14use objc2_core_foundation::{CFBoolean, CFDictionary, CFNumber, CFRange, CFString, CFType};
15use objc2_core_graphics::{CGPreflightScreenCaptureAccess, CGRequestScreenCaptureAccess};
16
17use self::ax::AxElement;
18use crate::app::AppInfo;
19use crate::backend::{Backend, Capture, CaptureRequest, Delivery, InputTarget, MouseButton, Permissions};
20use crate::error::{Error, Permission, Result};
21use crate::geometry::Point;
22use crate::keys::KeyChord;
23use crate::state::WindowInfo;
24use crate::text::utf16_offset;
25
26const INSERTABLE_ROLES: [&str; 3] = ["AXTextField", "AXTextArea", "AXComboBox"];
27
28/// The macOS [`Backend`].
29///
30/// Requires macOS 14 or later, and the Accessibility and Screen Recording permissions for
31/// the host process (usually the terminal or app that runs your binary).
32#[derive(Debug, Default)]
33pub struct MacBackend {
34    installed: apps::InstalledCache,
35}
36
37impl MacBackend {
38    /// Creates the backend. Never fails; missing permissions surface on first use.
39    ///
40    /// # Errors
41    ///
42    /// Infallible today; the `Result` keeps the signature uniform across platforms.
43    pub fn new() -> Result<Self> {
44        Ok(Self::default())
45    }
46}
47
48fn require_accessibility() -> Result<()> {
49    // SAFETY: AXIsProcessTrusted has no preconditions.
50    if unsafe { AXIsProcessTrusted() } { Ok(()) } else { Err(Error::PermissionDenied(Permission::Accessibility)) }
51}
52
53fn pid_of(app: &AppInfo) -> Result<i32> {
54    app.pid.and_then(|p| i32::try_from(p).ok()).ok_or_else(|| Error::invalid(format!("{} is not running", app.name)))
55}
56
57impl Backend for MacBackend {
58    type Handle = AxElement;
59
60    fn list_apps(&self) -> Result<Vec<AppInfo>> {
61        let mut apps = apps::running_apps();
62        let installed: Vec<AppInfo> = self
63            .installed
64            .get()
65            .into_iter()
66            .filter(|candidate| {
67                !apps.iter().any(|running| {
68                    (candidate.bundle_id.is_some() && running.bundle_id == candidate.bundle_id)
69                        || (candidate.path.is_some() && running.path == candidate.path)
70                })
71            })
72            .collect();
73        apps.extend(installed);
74        Ok(apps)
75    }
76
77    fn launch_app(&self, app: &AppInfo) -> Result<AppInfo> {
78        apps::launch(app)
79    }
80
81    fn permissions(&self) -> Permissions {
82        // SAFETY: AXIsProcessTrusted has no preconditions.
83        let accessibility = unsafe { AXIsProcessTrusted() };
84        Permissions::new(accessibility, CGPreflightScreenCaptureAccess())
85    }
86
87    fn request_permissions(&self) -> Permissions {
88        // SAFETY: reading an immutable framework constant.
89        let key: &CFString = unsafe { kAXTrustedCheckOptionPrompt };
90        let options = CFDictionary::<CFString, CFType>::from_slices(&[key], &[CFBoolean::new(true).as_ref()]);
91        // SAFETY: `options` is a valid dictionary with the documented key.
92        let accessibility = unsafe { AXIsProcessTrustedWithOptions(Some(options.as_opaque())) };
93        let screen = CGPreflightScreenCaptureAccess() || CGRequestScreenCaptureAccess();
94        Permissions::new(accessibility, screen)
95    }
96
97    fn capture(&self, app: &AppInfo, request: &CaptureRequest) -> Result<Capture<AxElement>> {
98        require_accessibility()?;
99        let pid = pid_of(app)?;
100        let app_element = ax::application(pid);
101        // Chromium and Electron only build their tree once an assistive client asks for it.
102        let _ = ax::set_bool(&app_element, "AXManualAccessibility", true);
103
104        let mut notes = Vec::new();
105        let target = match window::main_window(&app_element, &app.name) {
106            Ok(target) => target,
107            // Some apps expose no AX window for a visible one; use the window server's geometry.
108            Err(Error::WindowNotFound(name)) => {
109                let server = window::server_main_window(pid).ok_or(Error::WindowNotFound(name))?;
110                notes.push("Accessibility exposes no window for this app; the tree is rooted at the app.".to_owned());
111                window::TargetWindow { element: app_element.clone(), frame: server.frame, title: server.title }
112            }
113            Err(e) => return Err(e),
114        };
115        let window_id = skylight::get()
116            .window_id(&target.element)
117            .filter(|id| window::server_windows(pid).iter().any(|w| w.id == *id))
118            .or_else(|| window::window_id(pid, target.frame, target.title.as_deref()));
119        let mut capture = Capture::new(WindowInfo::new(window_id.map(u64::from), target.title.clone(), target.frame));
120        capture.notes = notes;
121
122        if request.include_tree {
123            let focused = ax::element(&app_element, "AXFocusedUIElement");
124            let walk =
125                tree::walk(&target.element, target.frame, focused.as_ref(), request.max_nodes, request.max_depth);
126            capture.nodes = walk.nodes;
127            capture.truncated = walk.truncated;
128            capture.focused = walk.focused;
129            capture.selected_text =
130                focused.as_ref().and_then(|f| ax::string(f, "AXSelectedText")).filter(|s| !s.is_empty());
131        }
132
133        if request.include_screenshot {
134            match window_id.map(screenshot::capture_window) {
135                Some(Ok(image)) => capture.screenshot = Some(image),
136                Some(Err(e @ Error::PermissionDenied(_))) => return Err(e),
137                Some(Err(e)) => capture.notes.push(format!("Screenshot omitted: {e}")),
138                None => capture.notes.push("Screenshot omitted: the window has no window-server ID".to_owned()),
139            }
140        }
141        Ok(capture)
142    }
143
144    fn perform_action(&self, element: &AxElement, action: &str) -> Result<()> {
145        ax::perform(element, action)
146    }
147
148    fn set_value(&self, element: &AxElement, value: &str) -> Result<()> {
149        if !ax::is_settable(element, "AXValue") {
150            return Err(Error::action("this element's value is not settable"));
151        }
152        let numeric = ax::attribute(element, "AXValue").is_some_and(|v| v.downcast_ref::<CFNumber>().is_some());
153        if numeric {
154            let number: f64 = value.trim().parse().map_err(|_| Error::invalid(format!("{value:?} is not a number")))?;
155            ax::set(element, "AXValue", &CFNumber::new_f64(number))
156        } else {
157            ax::set(element, "AXValue", &CFString::from_str(value))
158        }
159    }
160
161    fn value(&self, element: &AxElement) -> Result<Option<String>> {
162        Ok(ax::string(element, "AXValue"))
163    }
164
165    fn select_text(&self, element: &AxElement, value: &str, range: Range<usize>) -> Result<()> {
166        let start = utf16_offset(value, range.start);
167        let len = utf16_offset(value, range.end) - start;
168        let cf_range = CFRange::new(
169            isize::try_from(start).map_err(|_| Error::invalid("selection start is out of range"))?,
170            isize::try_from(len).map_err(|_| Error::invalid("selection length is out of range"))?,
171        );
172        let _ = ax::set_bool(element, "AXFocused", true);
173        ax::set_range(element, "AXSelectedTextRange", cf_range)?;
174        // Verify, since some apps accept the write but ignore it.
175        let applied = ax::attribute(element, "AXSelectedTextRange").as_deref().and_then(ax::as_range);
176        match applied {
177            Some(r) if r.location == cf_range.location && r.length == cf_range.length => Ok(()),
178            Some(_) => Err(Error::action("the app did not apply the requested selection")),
179            None => Ok(()),
180        }
181    }
182
183    fn insert_text(&self, app: &AppInfo, text: &str) -> Result<bool> {
184        let app_element = ax::application(pid_of(app)?);
185        let Some(focused) = ax::element(&app_element, "AXFocusedUIElement") else { return Ok(false) };
186        let role = ax::string(&focused, "AXRole").unwrap_or_default();
187        if !INSERTABLE_ROLES.contains(&role.as_str()) || !ax::is_settable(&focused, "AXSelectedText") {
188            return Ok(false);
189        }
190        let before = ax::string(&focused, "AXValue");
191        ax::set(&focused, "AXSelectedText", &CFString::from_str(text))?;
192        // Some editors accept the write without changing anything; let the caller fall back.
193        Ok(ax::string(&focused, "AXValue") != before)
194    }
195
196    fn click(&self, target: InputTarget<'_>, point: Point, button: MouseButton, count: u32) -> Result<()> {
197        if target.delivery == Delivery::FocusedBackground
198            && button == MouseButton::Left
199            && let Some((pid, window)) = focus_target(target)
200        {
201            let local = Point::new(point.x - window.frame.min_x(), point.y - window.frame.min_y());
202            let id = window_server_id(window).unwrap_or_default();
203            if skylight::get().click(pid, id, point, local, count)? {
204                return Ok(());
205            }
206        }
207        input::click(target, point, button, count)
208    }
209
210    fn scroll(&self, target: InputTarget<'_>, point: Point, dx: f64, dy: f64) -> Result<()> {
211        input::scroll(target, point, dx, dy)
212    }
213
214    fn drag(&self, target: InputTarget<'_>, from: Point, to: Point) -> Result<()> {
215        input::drag(target, from, to)
216    }
217
218    fn type_text(&self, target: InputTarget<'_>, text: &str) -> Result<()> {
219        let _focus = match (target.delivery, focus_target(target)) {
220            (Delivery::FocusedBackground, Some((pid, window))) => focus(pid, window)?,
221            _ => None,
222        };
223        input::type_text(target, text)
224    }
225
226    fn press_key(&self, target: InputTarget<'_>, chord: &KeyChord) -> Result<()> {
227        let shortcut = menu::shortcut(chord);
228        // Background apps ignore key equivalents, so Command chords always get synthetic focus.
229        let wants_focus = target.delivery == Delivery::FocusedBackground
230            || (target.delivery == Delivery::Background && shortcut.is_some());
231        let Some((pid, window)) = focus_target(target).filter(|_| wants_focus) else {
232            return input::press_key(target, chord);
233        };
234        let _focus = focus(pid, window)?;
235        if let Some(item) = shortcut.and_then(|wanted| menu::find_item(&ax::application(pid), &wanted)) {
236            ax::perform(&item, "AXPress")?;
237            std::thread::sleep(std::time::Duration::from_millis(100));
238            return Ok(());
239        }
240        input::press_key(target, chord)
241    }
242}
243
244fn focus_target(target: InputTarget<'_>) -> Option<(i32, &WindowInfo)> {
245    let pid = pid_of(target.app).ok()?;
246    let window = target.window?;
247    window_server_id(window)?;
248    let frontmost = ax::element(&ax::system_wide(), "AXFocusedApplication").and_then(|app| ax::pid(&app));
249    (frontmost != Some(pid)).then_some((pid, window))
250}
251
252fn window_server_id(window: &WindowInfo) -> Option<u32> {
253    window.id.and_then(|id| u32::try_from(id).ok())
254}
255
256fn focus(pid: i32, window: &WindowInfo) -> Result<Option<skylight::FocusGuard<'static>>> {
257    window_server_id(window).map_or(Ok(None), |id| skylight::get().focus(pid, id))
258}
computer_use/platform/macos/mod.rs

computer_use/platform/macos/
mod.rs
