computer_use/platform/macos/
mod.rs

1mod apps;
2mod ax;
3mod input;
4mod keys;
5mod menu;
6mod screenshot;
7mod skylight;
8mod tree;
9mod window;
10
11use std::ops::Range;
12
13use objc2_application_services::{
14    AXIsProcessTrusted, AXIsProcessTrustedWithOptions, AXUIElement, kAXTrustedCheckOptionPrompt,
15};
16use objc2_core_foundation::{CFBoolean, CFDictionary, CFNumber, CFRange, CFString, CFType};
17use objc2_core_graphics::{CGPreflightScreenCaptureAccess, CGRequestScreenCaptureAccess};
18
19use self::ax::AxElement;
20use crate::app::AppInfo;
21use crate::backend::{Backend, Capture, CaptureRequest, Delivery, InputTarget, MouseButton, Permissions};
22use crate::element::RawNode;
23use crate::error::{Error, Permission, Result};
24use crate::geometry::Point;
25use crate::keys::KeyChord;
26use crate::state::WindowInfo;
27use crate::text::utf16_offset;
28
29const INSERTABLE_ROLES: [&str; 3] = ["AXTextField", "AXTextArea", "AXComboBox"];
30
31/// The macOS [`Backend`].
32///
33/// Requires macOS 14 or later, and the Accessibility and Screen Recording permissions for
34/// the host process (usually the terminal or app that runs your binary).
35#[derive(Debug, Default)]
36pub struct MacBackend {
37    installed: apps::InstalledCache,
38}
39
40impl MacBackend {
41    /// Creates the backend. Never fails; missing permissions surface on first use.
42    ///
43    /// # Errors
44    ///
45    /// Infallible today; the `Result` keeps the signature uniform across platforms.
46    pub fn new() -> Result<Self> {
47        Ok(Self::default())
48    }
49}
50
51/// A live macOS accessibility element: the [`MacBackend`] handle type.
52///
53/// Unlike a bare `CFRetained<AXUIElement>`, it is `Send` and `Sync`, so a
54/// [`Computer`](crate::Computer) over [`MacBackend`] can move between threads.
55#[derive(Debug, Clone, PartialEq, Eq, Hash)]
56pub struct MacElement(AxElement);
57
58#[allow(clippy::non_send_fields_in_send_ty)] // see the SAFETY comment
59// SAFETY: an AXUIElement is an immutable CoreFoundation reference to a remote element. CF
60// retain/release are atomic, and the Accessibility client API may be called from any thread:
61// every call is a synchronous request to the target process that touches no thread-local state.
62// The objc2 bindings leave it !Send only because they cannot know this about the type.
63unsafe impl Send for MacElement {}
64// SAFETY: as above; the element has no interior state that `&self` methods could race on.
65unsafe impl Sync for MacElement {}
66
67impl MacElement {
68    /// The underlying `AXUIElement`.
69    #[must_use]
70    pub fn as_ax(&self) -> &AXUIElement {
71        &self.0
72    }
73}
74
75fn wrap(nodes: Vec<RawNode<AxElement>>) -> Vec<RawNode<MacElement>> {
76    nodes.into_iter().map(|n| RawNode { handle: MacElement(n.handle), parent: n.parent, data: n.data }).collect()
77}
78
79fn require_accessibility() -> Result<()> {
80    // SAFETY: AXIsProcessTrusted has no preconditions.
81    if unsafe { AXIsProcessTrusted() } { Ok(()) } else { Err(Error::PermissionDenied(Permission::Accessibility)) }
82}
83
84fn pid_of(app: &AppInfo) -> Result<i32> {
85    app.pid.and_then(|p| i32::try_from(p).ok()).ok_or_else(|| Error::invalid(format!("{} is not running", app.name)))
86}
87
88impl Backend for MacBackend {
89    type Handle = MacElement;
90
91    fn list_apps(&self) -> Result<Vec<AppInfo>> {
92        let mut apps = apps::running_apps();
93        let installed: Vec<AppInfo> = self
94            .installed
95            .get()
96            .into_iter()
97            .filter(|candidate| {
98                !apps.iter().any(|running| {
99                    (candidate.bundle_id.is_some() && running.bundle_id == candidate.bundle_id)
100                        || (candidate.path.is_some() && running.path == candidate.path)
101                })
102            })
103            .collect();
104        apps.extend(installed);
105        Ok(apps)
106    }
107
108    fn launch_app(&self, app: &AppInfo) -> Result<AppInfo> {
109        apps::launch(app)
110    }
111
112    fn permissions(&self) -> Permissions {
113        // SAFETY: AXIsProcessTrusted has no preconditions.
114        let accessibility = unsafe { AXIsProcessTrusted() };
115        Permissions::new(accessibility, CGPreflightScreenCaptureAccess())
116    }
117
118    fn request_permissions(&self) -> Permissions {
119        // SAFETY: reading an immutable framework constant.
120        let key: &CFString = unsafe { kAXTrustedCheckOptionPrompt };
121        let options = CFDictionary::<CFString, CFType>::from_slices(&[key], &[CFBoolean::new(true).as_ref()]);
122        // SAFETY: `options` is a valid dictionary with the documented key.
123        let accessibility = unsafe { AXIsProcessTrustedWithOptions(Some(options.as_opaque())) };
124        let screen = CGPreflightScreenCaptureAccess() || CGRequestScreenCaptureAccess();
125        Permissions::new(accessibility, screen)
126    }
127
128    fn list_windows(&self, app: &AppInfo) -> Result<Vec<WindowInfo>> {
129        require_accessibility()?;
130        let pid = pid_of(app)?;
131        Ok(window::list(&ax::application(pid), pid))
132    }
133
134    fn capture(&self, app: &AppInfo, request: &CaptureRequest) -> Result<Capture<MacElement>> {
135        require_accessibility()?;
136        let pid = pid_of(app)?;
137        let app_element = ax::application(pid);
138        // Chromium and Electron only build their tree once an assistive client asks for it.
139        let _ = ax::set_bool(&app_element, "AXManualAccessibility", true);
140
141        let mut notes = Vec::new();
142        let server_only = |server: window::ServerWindow, notes: &mut Vec<String>| {
143            notes.push("Accessibility exposes no window for this app; the tree is rooted at the app.".to_owned());
144            let id = server.id;
145            (window::TargetWindow { element: app_element.clone(), frame: server.frame, title: server.title }, Some(id))
146        };
147        let (target, window_id) = match request.window {
148            Some(id) => match window::by_id(&app_element, pid, id, &app.name)? {
149                window::Selected::Ax(target) => {
150                    let id = u32::try_from(id).ok();
151                    (target, id)
152                }
153                window::Selected::ServerOnly(server) => server_only(server, &mut notes),
154            },
155            None => match window::main_window(&app_element, &app.name) {
156                Ok(target) => {
157                    let id = window::server_id(&target.element, pid, &window::server_windows(pid));
158                    (target, id)
159                }
160                // Some apps expose no AX window for a visible one; use the window server's geometry.
161                Err(Error::WindowNotFound(name)) => {
162                    let server = window::server_main_window(pid).ok_or(Error::WindowNotFound(name))?;
163                    server_only(server, &mut notes)
164                }
165                Err(e) => return Err(e),
166            },
167        };
168        let mut capture = Capture::new(WindowInfo::new(window_id.map(u64::from), target.title.clone(), target.frame));
169        capture.notes = notes;
170
171        if request.include_tree {
172            let focused = ax::element(&app_element, "AXFocusedUIElement");
173            let walk =
174                tree::walk(&target.element, target.frame, focused.as_ref(), request.max_nodes, request.max_depth);
175            let mut raw = walk.nodes;
176            capture.truncated = walk.truncated;
177            capture.focused = walk.focused;
178            if request.include_menu_bar
179                && let Some(bar) = ax::element(&app_element, "AXMenuBar")
180            {
181                let base = raw.len();
182                let budget = request.max_nodes.saturating_sub(base);
183                let bar_frame = ax::frame(&bar).unwrap_or(target.frame);
184                let menu = tree::walk(&bar, bar_frame, focused.as_ref(), budget, request.max_depth);
185                capture.truncated |= menu.truncated;
186                capture.focused = capture.focused.or_else(|| menu.focused.map(|i| base + i));
187                raw.extend(menu.nodes.into_iter().map(|mut n| {
188                    n.parent = n.parent.map(|p| base + p);
189                    n
190                }));
191            }
192            capture.nodes = wrap(raw);
193            capture.selected_text =
194                focused.as_ref().and_then(|f| ax::string(f, "AXSelectedText")).filter(|s| !s.is_empty());
195        }
196
197        if request.include_screenshot {
198            match window_id.map(screenshot::capture_window) {
199                Some(Ok(image)) => capture.screenshot = Some(image),
200                Some(Err(e @ Error::PermissionDenied(_))) => return Err(e),
201                Some(Err(e)) => capture.notes.push(format!("Screenshot omitted: {e}")),
202                None => capture.notes.push("Screenshot omitted: the window has no window-server ID".to_owned()),
203            }
204        }
205        Ok(capture)
206    }
207
208    fn perform_action(&self, element: &MacElement, action: &str) -> Result<()> {
209        ax::perform(&element.0, action)
210    }
211
212    fn set_value(&self, element: &MacElement, value: &str) -> Result<()> {
213        let element = &element.0;
214        if !ax::is_settable(element, "AXValue") {
215            return Err(Error::action("this element's value is not settable"));
216        }
217        let numeric = ax::attribute(element, "AXValue").is_some_and(|v| v.downcast_ref::<CFNumber>().is_some());
218        if numeric {
219            let number: f64 = value.trim().parse().map_err(|_| Error::invalid(format!("{value:?} is not a number")))?;
220            ax::set(element, "AXValue", &CFNumber::new_f64(number))
221        } else {
222            ax::set(element, "AXValue", &CFString::from_str(value))
223        }
224    }
225
226    fn value(&self, element: &MacElement) -> Result<Option<String>> {
227        Ok(ax::string(&element.0, "AXValue"))
228    }
229
230    fn select_text(&self, element: &MacElement, value: &str, range: Range<usize>) -> Result<()> {
231        let element = &element.0;
232        let start = utf16_offset(value, range.start);
233        let len = utf16_offset(value, range.end) - start;
234        let cf_range = CFRange::new(
235            isize::try_from(start).map_err(|_| Error::invalid("selection start is out of range"))?,
236            isize::try_from(len).map_err(|_| Error::invalid("selection length is out of range"))?,
237        );
238        let _ = ax::set_bool(element, "AXFocused", true);
239        ax::set_range(element, "AXSelectedTextRange", cf_range)?;
240        // Verify, since some apps accept the write but ignore it.
241        let applied = ax::attribute(element, "AXSelectedTextRange").as_deref().and_then(ax::as_range);
242        match applied {
243            Some(r) if r.location == cf_range.location && r.length == cf_range.length => Ok(()),
244            Some(_) => Err(Error::action("the app did not apply the requested selection")),
245            None => Ok(()),
246        }
247    }
248
249    fn insert_text(&self, app: &AppInfo, text: &str) -> Result<bool> {
250        let app_element = ax::application(pid_of(app)?);
251        let Some(focused) = ax::element(&app_element, "AXFocusedUIElement") else { return Ok(false) };
252        let role = ax::string(&focused, "AXRole").unwrap_or_default();
253        if !INSERTABLE_ROLES.contains(&role.as_str()) || !ax::is_settable(&focused, "AXSelectedText") {
254            return Ok(false);
255        }
256        let before = ax::string(&focused, "AXValue");
257        ax::set(&focused, "AXSelectedText", &CFString::from_str(text))?;
258        // Some editors accept the write without changing anything; let the caller fall back.
259        Ok(ax::string(&focused, "AXValue") != before)
260    }
261
262    fn click(&self, target: InputTarget<'_>, point: Point, button: MouseButton, count: u32) -> Result<()> {
263        if target.delivery == Delivery::FocusedBackground
264            && button == MouseButton::Left
265            && let Some((pid, window)) = focus_target(target)
266        {
267            let local = Point::new(point.x - window.frame.min_x(), point.y - window.frame.min_y());
268            let id = window_server_id(window).unwrap_or_default();
269            if skylight::get().click(pid, id, point, local, count)? {
270                return Ok(());
271            }
272        }
273        input::click(target, point, button, count)
274    }
275
276    fn scroll(&self, target: InputTarget<'_>, point: Point, dx: f64, dy: f64) -> Result<()> {
277        input::scroll(target, point, dx, dy)
278    }
279
280    fn drag(&self, target: InputTarget<'_>, from: Point, to: Point) -> Result<()> {
281        input::drag(target, from, to)
282    }
283
284    fn type_text(&self, target: InputTarget<'_>, text: &str) -> Result<()> {
285        let _focus = match (target.delivery, focus_target(target)) {
286            (Delivery::FocusedBackground, Some((pid, window))) => focus(pid, window)?,
287            _ => None,
288        };
289        input::type_text(target, text)
290    }
291
292    fn press_key(&self, target: InputTarget<'_>, chord: &KeyChord) -> Result<()> {
293        let shortcut = menu::shortcut(chord);
294        // Background apps ignore key equivalents, so Command chords always get synthetic focus.
295        let wants_focus = target.delivery == Delivery::FocusedBackground
296            || (target.delivery == Delivery::Background && shortcut.is_some());
297        let Some((pid, window)) = focus_target(target).filter(|_| wants_focus) else {
298            return input::press_key(target, chord);
299        };
300        let _focus = focus(pid, window)?;
301        if let Some(item) = shortcut.and_then(|wanted| menu::find_item(&ax::application(pid), &wanted)) {
302            ax::perform(&item, "AXPress")?;
303            std::thread::sleep(std::time::Duration::from_millis(100));
304            return Ok(());
305        }
306        input::press_key(target, chord)
307    }
308}
309
310fn focus_target(target: InputTarget<'_>) -> Option<(i32, &WindowInfo)> {
311    let pid = pid_of(target.app).ok()?;
312    let window = target.window?;
313    window_server_id(window)?;
314    let frontmost = ax::element(&ax::system_wide(), "AXFocusedApplication").and_then(|app| ax::pid(&app));
315    (frontmost != Some(pid)).then_some((pid, window))
316}
317
318fn window_server_id(window: &WindowInfo) -> Option<u32> {
319    window.id.and_then(|id| u32::try_from(id).ok())
320}
321
322fn focus(pid: i32, window: &WindowInfo) -> Result<Option<skylight::FocusGuard<'static>>> {
323    window_server_id(window).map_or(Ok(None), |id| skylight::get().focus(pid, id))
324}
computer_use/platform/macos/mod.rs

computer_use/platform/macos/
mod.rs
