computer_use/
lib.rs

1//! Cross-platform computer-use primitives: observe desktop apps through their accessibility
2//! trees and screenshots, and drive them with accessibility actions and synthetic input.
3//!
4//! Use it for UI automation, end-to-end testing of desktop apps, accessibility tooling, or
5//! AI agents. It provides:
6//!
7//! - [`Computer`]: a session for observing and acting on apps:
8//!   [`list_apps`](Computer::list_apps), [`list_windows`](Computer::list_windows),
9//!   [`get_app_state`](Computer::get_app_state), [`click`](Computer::click),
10//!   [`perform_secondary_action`](Computer::perform_secondary_action), [`scroll`](Computer::scroll),
11//!   [`scroll_at`](Computer::scroll_at), [`drag`](Computer::drag),
12//!   [`drag_screen`](Computer::drag_screen), [`type_text`](Computer::type_text),
13//!   [`press_key`](Computer::press_key), [`set_value`](Computer::set_value),
14//!   [`select_text`](Computer::select_text) and [`query`](Computer::query).
15//!   With the native backend it is `Send`, so it can be shared across threads behind a mutex.
16//! - [`AppState`]: a captured window with an indexed element tree, a PNG [`Screenshot`], and a
17//!   compact [text rendering](AppState::render) for logging, diffing or prompts.
18//! - [`Backend`]: the OS abstraction. [`NativeBackend`] is the implementation for the current
19//!   platform; implement the trait yourself for tests or other platforms.
20//!
21//! # Platform support
22//!
23//! | Platform | Tree | Screenshot | Input |
24//! |----------|------|------------|-------|
25//! | macOS 14+ | Accessibility (`AXUIElement`) | window-server capture, then `ScreenCaptureKit` | `CGEvent` posted to the app's process (optionally with `SkyLight` synthetic focus), or globally |
26//! | Windows 10+ | UI Automation | `PrintWindow` / GDI | window messages to the app, or `SendInput` |
27//! | Linux (X11) | AT-SPI over D-Bus | X11 `GetImage` | AT-SPI device event controller (global only) |
28//!
29//! On macOS the host process needs the **Accessibility** and **Screen Recording**
30//! permissions; see [`Computer::permissions`] and [`Computer::request_permissions`].
31//! Without them, apps may appear to have no windows. On Linux, accessibility must be enabled
32//! for the session (it is by default on GNOME and KDE); Wayland sessions get trees and
33//! actions but no screenshots, and synthetic input only reaches X11/`XWayland` apps.
34//!
35//! # Example
36//!
37//! ```no_run
38//! # #[cfg(target_os = "macos")]
39//! # fn main() -> computer_use::Result<()> {
40//! use computer_use::{CaptureOptions, ClickOptions, ClickTarget, Computer, ElementQuery};
41//!
42//! let mut computer = Computer::new()?;
43//! let state = computer.get_app_state("TextEdit", &CaptureOptions::default())?;
44//! println!("{state}");
45//!
46//! let text_area = state
47//!     .find(&ElementQuery::role("text area"))
48//!     .next()
49//!     .map(|e| e.index)
50//!     .expect("TextEdit has a text area");
51//! computer.click("TextEdit", ClickTarget::Element(text_area), ClickOptions::default())?;
52//! computer.type_text("TextEdit", "Hello from Rust!")?;
53//! # Ok(())
54//! # }
55//! # #[cfg(not(target_os = "macos"))]
56//! # fn main() {}
57//! ```
58
59#![cfg_attr(docsrs, feature(doc_cfg))]
60
61mod app;
62mod backend;
63mod computer;
64mod element;
65mod error;
66mod geometry;
67pub mod keys;
68mod platform;
69mod screenshot;
70mod state;
71pub mod text;
72
73pub use app::AppInfo;
74pub use backend::{Backend, Capture, CaptureRequest, Delivery, InputTarget, MouseButton, Permissions, ScrollDirection};
75pub use computer::{ClickMethod, ClickOptions, ClickOutcome, ClickTarget, Computer};
76pub use element::{Action, Element, ElementData, ElementStates, RawNode};
77pub use error::{Error, Permission, Result};
78pub use geometry::{Point, Rect, Size};
79#[cfg(any(target_os = "macos", target_os = "windows", target_os = "linux"))]
80pub use platform::NativeBackend;
81#[cfg(target_os = "linux")]
82pub use platform::linux::{LinuxBackend, LinuxElement};
83#[cfg(target_os = "macos")]
84pub use platform::macos::{MacBackend, MacElement};
85#[cfg(target_os = "windows")]
86pub use platform::windows::{WindowsBackend, WindowsElement};
87pub use screenshot::{RawImage, Screenshot};
88pub use state::{AppState, CaptureOptions, ElementQuery, WindowInfo, WindowSelector};
89pub use text::{SelectionMode, TextSelection};
computer_use/lib.rs

computer_use/
lib.rs
