computer_use/
lib.rs

1//! Cross-platform computer-use primitives: observe desktop apps through their accessibility
2//! trees and screenshots, and drive them with accessibility actions and synthetic input.
3//!
4//! Use it for UI automation, end-to-end testing of desktop apps, accessibility tooling, or
5//! AI agents. It provides:
6//!
7//! - [`Computer`]: a session for observing and acting on apps:
8//!   [`list_apps`](Computer::list_apps), [`get_app_state`](Computer::get_app_state),
9//!   [`click`](Computer::click), [`perform_secondary_action`](Computer::perform_secondary_action),
10//!   [`scroll`](Computer::scroll), [`drag`](Computer::drag), [`type_text`](Computer::type_text),
11//!   [`press_key`](Computer::press_key), [`set_value`](Computer::set_value),
12//!   [`select_text`](Computer::select_text) and [`query`](Computer::query).
13//! - [`AppState`]: a captured window with an indexed element tree, a PNG [`Screenshot`], and a
14//!   compact [text rendering](AppState::render) for logging, diffing or prompts.
15//! - [`Backend`]: the OS abstraction. [`NativeBackend`] is the implementation for the current
16//!   platform; implement the trait yourself for tests or other platforms.
17//!
18//! # Platform support
19//!
20//! | Platform | Tree | Screenshot | Input |
21//! |----------|------|------------|-------|
22//! | macOS 14+ | Accessibility (`AXUIElement`) | window-server capture, then `ScreenCaptureKit` | `CGEvent` posted to the app's process (optionally with `SkyLight` synthetic focus), or globally |
23//! | Windows 10+ | UI Automation | `PrintWindow` / GDI | window messages to the app, or `SendInput` |
24//! | Linux (X11) | AT-SPI over D-Bus | X11 `GetImage` | AT-SPI device event controller (global only) |
25//!
26//! On macOS the host process needs the **Accessibility** and **Screen Recording**
27//! permissions; see [`Computer::permissions`] and [`Computer::request_permissions`].
28//! Without them, apps may appear to have no windows. On Linux, accessibility must be enabled
29//! for the session (it is by default on GNOME and KDE); Wayland sessions get trees and
30//! actions but no screenshots, and synthetic input only reaches X11/`XWayland` apps.
31//!
32//! # Example
33//!
34//! ```no_run
35//! # #[cfg(target_os = "macos")]
36//! # fn main() -> computer_use::Result<()> {
37//! use computer_use::{CaptureOptions, ClickOptions, ClickTarget, Computer, ElementQuery};
38//!
39//! let mut computer = Computer::new()?;
40//! let state = computer.get_app_state("TextEdit", &CaptureOptions::default())?;
41//! println!("{state}");
42//!
43//! let text_area = state
44//!     .find(&ElementQuery::role("text area"))
45//!     .next()
46//!     .map(|e| e.index)
47//!     .expect("TextEdit has a text area");
48//! computer.click("TextEdit", ClickTarget::Element(text_area), ClickOptions::default())?;
49//! computer.type_text("TextEdit", "Hello from Rust!")?;
50//! # Ok(())
51//! # }
52//! # #[cfg(not(target_os = "macos"))]
53//! # fn main() {}
54//! ```
55
56#![cfg_attr(docsrs, feature(doc_cfg))]
57
58mod app;
59mod backend;
60mod computer;
61mod element;
62mod error;
63mod geometry;
64pub mod keys;
65mod platform;
66mod screenshot;
67mod state;
68pub mod text;
69
70pub use app::AppInfo;
71pub use backend::{Backend, Capture, CaptureRequest, Delivery, InputTarget, MouseButton, Permissions, ScrollDirection};
72pub use computer::{ClickMethod, ClickOptions, ClickOutcome, ClickTarget, Computer};
73pub use element::{Action, Element, ElementData, ElementStates, RawNode};
74pub use error::{Error, Permission, Result};
75pub use geometry::{Point, Rect, Size};
76#[cfg(any(target_os = "macos", target_os = "windows", target_os = "linux"))]
77pub use platform::NativeBackend;
78#[cfg(target_os = "linux")]
79pub use platform::linux::{LinuxBackend, LinuxElement};
80#[cfg(target_os = "macos")]
81pub use platform::macos::MacBackend;
82#[cfg(target_os = "windows")]
83pub use platform::windows::WindowsBackend;
84pub use screenshot::{RawImage, Screenshot};
85pub use state::{AppState, CaptureOptions, ElementQuery, WindowInfo};
86pub use text::{SelectionMode, TextSelection};
computer_use/lib.rs

computer_use/
lib.rs
