mod capture;
pub(crate) mod clipboard;
pub(crate) mod command;
pub(crate) mod held;
mod input;
#[cfg(target_os = "linux")]
mod linux;
#[cfg(target_os = "macos")]
mod macos;
#[cfg(target_os = "windows")]
mod windows;
use crate::element::{ElementError, ElementOutcome, ElementRequest, HitInfo};
use crate::error::ComputerError;
use crate::protocol::AppleEventLanguage;
use crate::target::{AppInfo, DoctorReport, LaunchResult, QuitResult, WindowBounds, WindowList};
use crate::tree::{TreeQuery, TreeReport};
use crate::types::{DisplayInfo, Frame, FrontmostInfo, Modifier, MouseButton, RawCapture, Region};
use crate::window_groups::SpacesReport;
use serde::Serialize;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct WindowTarget {
pub window_id: u32,
pub pid: i32,
pub bounds: WindowBounds,
}
pub(crate) fn unsupported(operation: &str) -> ComputerError {
ComputerError::Unsupported(format!(
"{operation} is not available on {}",
std::env::consts::OS
))
}
pub(crate) use command::run_with_stdin;
pub use input::validate_key;
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct MenuNode {
pub title: String,
pub enabled: bool,
pub children: Vec<MenuNode>,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct MenuListReport {
pub app: String,
pub sections: Vec<MenuNode>,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct MenuClickReport {
pub pressed: bool,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct AppleEventReport {
pub script_bytes: usize,
pub script_hash: String,
pub stdout: String,
pub stderr: String,
pub exit_status: i32,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct DockItem {
pub title: String,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct DockListReport {
pub items: Vec<DockItem>,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct DockClickReport {
pub pressed: bool,
}
pub trait DesktopBackend: Send + Sync {
fn screenshot(&self, region: Option<Region>) -> Result<Frame, ComputerError>;
fn capture_screen(&self, region: Option<Region>) -> Result<RawCapture, ComputerError> {
capture::capture_screen(region)
}
fn capture_window(&self, target: &WindowTarget) -> Result<RawCapture, ComputerError> {
#[cfg(target_os = "macos")]
{
macos::capture_window(target)
}
#[cfg(target_os = "linux")]
{
linux::capture_window(target)
}
#[cfg(not(any(target_os = "macos", target_os = "linux")))]
{
let _ = target.pid;
let _ = target.window_id;
capture::crop_window(target.bounds)
}
}
fn move_mouse(&self, x: i32, y: i32) -> Result<(), ComputerError>;
fn simulate_click(&self, x: i32, y: i32, button: MouseButton) -> Result<(), ComputerError>;
fn simulate_type(&self, text: &str) -> Result<(), ComputerError>;
fn simulate_key(&self, modifiers: &[Modifier], key: &str) -> Result<(), ComputerError>;
fn simulate_scroll(&self, dx: i32, dy: i32) -> Result<(), ComputerError>;
fn cursor_position(&self) -> Result<(i32, i32), ComputerError>;
fn list_displays(&self) -> Result<Vec<DisplayInfo>, ComputerError>;
fn frontmost(&self) -> FrontmostInfo;
fn inspect_accessibility(&self, app_name: Option<&str>) -> Result<String, ComputerError>;
fn list_apps(&self, _all: bool) -> Result<Vec<AppInfo>, ComputerError> {
Err(unsupported("list_apps"))
}
fn list_windows(&self) -> Result<WindowList, ComputerError> {
Err(unsupported("list_windows"))
}
fn launch(&self, _app: &str) -> Result<LaunchResult, ComputerError> {
Err(unsupported("launch"))
}
fn activate(&self, _app: &str, _window: Option<&str>) -> Result<LaunchResult, ComputerError> {
Err(unsupported("activate"))
}
fn quit(&self, _app: &str, _force: bool) -> Result<QuitResult, ComputerError> {
Err(unsupported("quit"))
}
fn menu_list(&self, _app: &str, _depth: u32) -> Result<MenuListReport, ComputerError> {
Err(unsupported("menu_list"))
}
fn menu_click(
&self,
_app: &str,
_path: &crate::menu_path::MenuPath,
) -> Result<MenuClickReport, ComputerError> {
Err(unsupported("menu_click"))
}
fn apple_event(
&self,
_app: &str,
_script: &str,
_language: AppleEventLanguage,
) -> Result<AppleEventReport, ComputerError> {
Err(unsupported("apple_event"))
}
fn dock_list(&self) -> Result<DockListReport, ComputerError> {
Err(unsupported("dock_list"))
}
fn dock_click(&self, _title: &str) -> Result<DockClickReport, ComputerError> {
Err(unsupported("dock_click"))
}
fn spaces(&self) -> Result<SpacesReport, ComputerError> {
Err(unsupported("spaces"))
}
fn doctor(
&self,
_options: crate::doctor::DoctorOptions,
) -> Result<DoctorReport, ComputerError> {
Err(unsupported("doctor"))
}
fn tree(&self, _query: &TreeQuery) -> Result<TreeReport, ComputerError> {
Err(unsupported("tree"))
}
fn end_session(&self, _session: Option<&str>) -> Result<Vec<i32>, ComputerError> {
Ok(Vec::new())
}
fn element_action(&self, request: &ElementRequest) -> Result<ElementOutcome, ElementError> {
#[cfg(target_os = "macos")]
{
macos::element_action(request)
}
#[cfg(not(target_os = "macos"))]
{
Err(ElementError::new(unsupported(request.op.action_name())))
}
}
fn display_block(&self) -> Option<ComputerError> {
None
}
fn browser_url(&self, pid: i32) -> Result<Option<String>, ComputerError> {
#[cfg(target_os = "macos")]
{
macos::browser_url(pid)
}
#[cfg(not(target_os = "macos"))]
{
let _ = pid;
Ok(None)
}
}
fn peek_element(
&self,
app: Option<&str>,
window: Option<&str>,
session: Option<&str>,
index: u32,
) -> Result<Option<crate::element::ElementPeek>, ComputerError> {
#[cfg(target_os = "macos")]
{
macos::peek_element(app, window, session, index)
}
#[cfg(not(target_os = "macos"))]
{
let _ = (app, window, session, index);
Ok(None)
}
}
fn user_input_pause_warning(&self) -> Option<&'static str> {
#[cfg(test)]
{
None
}
#[cfg(all(not(test), target_os = "macos"))]
{
macos::input::ensure_installed()
}
#[cfg(all(not(test), not(target_os = "macos")))]
{
Some("user-input pause unavailable")
}
}
fn hit_test(&self, x: i32, y: i32) -> Result<Option<HitInfo>, ComputerError> {
#[cfg(target_os = "macos")]
{
macos::hit_test(x, y)
}
#[cfg(not(target_os = "macos"))]
{
let _ = (x, y);
Ok(None)
}
}
fn input_support(&self) -> crate::tier::Capabilities {
#[cfg(target_os = "macos")]
{
crate::tier::Capabilities { event: true }
}
#[cfg(all(target_os = "linux", feature = "portal"))]
{
crate::tier::Capabilities { event: true }
}
#[cfg(all(target_os = "linux", not(feature = "portal")))]
{
crate::tier::Capabilities {
event: linux::x11_event_capable(false),
}
}
#[cfg(not(any(target_os = "macos", target_os = "linux")))]
{
crate::tier::Capabilities { event: false }
}
}
fn marks_for_tree(
&self,
_query: &TreeQuery,
_map: &crate::image_util::ScreenshotMap,
) -> Result<Vec<crate::vision::MarkBox>, ComputerError> {
Ok(Vec::new())
}
fn recognize_text(
&self,
_image: &image::RgbaImage,
_map: &crate::image_util::ScreenshotMap,
) -> Result<crate::vision::OcrResult, ComputerError> {
Err(unsupported("ocr"))
}
fn secure_ocr_frames(
&self,
_query: &TreeQuery,
_map: &crate::image_util::ScreenshotMap,
) -> Result<Vec<crate::vision::ocr::ImageRect>, ComputerError> {
Ok(Vec::new())
}
fn clipboard_get(&self) -> Result<clipboard::ClipboardContents, ComputerError> {
clipboard::clipboard_get()
}
fn clipboard_set_text(&self, text: &str) -> Result<(), ComputerError> {
clipboard::clipboard_set_text(text)
}
fn clipboard_set_image(
&self,
width: u32,
height: u32,
rgba: &[u8],
) -> Result<(), ComputerError> {
clipboard::clipboard_set_image(width, height, rgba)
}
fn release_held(&self) -> Result<(), ComputerError> {
held::release_all()
}
fn perform_input(
&self,
job: &crate::tier::InputJob,
) -> Result<crate::tier::InputReport, ComputerError> {
#[cfg(target_os = "macos")]
{
macos::perform_input(job)
}
#[cfg(not(target_os = "macos"))]
{
off_platform_input(self, job)
}
}
}
#[cfg(not(target_os = "macos"))]
fn off_platform_input<B: DesktopBackend + ?Sized>(
backend: &B,
job: &crate::tier::InputJob,
) -> Result<crate::tier::InputReport, ComputerError> {
use crate::tier::{InputKind, InputReport, TierName};
match job.tier {
TierName::Event => {
return Err(unsupported("tier event"));
}
TierName::Synthetic => {
return Err(unsupported("tier synthetic"));
}
TierName::Ax => {
return Err(unsupported("tier ax for coordinate and keyboard input"));
}
TierName::Activate => {}
}
if job.require_background {
return Err(ComputerError::ExecutionFailed(
"--require-background refuses tier activate because it uses the foreground cursor"
.into(),
));
}
match &job.kind {
InputKind::Click {
x,
y,
button,
count,
} => {
for index in 0..*count {
if index > 0 {
std::thread::sleep(std::time::Duration::from_millis(50));
}
backend.simulate_click(*x, *y, *button)?;
}
}
InputKind::Move { x, y } => backend.move_mouse(*x, *y)?,
InputKind::Scroll { x, y, dx, dy } => {
if let (Some(x), Some(y)) = (x, y) {
backend.move_mouse(*x, *y)?;
}
backend.simulate_scroll(*dx, *dy)?;
}
InputKind::Type { text } => backend.simulate_type(text)?,
InputKind::Key { strokes } => {
for stroke in strokes {
backend.simulate_key(&stroke.modifiers, &stroke.key)?;
}
}
InputKind::Drag { points, button } => input::drag(points, *button)?,
InputKind::MouseDown { x, y, button } => input::button_down(*x, *y, *button)?,
InputKind::MouseUp { x, y, button } => input::button_up(*x, *y, *button)?,
InputKind::HoldKey { modifiers, key, ms } => input::hold_key(modifiers, key, *ms)?,
}
Ok(InputReport {
tier: "activate",
cursor_moved: true,
focus_restored: None,
warnings: Vec::new(),
scroll: None,
})
}
pub(crate) fn serve_overlay() -> Result<(), ComputerError> {
#[cfg(all(target_os = "macos", not(test)))]
{
macos::overlay::serve()
}
#[cfg(all(target_os = "macos", test))]
{
Err(ComputerError::Unsupported(
"overlay serve does not open a window in tests".into(),
))
}
#[cfg(not(target_os = "macos"))]
{
Err(unsupported("overlay serve"))
}
}
pub fn shutdown_sessions() {
#[cfg(target_os = "macos")]
macos::shutdown_sessions();
}
pub use DesktopBackend as Backend;
pub fn create_desktop_backend() -> Box<dyn DesktopBackend> {
#[cfg(target_os = "macos")]
{
Box::new(macos::MacOsDesktopBackend)
}
#[cfg(target_os = "linux")]
{
Box::new(linux::LinuxDesktopBackend)
}
#[cfg(target_os = "windows")]
{
Box::new(windows::WindowsDesktopBackend)
}
#[cfg(not(any(target_os = "macos", target_os = "linux", target_os = "windows")))]
{
compile_error!("computer-use supports macOS, Linux, and Windows");
}
}
macro_rules! impl_platform_backend {
($ty:ty, $inspect:path, $front:path, $doctor:path, $list_apps:path, $list_windows:path, $launch:path, $activate:path, $quit:path, $tree:path, $end_session:path) => {
impl DesktopBackend for $ty {
fn screenshot(&self, region: Option<Region>) -> Result<Frame, ComputerError> {
capture::capture(region)
}
fn move_mouse(&self, x: i32, y: i32) -> Result<(), ComputerError> {
input::move_mouse(x, y)
}
fn simulate_click(
&self,
x: i32,
y: i32,
button: MouseButton,
) -> Result<(), ComputerError> {
input::click(x, y, button)
}
fn simulate_type(&self, text: &str) -> Result<(), ComputerError> {
input::type_text(text)
}
fn simulate_key(&self, modifiers: &[Modifier], key: &str) -> Result<(), ComputerError> {
input::press_key(modifiers, key)
}
fn simulate_scroll(&self, dx: i32, dy: i32) -> Result<(), ComputerError> {
input::scroll(dx, dy)
}
fn cursor_position(&self) -> Result<(i32, i32), ComputerError> {
input::cursor_position()
}
fn list_displays(&self) -> Result<Vec<DisplayInfo>, ComputerError> {
capture::list_displays()
}
fn display_block(&self) -> Option<ComputerError> {
#[cfg(target_os = "macos")]
{
macos::session_display_block()
}
#[cfg(not(target_os = "macos"))]
{
None
}
}
fn frontmost(&self) -> FrontmostInfo {
$front()
}
fn inspect_accessibility(
&self,
app_name: Option<&str>,
) -> Result<String, ComputerError> {
$inspect(app_name)
}
fn list_apps(&self, all: bool) -> Result<Vec<AppInfo>, ComputerError> {
$list_apps(all)
}
fn list_windows(&self) -> Result<WindowList, ComputerError> {
$list_windows()
}
fn launch(&self, app: &str) -> Result<LaunchResult, ComputerError> {
$launch(app)
}
fn activate(
&self,
app: &str,
window: Option<&str>,
) -> Result<LaunchResult, ComputerError> {
$activate(app, window)
}
fn quit(&self, app: &str, force: bool) -> Result<QuitResult, ComputerError> {
$quit(app, force)
}
fn doctor(
&self,
options: crate::doctor::DoctorOptions,
) -> Result<DoctorReport, ComputerError> {
$doctor(options)
}
fn tree(&self, query: &TreeQuery) -> Result<TreeReport, ComputerError> {
$tree(query)
}
fn end_session(&self, session: Option<&str>) -> Result<Vec<i32>, ComputerError> {
$end_session(session)
}
#[cfg(target_os = "macos")]
fn menu_list(&self, app: &str, depth: u32) -> Result<MenuListReport, ComputerError> {
macos::menu::list(app, depth)
}
#[cfg(target_os = "macos")]
fn menu_click(
&self,
app: &str,
path: &crate::menu_path::MenuPath,
) -> Result<MenuClickReport, ComputerError> {
macos::menu::click(app, path)
}
#[cfg(target_os = "macos")]
fn apple_event(
&self,
app: &str,
script: &str,
language: AppleEventLanguage,
) -> Result<AppleEventReport, ComputerError> {
let _ = app;
let outcome = macos::apple::run(script, language)?;
Ok(AppleEventReport {
script_bytes: outcome.script_bytes,
script_hash: outcome.script_hash,
stdout: outcome.stdout,
stderr: outcome.stderr,
exit_status: outcome.exit_status,
})
}
#[cfg(target_os = "macos")]
fn dock_list(&self) -> Result<DockListReport, ComputerError> {
macos::dock::list()
}
#[cfg(target_os = "macos")]
fn dock_click(&self, title: &str) -> Result<DockClickReport, ComputerError> {
macos::dock::click(title)
}
#[cfg(target_os = "macos")]
fn spaces(&self) -> Result<crate::window_groups::SpacesReport, ComputerError> {
Ok(macos::spaces::report())
}
#[cfg(target_os = "macos")]
fn marks_for_tree(
&self,
query: &TreeQuery,
map: &crate::image_util::ScreenshotMap,
) -> Result<Vec<crate::vision::MarkBox>, ComputerError> {
macos::marks_for_tree(query, map)
}
#[cfg(target_os = "macos")]
fn recognize_text(
&self,
image: &image::RgbaImage,
map: &crate::image_util::ScreenshotMap,
) -> Result<crate::vision::OcrResult, ComputerError> {
macos::recognize_text(image, map)
}
#[cfg(target_os = "macos")]
fn secure_ocr_frames(
&self,
query: &TreeQuery,
map: &crate::image_util::ScreenshotMap,
) -> Result<Vec<crate::vision::ocr::ImageRect>, ComputerError> {
macos::secure_ocr_frames(query, map)
}
#[cfg(target_os = "windows")]
fn element_action(
&self,
request: &ElementRequest,
) -> Result<ElementOutcome, ElementError> {
windows::element_action(request)
}
#[cfg(target_os = "linux")]
fn element_action(
&self,
request: &ElementRequest,
) -> Result<ElementOutcome, ElementError> {
linux::element_action(request)
}
#[cfg(target_os = "linux")]
fn perform_input(
&self,
job: &crate::tier::InputJob,
) -> Result<crate::tier::InputReport, ComputerError> {
linux::perform_input(job)
}
#[cfg(target_os = "windows")]
fn capture_window(&self, target: &WindowTarget) -> Result<RawCapture, ComputerError> {
windows::capture_window(target)
}
#[cfg(target_os = "windows")]
fn marks_for_tree(
&self,
query: &TreeQuery,
map: &crate::image_util::ScreenshotMap,
) -> Result<Vec<crate::vision::MarkBox>, ComputerError> {
windows::marks_for_tree(query, map)
}
#[cfg(target_os = "windows")]
fn recognize_text(
&self,
image: &image::RgbaImage,
map: &crate::image_util::ScreenshotMap,
) -> Result<crate::vision::OcrResult, ComputerError> {
windows::recognize_text(image, map)
}
#[cfg(target_os = "windows")]
fn secure_ocr_frames(
&self,
query: &TreeQuery,
map: &crate::image_util::ScreenshotMap,
) -> Result<Vec<crate::vision::ocr::ImageRect>, ComputerError> {
windows::secure_ocr_frames(query, map)
}
}
};
}
#[cfg(target_os = "macos")]
impl_platform_backend!(
macos::MacOsDesktopBackend,
macos::inspect_accessibility,
macos::frontmost,
macos::doctor,
macos::list_apps,
macos::list_windows,
macos::launch,
macos::activate,
macos::quit,
macos::tree,
macos::end_session
);
#[cfg(target_os = "linux")]
impl_platform_backend!(
linux::LinuxDesktopBackend,
linux::inspect_accessibility,
linux::frontmost,
linux::doctor,
linux::list_apps,
linux::list_windows,
linux::launch,
linux::activate,
linux::quit,
linux::tree,
linux::end_session
);
#[cfg(target_os = "windows")]
impl_platform_backend!(
windows::WindowsDesktopBackend,
windows::inspect_accessibility,
windows::frontmost,
windows::doctor,
windows::list_apps,
windows::list_windows,
windows::launch,
windows::activate,
windows::quit,
windows::tree,
windows::end_session
);
#[cfg(target_os = "linux")]
pub use linux::LinuxDesktopBackend;
#[cfg(target_os = "macos")]
pub use macos::MacOsDesktopBackend;
#[cfg(target_os = "windows")]
pub use windows::WindowsDesktopBackend;
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn create_desktop_backend_is_a_trait_object() {
let backend = create_desktop_backend();
let _backend: &dyn DesktopBackend = backend.as_ref();
}
}