computer_use/platform/linux/
mod.rs

1mod a11y;
2mod apps;
3mod input;
4mod keys;
5mod screenshot;
6mod session;
7
8use std::ops::Range;
9use std::sync::Arc;
10
11use zbus::blocking::Connection;
12
13use self::screenshot::Shot;
14use self::session::SessionEnv;
15
16use crate::app::AppInfo;
17use crate::backend::{Backend, Capture, CaptureRequest, InputTarget, MouseButton};
18use crate::error::{Error, Result};
19use crate::geometry::Point;
20use crate::keys::KeyChord;
21use crate::state::WindowInfo;
22
23/// A live AT-SPI accessible object: the owning app's bus name plus the object path.
24///
25/// Proxies for the object's interfaces are built on demand from this pair.
26#[derive(Debug, Clone, PartialEq, Eq, Hash)]
27pub struct LinuxElement {
28    bus: Arc<str>,
29    path: Arc<str>,
30}
31
32impl LinuxElement {
33    pub(crate) fn new(bus: &str, path: &str) -> Self {
34        Self { bus: bus.into(), path: path.into() }
35    }
36
37    /// The bus name of the app that owns the object.
38    #[must_use]
39    pub fn bus_name(&self) -> &str {
40        &self.bus
41    }
42
43    /// The object path of the accessible.
44    #[must_use]
45    pub fn object_path(&self) -> &str {
46        &self.path
47    }
48}
49
50/// The Linux implementation of [`Backend`], built on AT-SPI.
51#[derive(Debug, Clone)]
52pub struct LinuxBackend {
53    conn: Connection,
54    session: Arc<SessionEnv>,
55}
56
57impl LinuxBackend {
58    /// Connects to the AT-SPI accessibility bus.
59    ///
60    /// Session variables this process lacks (`DISPLAY`, `DBUS_SESSION_BUS_ADDRESS`,
61    /// `XAUTHORITY`, `XDG_RUNTIME_DIR`, `AT_SPI_BUS_ADDRESS`, ...) are discovered from the
62    /// user's running desktop processes, so the backend also works from services, cron jobs and SSH sessions.
63    ///
64    /// Also turns on the desktop's accessibility flag (`org.a11y.Status.IsEnabled`),
65    /// which some apps need in order to expose their UI. Apps of that kind that were
66    /// already running before the flag was on must be restarted to show up.
67    ///
68    /// # Errors
69    ///
70    /// Fails if there is no D-Bus session or the accessibility bus is not running.
71    pub fn new() -> Result<Self> {
72        let session = SessionEnv::discover();
73        a11y::enable_accessibility(&session);
74        Ok(Self { conn: a11y::connect(&session)?, session: Arc::new(session) })
75    }
76}
77
78impl LinuxBackend {
79    /// Fails when synthesized input cannot reach the target app.
80    ///
81    /// Keyboard and pointer events are injected through the X server, so under Wayland they only
82    /// reach X11 (`XWayland`) apps. Reporting success for a native Wayland app would hide that
83    /// nothing happened.
84    fn ensure_input_reaches(&self, target: &InputTarget<'_>) -> Result<()> {
85        if self.session.get("WAYLAND_DISPLAY").is_none() {
86            return Ok(());
87        }
88        let Some(pid) = target.app.pid else { return Ok(()) };
89        if screenshot::has_x11_window(&self.session, pid) == Some(false) {
90            return Err(Error::Unsupported(format!(
91                "{} uses a native Wayland window, which synthesized keyboard and pointer input cannot reach; \
92                 use accessibility actions instead (click an element by index, set a field's value, or perform an element action)",
93                target.app.name
94            )));
95        }
96        Ok(())
97    }
98}
99
100impl Backend for LinuxBackend {
101    type Handle = LinuxElement;
102
103    fn list_apps(&self) -> Result<Vec<AppInfo>> {
104        apps::list_apps(&self.conn)
105    }
106
107    fn launch_app(&self, app: &AppInfo) -> Result<AppInfo> {
108        apps::launch_app(&self.conn, &self.session, app)
109    }
110
111    fn list_windows(&self, app: &AppInfo) -> Result<Vec<WindowInfo>> {
112        let root = apps::find_running(&self.conn, app)?;
113        a11y::list_windows(&self.conn, &root)
114    }
115
116    fn capture(&self, app: &AppInfo, request: &CaptureRequest) -> Result<Capture<Self::Handle>> {
117        let root = apps::find_running(&self.conn, app)?;
118        let window = a11y::main_window(&self.conn, &root, &app.name, request.window)?;
119        let mut capture = Capture::new(window.info);
120        if request.include_tree {
121            let walk = a11y::walk(&self.conn, &window.element, request.max_nodes, request.max_depth)?;
122            capture.nodes = walk.nodes;
123            capture.truncated = walk.truncated;
124            capture.focused = walk.focused;
125            capture.selected_text = walk.selected_text;
126        }
127        if request.include_screenshot {
128            let title = capture.window.title.as_deref();
129            match screenshot::capture(&self.session, capture.window.frame, app.pid, title)? {
130                Shot::Captured(frame, image) => {
131                    capture.window.frame = frame;
132                    capture.screenshot = Some(image);
133                }
134                Shot::Omitted(note) => capture.notes.push(note),
135            }
136        }
137        Ok(capture)
138    }
139
140    fn perform_action(&self, element: &Self::Handle, action: &str) -> Result<()> {
141        a11y::perform_action(&self.conn, element, action)
142    }
143
144    fn set_value(&self, element: &Self::Handle, value: &str) -> Result<()> {
145        a11y::set_value(&self.conn, element, value)
146    }
147
148    fn value(&self, element: &Self::Handle) -> Result<Option<String>> {
149        a11y::value(&self.conn, element)
150    }
151
152    fn select_text(&self, element: &Self::Handle, value: &str, range: Range<usize>) -> Result<()> {
153        a11y::select_text(&self.conn, element, value, range)
154    }
155
156    fn click(&self, target: InputTarget<'_>, point: Point, button: MouseButton, count: u32) -> Result<()> {
157        self.ensure_input_reaches(&target)?;
158        input::click(&self.conn, point, button, count)
159    }
160
161    fn scroll(&self, target: InputTarget<'_>, point: Point, dx: f64, dy: f64) -> Result<()> {
162        self.ensure_input_reaches(&target)?;
163        input::scroll(&self.conn, point, dx, dy)
164    }
165
166    fn drag(&self, target: InputTarget<'_>, from: Point, to: Point) -> Result<()> {
167        self.ensure_input_reaches(&target)?;
168        input::drag(&self.conn, from, to)
169    }
170
171    fn type_text(&self, target: InputTarget<'_>, text: &str) -> Result<()> {
172        self.ensure_input_reaches(&target)?;
173        input::type_text(&self.conn, text)
174    }
175
176    fn press_key(&self, target: InputTarget<'_>, chord: &KeyChord) -> Result<()> {
177        self.ensure_input_reaches(&target)?;
178        input::press_key(&self.conn, chord)
179    }
180}
computer_use/platform/linux/mod.rs

computer_use/platform/linux/
mod.rs
