computer_use/platform/linux/
mod.rs

1mod a11y;
2mod apps;
3mod input;
4mod keys;
5mod screenshot;
6mod session;
7
8use std::ops::Range;
9use std::sync::Arc;
10
11use zbus::blocking::Connection;
12
13use self::screenshot::Shot;
14use self::session::SessionEnv;
15
16use crate::app::AppInfo;
17use crate::backend::{Backend, Capture, CaptureRequest, InputTarget, MouseButton};
18use crate::error::Result;
19use crate::geometry::Point;
20use crate::keys::KeyChord;
21use crate::state::WindowInfo;
22
23/// A live AT-SPI accessible object: the owning app's bus name plus the object path.
24///
25/// Proxies for the object's interfaces are built on demand from this pair.
26#[derive(Debug, Clone, PartialEq, Eq, Hash)]
27pub struct LinuxElement {
28    bus: Arc<str>,
29    path: Arc<str>,
30}
31
32impl LinuxElement {
33    pub(crate) fn new(bus: &str, path: &str) -> Self {
34        Self { bus: bus.into(), path: path.into() }
35    }
36
37    /// The bus name of the app that owns the object.
38    #[must_use]
39    pub fn bus_name(&self) -> &str {
40        &self.bus
41    }
42
43    /// The object path of the accessible.
44    #[must_use]
45    pub fn object_path(&self) -> &str {
46        &self.path
47    }
48}
49
50/// The Linux implementation of [`Backend`], built on AT-SPI.
51#[derive(Debug, Clone)]
52pub struct LinuxBackend {
53    conn: Connection,
54    session: Arc<SessionEnv>,
55}
56
57impl LinuxBackend {
58    /// Connects to the AT-SPI accessibility bus.
59    ///
60    /// Session variables this process lacks (`DISPLAY`, `DBUS_SESSION_BUS_ADDRESS`,
61    /// `XAUTHORITY`, `XDG_RUNTIME_DIR`, `AT_SPI_BUS_ADDRESS`, ...) are discovered from the
62    /// user's running desktop processes, so the backend also works from services, cron jobs and SSH sessions.
63    ///
64    /// Also turns on the desktop's accessibility flag (`org.a11y.Status.IsEnabled`),
65    /// which some apps need in order to expose their UI. Apps of that kind that were
66    /// already running before the flag was on must be restarted to show up.
67    ///
68    /// # Errors
69    ///
70    /// Fails if there is no D-Bus session or the accessibility bus is not running.
71    pub fn new() -> Result<Self> {
72        let session = SessionEnv::discover();
73        a11y::enable_accessibility(&session);
74        Ok(Self { conn: a11y::connect(&session)?, session: Arc::new(session) })
75    }
76}
77
78impl Backend for LinuxBackend {
79    type Handle = LinuxElement;
80
81    fn list_apps(&self) -> Result<Vec<AppInfo>> {
82        apps::list_apps(&self.conn)
83    }
84
85    fn launch_app(&self, app: &AppInfo) -> Result<AppInfo> {
86        apps::launch_app(&self.conn, &self.session, app)
87    }
88
89    fn list_windows(&self, app: &AppInfo) -> Result<Vec<WindowInfo>> {
90        let root = apps::find_running(&self.conn, app)?;
91        a11y::list_windows(&self.conn, &root)
92    }
93
94    fn capture(&self, app: &AppInfo, request: &CaptureRequest) -> Result<Capture<Self::Handle>> {
95        let root = apps::find_running(&self.conn, app)?;
96        let window = a11y::main_window(&self.conn, &root, &app.name, request.window)?;
97        let mut capture = Capture::new(window.info);
98        if request.include_tree {
99            let walk = a11y::walk(&self.conn, &window.element, request.max_nodes, request.max_depth)?;
100            capture.nodes = walk.nodes;
101            capture.truncated = walk.truncated;
102            capture.focused = walk.focused;
103            capture.selected_text = walk.selected_text;
104        }
105        if request.include_screenshot {
106            match screenshot::capture(&self.session, capture.window.frame)? {
107                Shot::Captured(frame, image) => {
108                    capture.window.frame = frame;
109                    capture.screenshot = Some(image);
110                }
111                Shot::Omitted(note) => capture.notes.push(note),
112            }
113        }
114        Ok(capture)
115    }
116
117    fn perform_action(&self, element: &Self::Handle, action: &str) -> Result<()> {
118        a11y::perform_action(&self.conn, element, action)
119    }
120
121    fn set_value(&self, element: &Self::Handle, value: &str) -> Result<()> {
122        a11y::set_value(&self.conn, element, value)
123    }
124
125    fn value(&self, element: &Self::Handle) -> Result<Option<String>> {
126        a11y::value(&self.conn, element)
127    }
128
129    fn select_text(&self, element: &Self::Handle, value: &str, range: Range<usize>) -> Result<()> {
130        a11y::select_text(&self.conn, element, value, range)
131    }
132
133    fn click(&self, _target: InputTarget<'_>, point: Point, button: MouseButton, count: u32) -> Result<()> {
134        input::click(&self.conn, point, button, count)
135    }
136
137    fn scroll(&self, _target: InputTarget<'_>, point: Point, dx: f64, dy: f64) -> Result<()> {
138        input::scroll(&self.conn, point, dx, dy)
139    }
140
141    fn drag(&self, _target: InputTarget<'_>, from: Point, to: Point) -> Result<()> {
142        input::drag(&self.conn, from, to)
143    }
144
145    fn type_text(&self, _target: InputTarget<'_>, text: &str) -> Result<()> {
146        input::type_text(&self.conn, text)
147    }
148
149    fn press_key(&self, _target: InputTarget<'_>, chord: &KeyChord) -> Result<()> {
150        input::press_key(&self.conn, chord)
151    }
152}
computer_use/platform/linux/mod.rs

computer_use/platform/linux/
mod.rs
