computer_use/platform/linux/
mod.rs

1mod a11y;
2mod apps;
3mod input;
4mod keys;
5mod screenshot;
6mod session;
7
8use std::ops::Range;
9use std::sync::Arc;
10
11use zbus::blocking::Connection;
12
13use self::screenshot::Shot;
14use self::session::SessionEnv;
15
16use crate::app::AppInfo;
17use crate::backend::{Backend, Capture, CaptureRequest, InputTarget, MouseButton};
18use crate::error::Result;
19use crate::geometry::Point;
20use crate::keys::KeyChord;
21use crate::state::WindowInfo;
22
23/// A live AT-SPI accessible object: the owning app's bus name plus the object path.
24///
25/// Proxies for the object's interfaces are built on demand from this pair.
26#[derive(Debug, Clone, PartialEq, Eq, Hash)]
27pub struct LinuxElement {
28    bus: Arc<str>,
29    path: Arc<str>,
30}
31
32impl LinuxElement {
33    pub(crate) fn new(bus: &str, path: &str) -> Self {
34        Self { bus: bus.into(), path: path.into() }
35    }
36
37    /// The bus name of the app that owns the object.
38    #[must_use]
39    pub fn bus_name(&self) -> &str {
40        &self.bus
41    }
42
43    /// The object path of the accessible.
44    #[must_use]
45    pub fn object_path(&self) -> &str {
46        &self.path
47    }
48}
49
50/// The Linux implementation of [`Backend`], built on AT-SPI.
51#[derive(Debug, Clone)]
52pub struct LinuxBackend {
53    conn: Connection,
54    session: Arc<SessionEnv>,
55}
56
57impl LinuxBackend {
58    /// Connects to the AT-SPI accessibility bus.
59    ///
60    /// Session variables this process lacks (`DISPLAY`, `DBUS_SESSION_BUS_ADDRESS`,
61    /// `XAUTHORITY`, `XDG_RUNTIME_DIR`, `AT_SPI_BUS_ADDRESS`, ...) are discovered from the
62    /// user's running desktop processes, so the backend also works from services, cron jobs and SSH sessions.
63    ///
64    /// # Errors
65    ///
66    /// Fails if there is no D-Bus session or the accessibility bus is not running.
67    pub fn new() -> Result<Self> {
68        let session = SessionEnv::discover();
69        Ok(Self { conn: a11y::connect(&session)?, session: Arc::new(session) })
70    }
71}
72
73impl Backend for LinuxBackend {
74    type Handle = LinuxElement;
75
76    fn list_apps(&self) -> Result<Vec<AppInfo>> {
77        apps::list_apps(&self.conn)
78    }
79
80    fn launch_app(&self, app: &AppInfo) -> Result<AppInfo> {
81        apps::launch_app(&self.conn, &self.session, app)
82    }
83
84    fn list_windows(&self, app: &AppInfo) -> Result<Vec<WindowInfo>> {
85        let root = apps::find_running(&self.conn, app)?;
86        a11y::list_windows(&self.conn, &root)
87    }
88
89    fn capture(&self, app: &AppInfo, request: &CaptureRequest) -> Result<Capture<Self::Handle>> {
90        let root = apps::find_running(&self.conn, app)?;
91        let window = a11y::main_window(&self.conn, &root, &app.name, request.window)?;
92        let mut capture = Capture::new(window.info);
93        if request.include_tree {
94            let walk = a11y::walk(&self.conn, &window.element, request.max_nodes, request.max_depth)?;
95            capture.nodes = walk.nodes;
96            capture.truncated = walk.truncated;
97            capture.focused = walk.focused;
98            capture.selected_text = walk.selected_text;
99        }
100        if request.include_screenshot {
101            match screenshot::capture(&self.session, capture.window.frame)? {
102                Shot::Captured(frame, image) => {
103                    capture.window.frame = frame;
104                    capture.screenshot = Some(image);
105                }
106                Shot::Omitted(note) => capture.notes.push(note),
107            }
108        }
109        Ok(capture)
110    }
111
112    fn perform_action(&self, element: &Self::Handle, action: &str) -> Result<()> {
113        a11y::perform_action(&self.conn, element, action)
114    }
115
116    fn set_value(&self, element: &Self::Handle, value: &str) -> Result<()> {
117        a11y::set_value(&self.conn, element, value)
118    }
119
120    fn value(&self, element: &Self::Handle) -> Result<Option<String>> {
121        a11y::value(&self.conn, element)
122    }
123
124    fn select_text(&self, element: &Self::Handle, value: &str, range: Range<usize>) -> Result<()> {
125        a11y::select_text(&self.conn, element, value, range)
126    }
127
128    fn click(&self, _target: InputTarget<'_>, point: Point, button: MouseButton, count: u32) -> Result<()> {
129        input::click(&self.conn, point, button, count)
130    }
131
132    fn scroll(&self, _target: InputTarget<'_>, point: Point, dx: f64, dy: f64) -> Result<()> {
133        input::scroll(&self.conn, point, dx, dy)
134    }
135
136    fn drag(&self, _target: InputTarget<'_>, from: Point, to: Point) -> Result<()> {
137        input::drag(&self.conn, from, to)
138    }
139
140    fn type_text(&self, _target: InputTarget<'_>, text: &str) -> Result<()> {
141        input::type_text(&self.conn, text)
142    }
143
144    fn press_key(&self, _target: InputTarget<'_>, chord: &KeyChord) -> Result<()> {
145        input::press_key(&self.conn, chord)
146    }
147}
computer_use/platform/linux/mod.rs

computer_use/platform/linux/
mod.rs
