computer_use/platform/linux/
mod.rs

1mod a11y;
2mod apps;
3mod input;
4mod keys;
5mod screenshot;
6mod session;
7
8use std::ops::Range;
9use std::sync::Arc;
10
11use zbus::blocking::Connection;
12
13use self::screenshot::Shot;
14use self::session::SessionEnv;
15
16use crate::app::AppInfo;
17use crate::backend::{Backend, Capture, CaptureRequest, InputTarget, MouseButton};
18use crate::error::Result;
19use crate::geometry::Point;
20use crate::keys::KeyChord;
21
22/// A live AT-SPI accessible object: the owning app's bus name plus the object path.
23///
24/// Proxies for the object's interfaces are built on demand from this pair.
25#[derive(Debug, Clone, PartialEq, Eq, Hash)]
26pub struct LinuxElement {
27    bus: Arc<str>,
28    path: Arc<str>,
29}
30
31impl LinuxElement {
32    pub(crate) fn new(bus: &str, path: &str) -> Self {
33        Self { bus: bus.into(), path: path.into() }
34    }
35
36    /// The bus name of the app that owns the object.
37    #[must_use]
38    pub fn bus_name(&self) -> &str {
39        &self.bus
40    }
41
42    /// The object path of the accessible.
43    #[must_use]
44    pub fn object_path(&self) -> &str {
45        &self.path
46    }
47}
48
49/// The Linux implementation of [`Backend`], built on AT-SPI.
50#[derive(Debug, Clone)]
51pub struct LinuxBackend {
52    conn: Connection,
53    session: Arc<SessionEnv>,
54}
55
56impl LinuxBackend {
57    /// Connects to the AT-SPI accessibility bus.
58    ///
59    /// Session variables this process lacks (`DISPLAY`, `DBUS_SESSION_BUS_ADDRESS`,
60    /// `XAUTHORITY`, `XDG_RUNTIME_DIR`, `AT_SPI_BUS_ADDRESS`, ...) are discovered from the
61    /// user's running desktop processes, so the backend also works from services, cron jobs and SSH sessions.
62    ///
63    /// # Errors
64    ///
65    /// Fails if there is no D-Bus session or the accessibility bus is not running.
66    pub fn new() -> Result<Self> {
67        let session = SessionEnv::discover();
68        Ok(Self { conn: a11y::connect(&session)?, session: Arc::new(session) })
69    }
70}
71
72impl Backend for LinuxBackend {
73    type Handle = LinuxElement;
74
75    fn list_apps(&self) -> Result<Vec<AppInfo>> {
76        apps::list_apps(&self.conn)
77    }
78
79    fn launch_app(&self, app: &AppInfo) -> Result<AppInfo> {
80        apps::launch_app(&self.conn, &self.session, app)
81    }
82
83    fn capture(&self, app: &AppInfo, request: &CaptureRequest) -> Result<Capture<Self::Handle>> {
84        let root = apps::find_running(&self.conn, app)?;
85        let window = a11y::main_window(&self.conn, &root, &app.name)?;
86        let mut capture = Capture::new(window.info);
87        if request.include_tree {
88            let walk = a11y::walk(&self.conn, &window.element, request.max_nodes, request.max_depth)?;
89            capture.nodes = walk.nodes;
90            capture.truncated = walk.truncated;
91            capture.focused = walk.focused;
92            capture.selected_text = walk.selected_text;
93        }
94        if request.include_screenshot {
95            match screenshot::capture(&self.session, capture.window.frame)? {
96                Shot::Captured(frame, image) => {
97                    capture.window.frame = frame;
98                    capture.screenshot = Some(image);
99                }
100                Shot::Omitted(note) => capture.notes.push(note),
101            }
102        }
103        Ok(capture)
104    }
105
106    fn perform_action(&self, element: &Self::Handle, action: &str) -> Result<()> {
107        a11y::perform_action(&self.conn, element, action)
108    }
109
110    fn set_value(&self, element: &Self::Handle, value: &str) -> Result<()> {
111        a11y::set_value(&self.conn, element, value)
112    }
113
114    fn value(&self, element: &Self::Handle) -> Result<Option<String>> {
115        a11y::value(&self.conn, element)
116    }
117
118    fn select_text(&self, element: &Self::Handle, value: &str, range: Range<usize>) -> Result<()> {
119        a11y::select_text(&self.conn, element, value, range)
120    }
121
122    fn click(&self, _target: InputTarget<'_>, point: Point, button: MouseButton, count: u32) -> Result<()> {
123        input::click(&self.conn, point, button, count)
124    }
125
126    fn scroll(&self, _target: InputTarget<'_>, point: Point, dx: f64, dy: f64) -> Result<()> {
127        input::scroll(&self.conn, point, dx, dy)
128    }
129
130    fn drag(&self, _target: InputTarget<'_>, from: Point, to: Point) -> Result<()> {
131        input::drag(&self.conn, from, to)
132    }
133
134    fn type_text(&self, _target: InputTarget<'_>, text: &str) -> Result<()> {
135        input::type_text(&self.conn, text)
136    }
137
138    fn press_key(&self, _target: InputTarget<'_>, chord: &KeyChord) -> Result<()> {
139        input::press_key(&self.conn, chord)
140    }
141}
computer_use/platform/linux/mod.rs

computer_use/platform/linux/
mod.rs
