#![cfg(target_os = "linux")]
#![allow(
clippy::cast_lossless,
clippy::cast_possible_truncation,
clippy::cast_possible_wrap,
clippy::cast_precision_loss,
clippy::cast_sign_loss,
clippy::module_name_repetitions,
clippy::semicolon_if_nothing_returned
)]
use super::core::{
self, AppInfo, Backend, Capture, CaptureBudget, ElementAct, Impact, Locator, Modifier,
MouseButton, Observation, RawInput, ScrollDirection, SurfaceGeometry, TargetSpec, Truncation,
UiNode, UnreadWindows, WindowInfo, WindowKey, WindowListing,
};
use crate::util::{UnwrapPoison, with_block_in_place};
use anyhow::Context as _;
use anyhow::anyhow;
use async_recursion::async_recursion;
use atspi::proxy::accessible::{AccessibleProxy, ObjectRefExt};
use atspi::proxy::bus::BusProxy;
use atspi::proxy::proxy_ext::{Proxies, ProxyExt};
use atspi::{CoordType, ObjectRef, ObjectRefOwned, Role, State as AtspiState};
use enigo::{Axis, Button, Coordinate, Direction, Enigo, Key, Keyboard, Mouse, Settings};
use futures_util::StreamExt as _;
use std::collections::HashMap;
use std::sync::Mutex;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Duration;
use tokio::io::AsyncReadExt as _;
use tracing::{debug, warn};
use x11rb::connection::{Connection as _, RequestConnection as _};
use x11rb::protocol::{randr, xproto};
use x11rb::rust_connection::RustConnection;
pub(crate) static LINUX_BACKEND: LinuxBackend = LinuxBackend {
connection: tokio::sync::Mutex::const_new(None),
};
pub(crate) struct LinuxBackend {
connection: tokio::sync::Mutex<Option<zbus::Connection>>,
}
const MAX_DEPTH: usize = 25;
const VALUE_READ_CHARS: i32 = core::RENDER_VALUE_CHARS as i32 + 1;
const INTERP_STEPS: usize = 12;
const CONN_DEGRADED_MSG: &str =
"AT-SPI2 accessibility bus unavailable — is at-spi2 installed and the session bus running?";
const A11Y_REGISTRY: &str = "org.a11y.atspi.Registry";
fn ensure_link(conn: &zbus::Connection) -> Result<(), anyhow::Error> {
if conn.is_closed() {
return Err(core::taxonomy_error(
core::ERR_DEGRADED,
"the accessibility connection was lost while the call ran, so what it did read is \
incomplete — the next call reconnects; retry",
));
}
Ok(())
}
const TEXT_VALUE_ROLES: &[&str] = &[
"text",
"textfield",
"entry",
"paragraph",
"label",
"static",
"tooltip",
];
const KNOWN_KEYS: &str = "return, enter, tab, space, delete/backspace, forwarddelete, escape, \
home, end, pageup, pagedown, up, down, left, right, f1..f12, capslock, insert, minus, equal, \
comma, period, slash, semicolon, quote, backslash, backtick, leftbracket, rightbracket, a-z, 0-9";
impl LinuxBackend {
async fn conn(&self) -> Result<zbus::Connection, anyhow::Error> {
let mut held = self.connection.lock().await;
if let Some(conn) = held.as_ref() {
if conn.is_closed() {
warn!("at-spi connection closed — rebuilding it for this call");
*held = None;
} else {
return Ok(conn.clone());
}
}
match tokio::time::timeout(core::EXCHANGE_DEADLINE, open_a11y_bus()).await {
Ok(Ok(conn)) => {
*held = Some(conn.clone());
Ok(conn)
}
Ok(Err(e)) => {
debug!(error = %e, "atspi connection failed");
Err(core::taxonomy_error(core::ERR_DEGRADED, CONN_DEGRADED_MSG))
}
Err(_) => {
debug!("atspi connection attempt timed out");
Err(core::taxonomy_error(
core::ERR_DEGRADED,
format!(
"{CONN_DEGRADED_MSG} (connecting did not answer within {}s — the next call \
retries)",
core::EXCHANGE_DEADLINE.as_secs()
),
))
}
}
}
async fn surface_geometry_with(
&self,
target: &TargetSpec,
budget: &mut core::Budget,
) -> Result<SurfaceGeometry, anyhow::Error> {
match target {
TargetSpec::Screen => {
if wayland() {
match *SCREEN_DIMS.lock().unwrap_poison() {
Some((w, h)) => Ok(SurfaceGeometry {
x: 0.0,
y: 0.0,
width: f64::from(w),
height: f64::from(h),
}),
None => Err(core::taxonomy_error(
core::ERR_DEGRADED,
if super::capture_known_available() {
"no cached Wayland screen size — capture the screen first; \
accessibility observe/act remain available"
} else {
"no cached Wayland screen size — accessibility observe/act remain \
available"
},
)),
}
} else {
with_block_in_place(x11_screen_geometry)
}
}
other => {
let conn = self.conn().await?;
let window = resolve_surface(&conn, other, budget).await?;
let scale = a11y_pixel_scale(&conn, pixel_space(), budget).await?;
Ok(core::scale_geometry(window.surface, scale))
}
}
}
async fn capture_screen_channel(
&self,
budget: &CaptureBudget,
) -> Result<(Capture, SurfaceGeometry), anyhow::Error> {
if wayland() {
let cap = portal_screenshot(budget).await?;
let geo = SurfaceGeometry {
x: 0.0,
y: 0.0,
width: f64::from(cap.width),
height: f64::from(cap.height),
};
*SCREEN_DIMS.lock().unwrap_poison() = Some((cap.width, cap.height));
Ok((cap, geo))
} else {
with_block_in_place(|| x11_capture_screen(budget))
}
}
}
async fn open_a11y_bus() -> anyhow::Result<zbus::Connection> {
let session = zbus::Connection::session()
.await
.context("connect to the session bus")?;
let bus = BusProxy::new(&session)
.await
.context("open the a11y bus address proxy")?;
let address: zbus::Address = bus
.get_address()
.await
.context("read the a11y bus address")?
.parse()
.context("parse the a11y bus address")?;
zbus::connection::Builder::address(address)
.context("prepare the a11y connection")?
.method_timeout(core::EXCHANGE_DEADLINE)
.build()
.await
.context("connect to the a11y bus")
}
async fn registry_root(conn: &zbus::Connection) -> anyhow::Result<AccessibleProxy<'_>> {
AccessibleProxy::builder(conn)
.destination(A11Y_REGISTRY)
.map_err(|e| dbus_err(e, "access registry root"))?
.cache_properties(zbus::proxy::CacheProperties::No)
.build()
.await
.map_err(|e| dbus_err(e, "access registry root"))
}
#[derive(Clone, Copy, PartialEq, Eq)]
enum DisplayServer {
X11,
Wayland,
}
static DISPLAY_SERVER: std::sync::OnceLock<DisplayServer> = std::sync::OnceLock::new();
const LOGIND_DEST: &str = "org.freedesktop.login1";
const LOGIND_PATH: &str = "/org/freedesktop/login1";
const STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(3);
static SYSTEM_BUS: tokio::sync::Mutex<Option<zbus::Connection>> =
tokio::sync::Mutex::const_new(None);
async fn system_bus() -> Option<zbus::Connection> {
let mut held = SYSTEM_BUS.lock().await;
if let Some(conn) = held.as_ref() {
if !conn.is_closed() {
return Some(conn.clone());
}
*held = None;
}
let conn = zbus::Connection::system().await.ok()?;
*held = Some(conn.clone());
Some(conn)
}
fn display_server() -> Option<DisplayServer> {
if let Some(known) = DISPLAY_SERVER.get() {
return Some(*known);
}
let probed = with_block_in_place(probe_display_server)?;
let _ = DISPLAY_SERVER.set(probed);
Some(probed)
}
fn probe_display_server() -> Option<DisplayServer> {
if std::env::var_os("DISPLAY").is_some()
&& let Ok((conn, _screen)) = x11rb::connect(None)
{
let xwayland = xproto::query_extension(&conn, b"XWAYLAND")
.ok()
.and_then(|cookie| cookie.reply().ok())
.is_some_and(|reply| reply.present);
return Some(if xwayland {
DisplayServer::Wayland
} else {
DisplayServer::X11
});
}
std::env::var_os("WAYLAND_DISPLAY").map(|_| DisplayServer::Wayland)
}
fn wayland() -> bool {
display_server() == Some(DisplayServer::Wayland)
}
async fn locked_reason() -> Option<String> {
let locked = tokio::time::timeout(STATE_PROBE_TIMEOUT, async {
let conn = system_bus().await?;
let manager = zbus::Proxy::new(
&conn,
LOGIND_DEST,
LOGIND_PATH,
"org.freedesktop.login1.Manager",
)
.await
.ok()?;
let (session,): (zbus::zvariant::OwnedObjectPath,) =
manager.call("GetSession", &("self",)).await.ok()?;
let proxy = zbus::Proxy::new(
&conn,
LOGIND_DEST,
session.as_str(),
"org.freedesktop.login1.Session",
)
.await
.ok()?;
proxy.get_property::<bool>("LockedHint").await.ok()
})
.await
.ok()
.flatten()?;
locked.then(|| "the session's lock screen is active".to_string())
}
fn role_name(role: Role) -> String {
match role {
Role::Button => "pushbutton".to_string(),
Role::Entry => "textfield".to_string(),
Role::PageTab => "tab".to_string(),
Role::PageTabList => "tabgroup".to_string(),
Role::ListItem => "listitem".to_string(),
Role::ScrollBar => "scrollbar".to_string(),
other => format!("{other:?}").to_lowercase(),
}
}
fn is_text_role(role: &str) -> bool {
TEXT_VALUE_ROLES
.iter()
.any(|r| r.eq_ignore_ascii_case(role))
}
fn is_window_role(role: Role) -> bool {
matches!(role, Role::Window | Role::Frame | Role::Dialog)
}
fn surface_from_rects(rects: &[(i32, i32, i32, i32)]) -> SurfaceGeometry {
if rects.is_empty() {
return SurfaceGeometry {
x: 0.0,
y: 0.0,
width: 0.0,
height: 0.0,
};
}
let mut min_x = i32::MAX;
let mut min_y = i32::MAX;
let mut max_x = i32::MIN;
let mut max_y = i32::MIN;
for &(x, y, w, h) in rects {
min_x = min_x.min(x);
min_y = min_y.min(y);
max_x = max_x.max(x + w);
max_y = max_y.max(y + h);
}
SurfaceGeometry {
x: f64::from(min_x),
y: f64::from(min_y),
width: f64::from(max_x - min_x),
height: f64::from(max_y - min_y),
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum BusFault {
Denied,
Gone,
Transient,
Other,
}
fn classify_bus_fault(msg: &str) -> BusFault {
let lower = msg.to_ascii_lowercase();
if lower.contains("accessdenied")
|| lower.contains("notauthorized")
|| lower.contains("authfailed")
{
BusFault::Denied
} else if lower.contains("serviceunknown")
|| lower.contains("unknownobject")
|| lower.contains("namehasnoowner")
|| lower.contains("unknownmethod")
{
BusFault::Gone
} else if lower.contains("noreply")
|| lower.contains("timeout")
|| lower.contains("timedout")
|| lower.contains("timed out")
|| lower.contains("i/o error")
|| lower.contains("failed to connect")
{
BusFault::Transient
} else {
BusFault::Other
}
}
fn dbus_err(err: impl std::fmt::Display, ctx: &str) -> anyhow::Error {
let msg = err.to_string();
bus_error(classify_bus_fault(&msg), &msg, ctx)
}
fn bus_error(fault: BusFault, msg: &str, ctx: &str) -> anyhow::Error {
match fault {
BusFault::Denied => {
core::taxonomy_error(core::ERR_PERMISSION_DENIED, format!("{ctx}: {msg}"))
}
BusFault::Gone => core::taxonomy_error(
core::ERR_STALE_ELEMENT,
format!("{ctx}: {msg} — app or window closed; re-enumerate"),
),
BusFault::Transient => core::taxonomy_error(
core::ERR_DEGRADED,
format!("{ctx}: {msg} — transient bus failure, retry"),
),
BusFault::Other => anyhow::anyhow!("{ctx}: {msg}"),
}
}
fn action_dbus_outcome(err: impl std::fmt::Display, ctx: &str) -> anyhow::Result<Impact> {
let msg = err.to_string();
match classify_bus_fault(&msg) {
BusFault::Transient => Ok(Impact::Unconfirmed(format!(
"the accessibility call '{ctx}' failed at the bus level: {msg}"
))),
fault => Err(bus_error(fault, &msg, ctx)),
}
}
fn window_key(object_ref: &ObjectRefOwned) -> WindowKey {
WindowKey::new(format!(
"{}|{}",
object_ref.name_as_str().unwrap_or_default(),
object_ref.path_as_str()
))
}
struct WindowHandle {
object_ref: ObjectRefOwned,
key: WindowKey,
surface: SurfaceGeometry,
title: String,
active: bool,
focused: bool,
}
struct ResolvedWindow {
object_ref: ObjectRefOwned,
key: WindowKey,
app_info: AppInfo,
title: String,
surface: SurfaceGeometry,
}
async fn enumerate_apps(
conn: &zbus::Connection,
budget: &core::Budget,
) -> Result<Vec<(ObjectRefOwned, AppInfo)>, anyhow::Error> {
let root = registry_root(conn).await?;
let app_refs = root
.get_children()
.await
.map_err(|e| dbus_err(e, "enumerate applications"))?;
let mut out = Vec::new();
for app_ref in app_refs {
budget.check()?;
let Some(accessible) = app_ref.as_accessible_proxy(conn).await.ok() else {
continue;
};
let name = core::cap_label(&accessible.name().await.unwrap_or_default());
if name.is_empty() {
continue;
}
out.push((app_ref, AppInfo { pid: None, name }));
}
ensure_link(conn)?;
Ok(out)
}
async fn app_info_of(
conn: &zbus::Connection,
app_ref: &ObjectRefOwned,
) -> Result<AppInfo, anyhow::Error> {
let accessible = app_ref
.as_accessible_proxy(conn)
.await
.map_err(|e| dbus_err(e, "open application"))?;
let name = core::cap_label(&accessible.name().await.unwrap_or_default());
Ok(AppInfo { pid: None, name })
}
async fn surface_identity(
target: &TargetSpec,
window: &AccessibleProxy<'_>,
resolved_app: &AppInfo,
) -> (String, Option<String>) {
let app_name = target
.app()
.map_or_else(|| resolved_app.name.clone(), |(_, name)| name.to_string());
(app_name, window.name().await.ok())
}
async fn resolve_app_object(
conn: &zbus::Connection,
app_name: &str,
app_pid: Option<u32>,
budget: &core::Budget,
) -> Result<ObjectRefOwned, anyhow::Error> {
let apps = enumerate_apps(conn, budget).await?;
let needle = app_name.to_lowercase();
let by_name = apps
.iter()
.find(|(_, info)| info.name.to_lowercase() == needle);
let by_pid = apps
.iter()
.find(|(_, info)| app_pid.is_some() && info.pid == app_pid);
by_name
.or(by_pid)
.map(|(r, _)| r.clone())
.ok_or_else(|| core::surface_gone(app_name))
}
async fn claims_too_many_children(accessible: &AccessibleProxy<'_>) -> bool {
accessible
.child_count()
.await
.is_ok_and(|count| count > core::MAX_CHILDREN as i32)
}
async fn app_window_handles(
app_ref: &ObjectRefOwned,
conn: &zbus::Connection,
app_name: &str,
budget: &core::Budget,
) -> Result<Vec<WindowHandle>, anyhow::Error> {
let mut out = Vec::new();
budget.check()?;
let accessible = app_ref
.as_accessible_proxy(conn)
.await
.map_err(|e| dbus_err(e, "read windows"))?;
if claims_too_many_children(&accessible).await {
return Err(core::taxonomy_error(
core::ERR_DEGRADED,
format!(
"{app_name} advertises more than {} children — its windows were not read",
core::MAX_CHILDREN
),
));
}
budget.check()?;
let mut children = accessible
.get_children()
.await
.map_err(|e| dbus_err(e, "read windows"))?;
children.truncate(core::MAX_CHILDREN);
for child in children {
budget.check()?;
let Ok(ca) = child.as_accessible_proxy(conn).await else {
continue;
};
let role = ca.get_role().await.unwrap_or(Role::Unknown);
budget.check()?;
if !is_window_role(role) {
continue;
}
let title = core::cap_label(&ca.name().await.unwrap_or_default());
budget.check()?;
let state = ca.get_state().await.unwrap_or_default();
budget.check()?;
let Some(surface) = window_screen_extents(&ca, budget).await? else {
continue;
};
out.push(WindowHandle {
key: window_key(&child),
object_ref: child,
surface,
title,
active: state.contains(AtspiState::Active),
focused: state.contains(AtspiState::Focused),
});
}
Ok(out)
}
async fn window_screen_extents(
accessible: &AccessibleProxy<'_>,
budget: &core::Budget,
) -> Result<Option<SurfaceGeometry>, anyhow::Error> {
budget.check()?;
let Some(proxies) = accessible.proxies().await.ok() else {
return Ok(None);
};
budget.check()?;
let Some(comp) = proxies.component().await.ok() else {
return Ok(None);
};
budget.check()?;
let Some((x, y, w, h)) = comp.get_extents(CoordType::Screen).await.ok() else {
return Ok(None);
};
Ok(Some(SurfaceGeometry {
x: f64::from(x),
y: f64::from(y),
width: f64::from(w),
height: f64::from(h),
}))
}
struct ResolvedSurface {
object_ref: ObjectRefOwned,
app_name: String,
title: Option<String>,
surface: SurfaceGeometry,
}
async fn resolve_surface(
conn: &zbus::Connection,
target: &TargetSpec,
budget: &mut core::Budget,
) -> Result<ResolvedSurface, anyhow::Error> {
let (object_ref, app_info) = resolve_window(conn, target, budget).await?;
let proxy = object_ref
.as_accessible_proxy(conn)
.await
.map_err(|e| dbus_err(e, "open window"))?;
let (app_name, title) = surface_identity(target, &proxy, &app_info).await;
budget.check()?;
let label = core::name_surface(budget, &app_name, title.as_deref());
refuse_minimized(&proxy).await?;
budget.check()?;
let surface = window_screen_extents(&proxy, budget)
.await?
.ok_or_else(|| core::unanswered(&label, "for its window geometry"))?;
Ok(ResolvedSurface {
object_ref,
app_name,
title,
surface,
})
}
async fn refuse_minimized(proxy: &AccessibleProxy<'_>) -> Result<(), anyhow::Error> {
if proxy
.get_state()
.await
.is_ok_and(|state| state.contains(AtspiState::Iconified))
{
return Err(core::minimized_window(&core::cap_label(
&proxy.name().await.unwrap_or_default(),
)));
}
Ok(())
}
async fn resolve_window_by_key(
conn: &zbus::Connection,
app_name: &str,
key: &WindowKey,
) -> Result<ObjectRefOwned, anyhow::Error> {
let object_ref = object_ref_from_key(key).ok_or_else(|| core::surface_gone(app_name))?;
let accessible = object_ref
.as_accessible_proxy(conn)
.await
.map_err(|e| window_gone_or_silent(e, app_name))?;
let role = accessible
.get_role()
.await
.map_err(|e| window_gone_or_silent(e, app_name))?;
if !is_window_role(role) {
return Err(core::surface_gone(app_name));
}
Ok(object_ref)
}
fn window_gone_or_silent(err: impl std::fmt::Display, app_name: &str) -> anyhow::Error {
let msg = err.to_string();
match classify_bus_fault(&msg) {
BusFault::Gone => core::surface_gone(app_name),
BusFault::Transient => core::unanswered(
&core::surface_label(app_name, None),
"when the call looked for it",
),
fault => bus_error(fault, &msg, "resolve the window"),
}
}
fn object_ref_from_key(key: &WindowKey) -> Option<ObjectRefOwned> {
let (name, path) = key.as_str().split_once('|')?;
let name = zbus::names::UniqueName::try_from(name).ok()?;
let path = zbus::zvariant::ObjectPath::try_from(path).ok()?;
Some(ObjectRef::new_owned(name.to_owned(), path.to_owned()))
}
async fn app_current_window(
conn: &zbus::Connection,
app_ref: &ObjectRefOwned,
app_name: &str,
budget: &core::Budget,
) -> Result<ObjectRefOwned, anyhow::Error> {
let wins = app_window_handles(app_ref, conn, app_name, budget).await?;
if let Some(w) = wins.iter().find(|w| w.active || w.focused) {
return Ok(w.object_ref.clone());
}
wins.into_iter()
.next()
.map(|w| w.object_ref)
.ok_or_else(|| core::surface_gone(app_name))
}
async fn resolve_focused_window(
conn: &zbus::Connection,
budget: &core::Budget,
) -> Result<ResolvedWindow, anyhow::Error> {
let apps = enumerate_apps(conn, budget).await?;
let mut matches = Vec::new();
let mut skipped = None;
for (app_ref, app_info) in &apps {
budget.check()?;
let slice = budget.slice(core::APP_SLICE, core::windows_label(&app_info.name));
match app_window_handles(app_ref, conn, &app_info.name, &slice).await {
Ok(handles) => {
for w in handles {
if w.active || w.focused {
matches.push(ResolvedWindow {
object_ref: w.object_ref,
key: w.key,
app_info: app_info.clone(),
title: w.title,
surface: w.surface,
});
}
}
}
Err(_) => {
if slice.expired() {
skipped = Some(slice);
}
}
}
}
match matches.len() {
1 => return Ok(matches.remove(0)),
0 => {}
n => {
return Err(core::taxonomy_error(
core::ERR_NOT_MATCHED,
format!(
"{n} windows report active/focused state — enumerate apps/windows to pick a \
surface"
),
));
}
}
budget.check()?;
if let Some(slice) = skipped {
return Err(core::taxonomy_error(
core::ERR_DEGRADED,
format!("{}, and no window reports active/focused", slice.note()),
));
}
Err(core::taxonomy_error(
core::ERR_NOT_MATCHED,
"no focused window found (no window reports active/focused) — enumerate apps/windows first",
))
}
async fn resolve_window(
conn: &zbus::Connection,
target: &TargetSpec,
budget: &core::Budget,
) -> Result<(ObjectRefOwned, AppInfo), anyhow::Error> {
match target {
TargetSpec::Screen => Err(core::screen_observe_error()),
TargetSpec::Focused => {
let w = resolve_focused_window(conn, budget).await?;
Ok((w.object_ref, w.app_info))
}
TargetSpec::App { app_pid, app_name } => {
let app_ref = resolve_app_object(conn, app_name, *app_pid, budget).await?;
let win_ref = app_current_window(conn, &app_ref, app_name, budget).await?;
let app_info = app_info_of(conn, &app_ref).await?;
Ok((win_ref, app_info))
}
TargetSpec::Window {
app_pid,
app_name,
key,
..
} => {
let win_ref = resolve_window_by_key(conn, app_name, key).await?;
Ok((
win_ref,
AppInfo {
pid: *app_pid,
name: app_name.clone(),
},
))
}
}
}
async fn a11y_screen_extents(
conn: &zbus::Connection,
budget: &core::Budget,
) -> Result<Option<SurfaceGeometry>, anyhow::Error> {
budget.check()?;
let Some(root) = registry_root(conn).await.ok() else {
return Ok(None);
};
window_screen_extents(&root, budget).await
}
fn pixel_space() -> Option<(u32, u32)> {
if wayland() {
*SCREEN_DIMS.lock().unwrap_poison()
} else {
with_block_in_place(x11_screen_geometry)
.ok()
.map(|geo| (geo.width.round() as u32, geo.height.round() as u32))
}
}
fn a11y_unmapped_guidance() -> &'static str {
if super::capture_known_available() {
"capture target: \"screen\" first — its still and zoom always work — and prefer ref \
actions, which need no pixels"
} else {
"prefer ref actions, which need no pixels"
}
}
async fn a11y_pixel_scale(
conn: &zbus::Connection,
pixels: Option<(u32, u32)>,
budget: &core::Budget,
) -> Result<f64, anyhow::Error> {
let Some((width, height)) = pixels else {
return Err(core::taxonomy_error(
core::ERR_DEGRADED,
format!(
"the screen's pixel size is not known, so accessibility extents cannot be mapped \
into it — {}",
a11y_unmapped_guidance()
),
));
};
let Some(extents) = a11y_screen_extents(conn, budget).await? else {
if !wayland() {
return Ok(1.0);
}
return Err(core::taxonomy_error(
core::ERR_DEGRADED,
format!(
"the accessibility stack did not report the screen's geometry, so its coordinates \
cannot be mapped into the captured pixels — {}",
a11y_unmapped_guidance()
),
));
};
match core::a11y_scale(&extents, (width, height)) {
Some(scale) => Ok(scale),
None if !wayland() => Ok(1.0),
None => Err(core::taxonomy_error(
core::ERR_DEGRADED,
format!(
"the accessibility extents could not be reconciled with the {width}x{height} \
screen pixels, so nothing was mapped — {}",
a11y_unmapped_guidance()
),
)),
}
}
async fn window_still(
conn: &zbus::Connection,
win: &SurfaceGeometry,
cap: &Capture,
cap_geo: &SurfaceGeometry,
budget: &CaptureBudget,
lookup: &core::Budget,
) -> Result<Capture, anyhow::Error> {
let scale = a11y_pixel_scale(conn, Some((cap.width, cap.height)), lookup).await?;
core::crop_rgba(
cap,
core::window_capture_rect(&core::scale_geometry(*win, scale), cap_geo)?,
budget,
"the window still",
)
}
#[async_recursion]
async fn walk_node(
object_ref: &ObjectRefOwned,
conn: &zbus::Connection,
depth: usize,
handles: &mut Vec<ObjectRefOwned>,
nodes: &mut usize,
truncation: &mut Truncation,
budget: &core::Budget,
) -> anyhow::Result<Option<UiNode>> {
budget.check()?;
if *nodes >= core::MAX_TREE_NODES {
truncation.hit_node_ceiling(core::MAX_TREE_NODES);
return Ok(None);
}
let Some(accessible) = object_ref.as_accessible_proxy(conn).await.ok() else {
return Ok(None);
};
*nodes += 1;
handles.push(object_ref.clone());
let role = role_name(accessible.get_role().await.unwrap_or(Role::Unknown));
let name = accessible
.name()
.await
.ok()
.map(|name| truncation.keep_name(&name));
let proxies = accessible.proxies().await.ok();
budget.check()?;
let value = tree_value(proxies.as_ref(), &role).await;
let frame = tree_frame(proxies.as_ref()).await;
let actions = tree_actions(proxies.as_ref()).await;
budget.check()?;
let focused = accessible
.get_state()
.await
.is_ok_and(|s| s.contains(AtspiState::Focused));
let mut children = Vec::new();
if depth >= MAX_DEPTH {
if node_has_children(&accessible).await {
truncation.cut_depth(MAX_DEPTH);
}
} else {
budget.check()?;
if let Some(child_refs) = walkable_children(&accessible, truncation).await {
for child in child_refs {
if *nodes >= core::MAX_TREE_NODES {
truncation.hit_node_ceiling(core::MAX_TREE_NODES);
break;
}
if let Some(child_node) =
walk_node(&child, conn, depth + 1, handles, nodes, truncation, budget).await?
{
children.push(child_node);
}
}
}
}
Ok(Some(UiNode {
role,
name,
value,
frame,
actions,
focused,
children,
}))
}
async fn node_has_children(accessible: &AccessibleProxy<'_>) -> bool {
if claims_too_many_children(accessible).await {
return true;
}
accessible
.get_children()
.await
.is_ok_and(|children| !children.is_empty())
}
async fn walkable_children(
accessible: &AccessibleProxy<'_>,
truncation: &mut Truncation,
) -> Option<Vec<ObjectRefOwned>> {
if claims_too_many_children(accessible).await {
truncation.cut_children(core::MAX_CHILDREN);
return None;
}
let mut children = accessible.get_children().await.ok()?;
if children.len() > core::MAX_CHILDREN {
children.truncate(core::MAX_CHILDREN);
truncation.trim_children(core::MAX_CHILDREN);
}
Some(children)
}
async fn build_tree(
root_ref: &ObjectRefOwned,
conn: &zbus::Connection,
budget: &core::Budget,
) -> anyhow::Result<(UiNode, Vec<ObjectRefOwned>, Truncation)> {
let mut handles = Vec::new();
let mut nodes = 0usize;
let mut truncation = Truncation::default();
let Some(node) = walk_node(
root_ref,
conn,
0,
&mut handles,
&mut nodes,
&mut truncation,
budget,
)
.await?
else {
return Err(core::taxonomy_error(
core::ERR_DEGRADED,
format!(
"{} did not answer for its accessibility root, so nothing was read — {}",
budget.target(),
core::unanswered_next_step()
),
));
};
ensure_link(conn)?;
Ok((node, handles, truncation))
}
async fn tree_value(proxies: Option<&Proxies<'_>>, role: &str) -> Option<String> {
let proxies = proxies?;
if !is_text_role(role) {
return None;
}
let text = proxies.text().await.ok()?;
let count = text.character_count().await.ok()?;
if count <= 0 {
return None;
}
text.get_text(0, count.min(VALUE_READ_CHARS)).await.ok()
}
async fn tree_frame(proxies: Option<&Proxies<'_>>) -> Option<(f64, f64, f64, f64)> {
let proxies = proxies?;
let comp = proxies.component().await.ok()?;
let (x, y, w, h) = comp.get_extents(CoordType::Window).await.ok()?;
if w <= 0 || h <= 0 {
return None;
}
Some((f64::from(x), f64::from(y), f64::from(w), f64::from(h)))
}
async fn tree_actions(proxies: Option<&Proxies<'_>>) -> Vec<String> {
let Some(proxies) = proxies else {
return Vec::new();
};
let Ok(action) = proxies.action().await else {
return Vec::new();
};
let Ok(actions) = action.get_actions().await else {
return Vec::new();
};
actions
.into_iter()
.filter_map(|a| (!a.name.is_empty()).then(|| a.name.to_lowercase()))
.collect()
}
async fn press_element(proxies: &Proxies<'_>) -> Result<Impact, anyhow::Error> {
let action = proxies
.action()
.await
.map_err(|e| dbus_err(e, "read actions"))?;
let actions = action
.get_actions()
.await
.map_err(|e| dbus_err(e, "list actions"))?;
let idx = actions
.iter()
.position(|a| a.name.eq_ignore_ascii_case("press"))
.or_else(|| {
actions
.iter()
.position(|a| a.name.eq_ignore_ascii_case("click"))
})
.or_else(|| {
actions
.iter()
.position(|a| a.name.eq_ignore_ascii_case("open"))
});
let Some(idx) = idx else {
let available = actions
.iter()
.map(|a| a.name.as_str())
.collect::<Vec<_>>()
.join(", ");
return Err(core::taxonomy_error(
core::ERR_UNSUPPORTED,
format!(
"element has no press/open action; available actions: {available} — click it by \
coordinates instead"
),
));
};
let ok = match action.do_action(idx as i32).await {
Ok(ok) => ok,
Err(e) => return action_dbus_outcome(e, "perform action"),
};
if ok {
Ok(Impact::Done)
} else {
Err(anyhow!("action '{}' reported failure", actions[idx].name))
}
}
async fn set_element_value(
proxies: &Proxies<'_>,
role: &str,
text: &str,
) -> Result<Impact, anyhow::Error> {
if role.to_lowercase().contains("password") {
return Err(core::taxonomy_error(
core::ERR_UNSUPPORTED,
"element is a password field — SetValue refused (use raw input to type)",
));
}
if let Ok(editable) = proxies.editable_text().await {
let ok = match editable.set_text_contents(text).await {
Ok(ok) => ok,
Err(e) => return action_dbus_outcome(e, "set text contents"),
};
return if ok {
Ok(Impact::Done)
} else {
Err(anyhow!("set-text-contents returned false"))
};
}
if let Ok(value) = proxies.value().await
&& let Ok(num) = text.trim().parse::<f64>()
{
if let Err(e) = value.set_current_value(num).await {
return action_dbus_outcome(e, "set current value");
}
return Ok(Impact::Done);
}
Err(core::taxonomy_error(
core::ERR_UNSUPPORTED,
"element has no editable-text or value interface — SetValue not available; use raw input",
))
}
async fn raise_window(
conn: &zbus::Connection,
window_ref: &ObjectRefOwned,
target: &TargetSpec,
) -> Result<(), anyhow::Error> {
let proxy = window_ref
.as_accessible_proxy(conn)
.await
.map_err(|e| dbus_err(e, "open window"))?;
if let Ok(proxies) = proxy.proxies().await
&& let Ok(component) = proxies.component().await
{
let _ = component.grab_focus().await;
}
for _ in 0..core::RAISE_SETTLE_STEPS {
let state = proxy
.get_state()
.await
.map_err(|e| dbus_err(e, "read window state"))?;
if state.contains(AtspiState::Active) || state.contains(AtspiState::Focused) {
return Ok(());
}
tokio::time::sleep(Duration::from_millis(core::RAISE_SETTLE_MS)).await;
}
Err(core::not_brought_forward(target.describe()))
}
fn enigo_mod_key(m: Modifier) -> Key {
match m {
Modifier::Cmd => Key::Meta,
Modifier::Ctrl => Key::Control,
Modifier::Alt => Key::Alt,
Modifier::Shift => Key::Shift,
}
}
fn enigo_button(b: MouseButton) -> Button {
match b {
MouseButton::Left => Button::Left,
MouseButton::Right => Button::Right,
MouseButton::Middle => Button::Middle,
}
}
fn char_key(c: char) -> Option<Key> {
match c {
' ' => Some(Key::Space),
'\t' => Some(Key::Tab),
_ if c.is_ascii() && !c.is_control() => Some(Key::Unicode(c)),
_ => None,
}
}
fn enigo_key(name: &str) -> Result<Key, String> {
if let Ok(c) = name.parse::<char>()
&& let Some(key) = char_key(c)
{
return Ok(key);
}
let k = name.to_lowercase();
let key = match k.as_str() {
"return" | "enter" => Key::Return,
"tab" => Key::Tab,
"space" => Key::Space,
"delete" | "backspace" => Key::Backspace,
"forwarddelete" => Key::Delete,
"escape" => Key::Escape,
"home" => Key::Home,
"end" => Key::End,
"pageup" => Key::PageUp,
"pagedown" => Key::PageDown,
"up" => Key::UpArrow,
"down" => Key::DownArrow,
"left" => Key::LeftArrow,
"right" => Key::RightArrow,
"capslock" => Key::CapsLock,
"insert" => Key::Insert,
"f1" => Key::F1,
"f2" => Key::F2,
"f3" => Key::F3,
"f4" => Key::F4,
"f5" => Key::F5,
"f6" => Key::F6,
"f7" => Key::F7,
"f8" => Key::F8,
"f9" => Key::F9,
"f10" => Key::F10,
"f11" => Key::F11,
"f12" => Key::F12,
"minus" => Key::Unicode('-'),
"equal" => Key::Unicode('='),
"comma" => Key::Unicode(','),
"period" | "dot" => Key::Unicode('.'),
"slash" => Key::Unicode('/'),
"semicolon" => Key::Unicode(';'),
"quote" => Key::Unicode('\''),
"backslash" => Key::Unicode('\\'),
"backtick" => Key::Unicode('`'),
"leftbracket" => Key::Unicode('['),
"rightbracket" => Key::Unicode(']'),
_ => {
return Err(format!(
"unknown key '{name}' — supported keys: {KNOWN_KEYS}"
));
}
};
Ok(key)
}
struct InputSession {
enigo: Enigo,
buttons: Vec<Button>,
}
impl InputSession {
fn new() -> Result<Self, anyhow::Error> {
let enigo =
Enigo::new(&Settings::default()).map_err(|e| anyhow!("enigo init failed: {e}"))?;
Ok(Self {
enigo,
buttons: Vec::new(),
})
}
fn press_button(&mut self, button: Button) -> Result<(), enigo::InputError> {
self.buttons.push(button);
self.enigo.button(button, Direction::Press)
}
fn release_button(&mut self, button: Button) -> Result<(), enigo::InputError> {
let released = self.enigo.button(button, Direction::Release);
if released.is_ok() {
self.buttons.retain(|held| *held != button);
}
released
}
}
impl Drop for InputSession {
fn drop(&mut self) {
for button in self.buttons.drain(..) {
let _ = self.enigo.button(button, Direction::Release);
}
}
}
fn press_modifiers(
session: &mut InputSession,
mods: &[Modifier],
) -> Result<(), (usize, enigo::InputError)> {
for (i, m) in mods.iter().enumerate() {
if let Err(e) = session.enigo.key(enigo_mod_key(*m), Direction::Press) {
return Err((i, e));
}
}
Ok(())
}
fn release_modifiers(session: &mut InputSession, mods: &[Modifier]) -> Result<(), anyhow::Error> {
for m in mods.iter().rev() {
session
.enigo
.key(enigo_mod_key(*m), Direction::Release)
.map_err(input_err)?;
}
Ok(())
}
fn release_note(session: &mut InputSession, mods: &[Modifier]) -> Option<String> {
if mods.is_empty() || release_modifiers(session, mods).is_ok() {
return None;
}
let names = mods
.iter()
.map(|m| match m {
Modifier::Cmd => "cmd",
Modifier::Ctrl => "ctrl",
Modifier::Alt => "alt",
Modifier::Shift => "shift",
})
.collect::<Vec<_>>()
.join(", ");
Some(format!(
"{names} may still be held down, so every later chord and typed character may be modified \
— ask the owner to clear it before acting again"
))
}
fn with_held(detail: String, held: Option<String>) -> String {
match held {
Some(held) => format!("{detail}; {held}"),
None => detail,
}
}
fn done_after_release(session: &mut InputSession, mods: &[Modifier]) -> Impact {
match release_note(session, mods) {
None => Impact::Done,
Some(note) => Impact::DoneWith(note),
}
}
fn press_failed(
session: &mut InputSession,
mods: &[Modifier],
what: &str,
err: &enigo::InputError,
) -> anyhow::Error {
core::taxonomy_error(
core::ERR_DEGRADED,
with_held(format!("{what} ({err})"), release_note(session, mods)),
)
}
fn scroll_params(direction: ScrollDirection, amount: u32) -> (i32, Axis) {
let a = i32::try_from(amount).unwrap_or(i32::MAX);
match direction {
ScrollDirection::Down => (a, Axis::Vertical),
ScrollDirection::Up => (-a, Axis::Vertical),
ScrollDirection::Right => (a, Axis::Horizontal),
ScrollDirection::Left => (-a, Axis::Horizontal),
}
}
fn pixel_point(
point: (f64, f64),
geo: Option<&SurfaceGeometry>,
) -> Result<(i32, i32), anyhow::Error> {
let geo = geo.ok_or_else(|| anyhow!("raw input needs surface geometry for this action"))?;
let (x, y) = core::normalized_to_surface(point.0, point.1, geo)?;
Ok((x.round() as i32, y.round() as i32))
}
#[expect(clippy::needless_pass_by_value)]
fn input_err(e: enigo::InputError) -> anyhow::Error {
anyhow!("input synthesis failed: {e}")
}
fn key_chord(session: &mut InputSession, chord: &str) -> Result<Impact, anyhow::Error> {
let (mods, key) = core::parse_key_chord(chord)?;
let key = enigo_key(&key).map_err(anyhow::Error::msg)?;
if let Err((failed, e)) = press_modifiers(session, &mods) {
return Err(press_failed(
session,
&mods[..=failed],
"the chord did not go through — a modifier press failed",
&e,
));
}
if let Err(e) = session.enigo.key(key, Direction::Press) {
let released = session.enigo.key(key, Direction::Release).is_ok();
let key_state = if released {
"the chord may already have fired, and the key was released again"
} else {
"the key may still be held down"
};
return Ok(Impact::Unconfirmed(with_held(
format!("the chord's key press errored ({e}) — {key_state}"),
release_note(session, &mods),
)));
}
if let Err(e) = session.enigo.key(key, Direction::Release) {
return Ok(Impact::Unconfirmed(with_held(
format!(
"the chord's key went down but its release failed ({e}) — the key may still be \
held down"
),
release_note(session, &mods),
)));
}
Ok(done_after_release(session, &mods))
}
fn drag_incomplete(fx: i32, fy: i32, released: bool) -> Impact {
let at = format!("the button went down at ({fx}, {fy})");
Impact::Unconfirmed(if released {
format!(
"the drag started ({at}) but the gesture did not complete; the button was released \
again"
)
} else {
format!(
"the drag started ({at}) but the gesture did not complete and the button may still be \
held down"
)
})
}
fn release_after_failed_press(session: &mut InputSession, button: Button) -> &'static str {
if session.release_button(button).is_ok() {
"was released again"
} else {
"may still be held down"
}
}
fn click(
session: &mut InputSession,
x: i32,
y: i32,
button: Button,
double: bool,
modifiers: &[Modifier],
) -> Result<Impact, anyhow::Error> {
session
.enigo
.move_mouse(x, y, Coordinate::Abs)
.map_err(input_err)?;
if let Err((failed, e)) = press_modifiers(session, modifiers) {
return Err(press_failed(
session,
&modifiers[..=failed],
"the click did not go through — a modifier press failed",
&e,
));
}
for press in 0..(if double { 2 } else { 1 }) {
if let Err(e) = session.press_button(button) {
let state = release_after_failed_press(session, button);
let detail = if press == 0 {
format!(
"the button press errored ({e}) — the target may have seen a click at ({x}, \
{y}), and the button {state}"
)
} else {
format!(
"the first of the two clicks was delivered and the second's button press \
errored ({e}) — the button {state}"
)
};
return Ok(Impact::Unconfirmed(with_held(
detail,
release_note(session, modifiers),
)));
}
if let Err(e) = session.release_button(button) {
let detail = format!(
"the button was pressed but its release failed ({e}) — it may still be held down"
);
return Ok(Impact::Unconfirmed(with_held(
detail,
release_note(session, modifiers),
)));
}
}
Ok(done_after_release(session, modifiers))
}
fn type_text(session: &mut InputSession, text: &str) -> Impact {
let total = text.chars().count();
for (typed, ch) in text.chars().enumerate() {
let key = Key::Unicode(ch);
if let Err(e) = session.enigo.key(key, Direction::Press) {
let state = if session.enigo.key(key, Direction::Release).is_ok() {
"that character may already have been typed, and its key was released again"
} else {
"that character may already have been typed and its key may still be held down"
};
return Impact::Unconfirmed(format!(
"{typed} of {total} characters were typed before the next one's key press errored \
({e}) — {state}"
));
}
if let Err(e) = session.enigo.key(key, Direction::Release) {
return Impact::Unconfirmed(format!(
"{} of {total} characters were typed, but the last one's key release failed ({e}) \
— its key may still be held down",
typed + 1
));
}
}
Impact::Done
}
fn scroll(session: &mut InputSession, length: i32, axis: Axis) -> Impact {
let button = match (length.is_positive(), axis) {
(true, Axis::Vertical) => Button::ScrollDown,
(false, Axis::Vertical) => Button::ScrollUp,
(true, Axis::Horizontal) => Button::ScrollRight,
(false, Axis::Horizontal) => Button::ScrollLeft,
};
let ticks = length.abs();
for tick in 0..ticks {
if let Err(e) = session.press_button(button) {
let state = release_after_failed_press(session, button);
return Impact::Unconfirmed(format!(
"{tick} of {ticks} scroll ticks were sent before the next one's press errored \
({e}) — its wheel button {state}"
));
}
if let Err(e) = session.release_button(button) {
return Impact::Unconfirmed(format!(
"{} of {ticks} scroll ticks were sent, but the last one's release failed ({e}) — \
its wheel button may still be held down",
tick + 1
));
}
}
Impact::Done
}
fn raw_input_blocking(
input: RawInput,
geo: Option<&SurfaceGeometry>,
) -> Result<Impact, anyhow::Error> {
let mut session = InputSession::new()?;
match input {
RawInput::Click {
point,
button,
double,
modifiers,
} => {
let (x, y) = pixel_point(point, geo)?;
click(&mut session, x, y, enigo_button(button), double, &modifiers)
}
RawInput::TypeText { text } => Ok(type_text(&mut session, &text)),
RawInput::KeyChord { chord } => key_chord(&mut session, &chord),
RawInput::Scroll {
point,
direction,
amount,
} => {
if let Some((nx, ny)) = point {
let (x, y) = pixel_point((nx, ny), geo)?;
session
.enigo
.move_mouse(x, y, Coordinate::Abs)
.map_err(input_err)?;
}
let (length, axis) = scroll_params(direction, amount);
Ok(scroll(&mut session, length, axis))
}
RawInput::Drag { from, to } => {
let (fx, fy) = pixel_point(from, geo)?;
let (tx, ty) = pixel_point(to, geo)?;
session
.enigo
.move_mouse(fx, fy, Coordinate::Abs)
.map_err(input_err)?;
if let Err(e) = session.press_button(Button::Left) {
let state = release_after_failed_press(&mut session, Button::Left);
return Ok(Impact::Unconfirmed(format!(
"the drag's button press errored ({e}) at ({fx}, {fy}) — the button {state}, \
so the target may have taken it as a click"
)));
}
for i in 1..=INTERP_STEPS {
let t = i as f64 / INTERP_STEPS as f64;
let x = (fx as f64 + (tx - fx) as f64 * t).round() as i32;
let y = (fy as f64 + (ty - fy) as f64 * t).round() as i32;
if session.enigo.move_mouse(x, y, Coordinate::Abs).is_err() {
let released = session.release_button(Button::Left).is_ok();
return Ok(drag_incomplete(fx, fy, released));
}
std::thread::sleep(Duration::from_millis(2));
}
if let Err(e) = session.release_button(Button::Left) {
return Ok(Impact::Unconfirmed(format!(
"the drag ran to its end but the button's release failed ({e}) — the button \
may still be held down"
)));
}
Ok(Impact::Done)
}
}
}
fn x11_monitor_rects(conn: &RustConnection, root: xproto::Window) -> Vec<(i32, i32, i32, i32)> {
let resources = randr::get_screen_resources_current(conn, root)
.ok()
.and_then(|c| c.reply().ok())
.map(|r| (r.crtcs, r.config_timestamp))
.or_else(|| {
randr::get_screen_resources(conn, root)
.ok()
.and_then(|c| c.reply().ok())
.map(|r| (r.crtcs, r.config_timestamp))
});
if let Some((crtcs, stamp)) = resources {
let mut rects = Vec::with_capacity(crtcs.len());
for crtc in crtcs {
let Some(info) = randr::get_crtc_info(conn, crtc, stamp)
.ok()
.and_then(|c| c.reply().ok())
else {
continue;
};
if info.mode == 0 || info.width == 0 || info.height == 0 {
continue;
}
rects.push((
i32::from(info.x),
i32::from(info.y),
i32::from(info.width),
i32::from(info.height),
));
}
if !rects.is_empty() {
return rects;
}
}
xproto::get_geometry(conn, root)
.ok()
.and_then(|c| c.reply().ok())
.map(|g| vec![(0, 0, i32::from(g.width), i32::from(g.height))])
.unwrap_or_default()
}
fn x11_screen_geometry() -> Result<SurfaceGeometry, anyhow::Error> {
let (conn, screen) = x11rb::connect(None)
.map_err(|e| core::taxonomy_error(core::ERR_DEGRADED, format!("no X11 display: {e}")))?;
let geo = surface_from_rects(&x11_monitor_rects(&conn, conn.setup().roots[screen].root));
if geo.width <= 0.0 || geo.height <= 0.0 {
return Err(core::taxonomy_error(
core::ERR_DEGRADED,
"X11 capture failed: no monitor geometry",
));
}
Ok(geo)
}
fn x11_capture_screen(budget: &CaptureBudget) -> Result<(Capture, SurfaceGeometry), anyhow::Error> {
let (conn, screen) = x11rb::connect(None)
.map_err(|e| core::taxonomy_error(core::ERR_DEGRADED, format!("no X11 display: {e}")))?;
let root = conn.setup().roots[screen].root;
let lsb_first = conn.setup().image_byte_order == xproto::ImageOrder::LSB_FIRST;
let cap_bytes = conn
.maximum_request_bytes()
.clamp(64 * 1024, 4 * 1024 * 1024);
let rects = x11_monitor_rects(&conn, root);
let geo = surface_from_rects(&rects);
let (w, h) = (geo.width.round() as u32, geo.height.round() as u32);
if w == 0 || h == 0 {
return Err(core::taxonomy_error(
core::ERR_DEGRADED,
"X11 capture failed: empty virtual desktop",
));
}
if w > i16::MAX as u32 || h > i16::MAX as u32 {
return Err(core::taxonomy_error(
core::ERR_DEGRADED,
"X11 capture failed: virtual desktop too large for the X11 protocol",
));
}
let (ox, oy) = (geo.x.round() as i32, geo.y.round() as i32);
let (out_w, out_h) = (w as usize, h as usize);
let mut composite = Capture::alloc(budget, "the virtual desktop", w, h)?;
for &(rx, ry, rw, rh) in &rects {
if rw <= 0 || rh <= 0 {
continue;
}
let tile_w = rw as u32;
let rows_per_tile = (cap_bytes / (tile_w as usize * 4)).max(1) as u32;
let mut y = 0i32;
while y < rh {
let tile_h = rows_per_tile.min((rh - y) as u32);
let tile_reply = xproto::get_image(
&conn,
xproto::ImageFormat::Z_PIXMAP,
root,
(rx - ox) as i16,
(ry - oy + y) as i16,
tile_w as u16,
tile_h as u16,
u32::MAX,
)
.ok()
.and_then(|c| c.reply().ok());
let Some(reply) = tile_reply else {
return Err(core::taxonomy_error(
core::ERR_DEGRADED,
"X11 capture failed",
));
};
let tile = core::x11_pixels_to_rgba(
&reply.data,
tile_w,
tile_h,
reply.depth,
lsb_first,
budget,
)?;
core::blit_rgba(
&mut composite.rgba,
out_w,
out_h,
&tile,
(rx - ox) as usize,
(ry - oy + y) as usize,
);
y += tile_h as i32;
}
}
Ok((composite, geo))
}
static SCREEN_DIMS: Mutex<Option<(u32, u32)>> = Mutex::new(None);
static PORTAL_TOKEN_SEQ: AtomicU64 = AtomicU64::new(0);
const PORTAL_BUS: &str = "org.freedesktop.portal.Desktop";
const PORTAL_PATH: &str = "/org/freedesktop/portal/desktop";
const PORTAL_IFACE: &str = "org.freedesktop.portal.Screenshot";
const REQUEST_IFACE: &str = "org.freedesktop.portal.Request";
const PORTAL_FILE_MAX_BYTES: u64 = 128 * 1024 * 1024;
async fn portal_available() -> bool {
tokio::time::timeout(Duration::from_secs(3), async {
let conn = zbus::Connection::session().await.ok()?;
let proxy = zbus::Proxy::new(
&conn,
"org.freedesktop.DBus",
"/org/freedesktop/DBus",
"org.freedesktop.DBus",
)
.await
.ok()?;
let owned: bool = proxy.call("NameHasOwner", &PORTAL_BUS).await.ok()?;
Some(owned)
})
.await
.ok()
.flatten()
.unwrap_or(false)
}
async fn portal_screenshot(budget: &CaptureBudget) -> Result<Capture, anyhow::Error> {
let conn = zbus::Connection::session().await.map_err(|e| {
core::taxonomy_error(
core::ERR_DEGRADED,
format!(
"Screenshot portal missing (no session bus: {e}) — no manual setup needed; \
accessibility observe/act remain available"
),
)
})?;
let token = format!(
"mahbot_{}_{}",
std::process::id(),
PORTAL_TOKEN_SEQ.fetch_add(1, Ordering::Relaxed)
);
let mut request = PortalRequest::new(&conn, &token);
let outcome = screenshot_request(&conn, &token, &mut request, budget).await;
request.close().await;
outcome
}
struct PortalRequest {
conn: zbus::Connection,
path: Option<zbus::zvariant::OwnedObjectPath>,
}
impl PortalRequest {
fn new(conn: &zbus::Connection, token: &str) -> Self {
Self {
conn: conn.clone(),
path: request_path(conn, token),
}
}
fn adopt(&mut self, path: zbus::zvariant::OwnedObjectPath) {
self.path = Some(path);
}
async fn close(mut self) {
if let Some(path) = self.path.take() {
close_request(&self.conn, &path).await;
}
}
}
impl Drop for PortalRequest {
fn drop(&mut self) {
let Some(path) = self.path.take() else {
return;
};
let Ok(handle) = tokio::runtime::Handle::try_current() else {
return;
};
let conn = self.conn.clone();
handle.spawn(async move { close_request(&conn, &path).await });
}
}
fn request_path(conn: &zbus::Connection, token: &str) -> Option<zbus::zvariant::OwnedObjectPath> {
let sender = conn
.unique_name()?
.as_str()
.trim_start_matches(':')
.replace('.', "_");
format!("{PORTAL_PATH}/request/{sender}/{token}")
.try_into()
.ok()
}
async fn close_request(conn: &zbus::Connection, path: &zbus::zvariant::ObjectPath<'_>) {
let _ = tokio::time::timeout(Duration::from_secs(3), async {
let proxy = zbus::Proxy::new(conn, PORTAL_BUS, path, REQUEST_IFACE)
.await
.ok()?;
proxy.call::<_, _, ()>("Close", &()).await.ok()
})
.await;
}
async fn screenshot_request(
conn: &zbus::Connection,
token: &str,
request: &mut PortalRequest,
budget: &CaptureBudget,
) -> Result<Capture, anyhow::Error> {
let rule = zbus::MatchRule::builder()
.msg_type(zbus::message::Type::Signal)
.interface(REQUEST_IFACE)?
.member("Response")?
.build();
let mut stream = zbus::MessageStream::for_match_rule(rule, conn, None)
.await
.map_err(|e| {
core::taxonomy_error(core::ERR_DEGRADED, format!("portal subscribe failed: {e}"))
})?;
let mut options = HashMap::new();
options.insert("handle_token", zbus::zvariant::Value::from(token));
options.insert("modal", zbus::zvariant::Value::from(false));
options.insert("interactive", zbus::zvariant::Value::from(false));
let proxy = zbus::Proxy::new(conn, PORTAL_BUS, PORTAL_PATH, PORTAL_IFACE)
.await
.map_err(|e| {
core::taxonomy_error(
core::ERR_DEGRADED,
format!("Screenshot portal unavailable ({e}); accessibility observe/act remain available"),
)
})?;
let (req_path,): (zbus::zvariant::OwnedObjectPath,) = tokio::time::timeout(
Duration::from_secs(10),
proxy.call("Screenshot", &("", options)),
)
.await
.map_err(|_| {
core::taxonomy_error(
core::ERR_DEGRADED,
"Screenshot portal call timed out; accessibility observe/act remain available",
)
})?
.map_err(|e| {
core::taxonomy_error(
core::ERR_DEGRADED,
format!(
"Screenshot portal call failed ({e}); accessibility observe/act remain available"
),
)
})?;
let expected_path = req_path.as_str().to_owned();
request.adopt(req_path);
let wait = async {
while let Some(msg) = stream.next().await {
let msg = msg.map_err(|e| {
core::taxonomy_error(
core::ERR_DEGRADED,
format!("portal response stream failed: {e}"),
)
})?;
if msg
.header()
.path()
.is_none_or(|p| p.as_str() != expected_path.as_str())
{
continue;
}
let (code, results): (u32, HashMap<String, zbus::zvariant::OwnedValue>) =
msg.body().deserialize().map_err(|e| {
core::taxonomy_error(
core::ERR_DEGRADED,
format!("portal response decode failed: {e}"),
)
})?;
return portal_response(code, results, budget).await;
}
Err(core::taxonomy_error(
core::ERR_DEGRADED,
"portal closed the Screenshot response stream; accessibility observe/act remain available",
))
};
tokio::time::timeout(Duration::from_secs(15), wait)
.await
.map_err(|_| {
core::taxonomy_error(
core::ERR_DEGRADED,
"Screenshot response timed out; accessibility observe/act remain available",
)
})?
}
async fn read_portal_file(path: &std::path::Path) -> Result<Vec<u8>, anyhow::Error> {
let unreadable = |e: std::io::Error| {
core::taxonomy_error(
core::ERR_DEGRADED,
format!("portal screenshot unreadable: {e}"),
)
};
let file = tokio::fs::File::open(path).await.map_err(unreadable)?;
let mut bytes = Vec::new();
file.take(PORTAL_FILE_MAX_BYTES + 1)
.read_to_end(&mut bytes)
.await
.map_err(unreadable)?;
if bytes.len() as u64 > PORTAL_FILE_MAX_BYTES {
return Err(core::taxonomy_error(
core::ERR_DEGRADED,
format!(
"portal screenshot is larger than the {} MiB this tool reads — not read; \
accessibility observe/act remain available",
PORTAL_FILE_MAX_BYTES / (1024 * 1024),
),
));
}
Ok(bytes)
}
async fn portal_response(
code: u32,
results: HashMap<String, zbus::zvariant::OwnedValue>,
budget: &CaptureBudget,
) -> Result<Capture, anyhow::Error> {
match code {
0 => {
let uri_value = results.get("uri").ok_or_else(|| {
core::taxonomy_error(core::ERR_DEGRADED, "portal Screenshot reply had no uri")
})?;
let uri = String::try_from(uri_value.try_clone().map_err(|e| {
core::taxonomy_error(core::ERR_DEGRADED, format!("portal uri unreadable: {e}"))
})?)
.map_err(|e| {
core::taxonomy_error(core::ERR_DEGRADED, format!("portal uri not a string: {e}"))
})?;
let path = core::portal_uri_to_path(&uri)?;
let bytes = read_portal_file(&path).await?;
let mut limits = crate::util::media_target::raster_decode_limits();
let decode_cap = crate::util::media_target::RASTER_DECODE_MAX_ALLOC_BYTES
.min(budget.remaining().saturating_mul(4));
limits.max_alloc = Some(decode_cap);
let img = with_block_in_place(move || {
crate::util::media_target::decode_raster_image(&bytes, limits)
})
.map_err(|e| {
core::taxonomy_error(
core::ERR_DEGRADED,
format!(
"portal screenshot undecodable: {e} (this tool decodes a portal image \
at most {} px on a side within {} MiB)",
crate::util::media_target::RASTER_DECODE_MAX_DIMENSION_PX,
decode_cap / (1024 * 1024),
),
)
})?;
let (w, h) = core::capture_size(
"the portal screenshot",
img.width() as usize,
img.height() as usize,
)?;
let claim = budget.charge("the portal screenshot", w, h)?;
Ok(Capture::adopt(claim, w, h, img.into_rgba8().into_raw()))
}
1 => Err(core::taxonomy_error(
core::ERR_DEGRADED,
"screenshot dismissed or denied in the system dialog — screenshots are an opt-in \
elevated permission, no auto-retry; accessibility observe/act remain available",
)),
_ => Err(core::taxonomy_error(
core::ERR_DEGRADED,
format!(
"portal Screenshot failed (response {code}); accessibility observe/act remain available"
),
)),
}
}
#[async_trait::async_trait]
impl Backend for LinuxBackend {
fn is_supported(&self) -> bool {
let gui =
std::env::var_os("DISPLAY").is_some() || std::env::var_os("WAYLAND_DISPLAY").is_some();
let bus = std::env::var_os("DBUS_SESSION_BUS_ADDRESS").is_some()
|| std::env::var_os("XDG_RUNTIME_DIR")
.is_some_and(|d| std::path::PathBuf::from(d).join("bus").exists());
gui && bus
}
async fn desktop_unavailable(&self) -> Option<String> {
locked_reason().await
}
async fn capture_available(&self) -> bool {
if wayland() {
portal_available().await
} else {
with_block_in_place(|| x11_screen_geometry().is_ok())
}
}
fn capture_unavailable_error(&self) -> anyhow::Error {
if wayland() {
core::taxonomy_error(
core::ERR_DEGRADED,
"Wayland capture unavailable (Screenshot portal missing or denied) — no manual \
setup needed, screenshots are an opt-in elevated permission; accessibility \
observe/act remain available",
)
} else {
core::taxonomy_error(
core::ERR_DEGRADED,
"X11 capture init failed — accessibility observe/act remain available",
)
}
}
async fn list_apps(&self) -> Result<Vec<AppInfo>, anyhow::Error> {
let conn = self.conn().await?;
let budget = core::Budget::new(
core::LIST_BUDGET,
"application list",
"the running applications",
);
let mut out: Vec<AppInfo> = enumerate_apps(&conn, &budget)
.await?
.into_iter()
.map(|(_, info)| info)
.collect();
out.sort_by(|a, b| a.name.cmp(&b.name));
Ok(out)
}
async fn list_windows(&self, app: Option<&AppInfo>) -> Result<WindowListing, anyhow::Error> {
let conn = self.conn().await?;
let mut listing = WindowListing::default();
let budget = core::Budget::new(
core::LIST_BUDGET,
"window listing",
app.map_or_else(
|| "the running applications".to_string(),
|app| core::windows_label(&app.name),
),
);
for (app_ref, app_info) in enumerate_apps(&conn, &budget).await? {
if let Some(filter) = app {
let name_ok = filter.name.eq_ignore_ascii_case(&app_info.name);
let pid_ok = filter.pid.is_some() && filter.pid == app_info.pid;
if !name_ok && !pid_ok {
continue;
}
}
if let Err(e) = budget.check() {
listing.unread.push(UnreadWindows {
app_name: app_info.name.clone(),
reason: format!("{e:#}"),
});
continue;
}
let slice = budget.slice(core::APP_SLICE, core::windows_label(&app_info.name));
match app_window_handles(&app_ref, &conn, &app_info.name, &slice).await {
Ok(handles) => {
for w in handles {
listing.windows.push(WindowInfo {
app_pid: app_info.pid,
app_name: app_info.name.clone(),
title: w.title,
key: w.key,
surface: w.surface,
});
}
}
Err(e) => listing.unread.push(UnreadWindows {
app_name: app_info.name.clone(),
reason: core::unread_reason(&e, &slice),
}),
}
}
Ok(listing)
}
async fn focused_window(&self) -> Result<WindowInfo, anyhow::Error> {
let conn = self.conn().await?;
let budget = core::Budget::new(
core::LIST_BUDGET,
"focused-window search",
"the running applications",
);
let w = resolve_focused_window(&conn, &budget).await?;
Ok(WindowInfo {
app_pid: w.app_info.pid,
app_name: w.app_info.name,
title: w.title,
key: w.key,
surface: w.surface,
})
}
async fn surface_geometry(
&self,
target: &TargetSpec,
) -> Result<SurfaceGeometry, anyhow::Error> {
let mut budget = core::Budget::new(core::LIST_BUDGET, "surface lookup", target.describe());
self.surface_geometry_with(target, &mut budget).await
}
async fn observe(&self, target: &TargetSpec) -> Result<Observation, anyhow::Error> {
if matches!(target, TargetSpec::Screen) {
return Err(core::screen_observe_error());
}
let conn = self.conn().await?;
let mut budget = core::Budget::new(core::OBSERVE_BUDGET, "observation", target.describe());
let window = resolve_surface(&conn, target, &mut budget).await?;
let (root, _handles, mut truncation) =
build_tree(&window.object_ref, &conn, &budget).await?;
let window_title = window.title.map(|title| truncation.keep_name(&title));
Ok(Observation {
app_name: window.app_name,
window_title,
surface: window.surface,
root,
truncation,
})
}
async fn act_on_element(
&self,
target: &TargetSpec,
locator: &Locator,
act: ElementAct,
) -> Result<Impact, anyhow::Error> {
let conn = self.conn().await?;
if matches!(target, TargetSpec::Screen) {
return Err(core::screen_act_error());
}
let mut budget = core::Budget::new(core::ACT_BUDGET, "element action", target.describe());
let (window_ref, app) = resolve_window(&conn, target, &budget).await?;
let proxy = window_ref
.as_accessible_proxy(&conn)
.await
.map_err(|e| dbus_err(e, "open window"))?;
let (app_name, title) = surface_identity(target, &proxy, &app).await;
core::name_surface(&mut budget, &app_name, title.as_deref());
let (root, handles, _truncation) = build_tree(&window_ref, &conn, &budget).await?;
let matched = core::resolve_locator_checked(&root, locator)?;
if handles.is_empty() {
return Err(core::taxonomy_error(
core::ERR_STALE_ELEMENT,
"element tree is empty — the window may have closed; re-observe",
));
}
let element_ref = &handles[core::pre_order_index(&root, matched)];
let element = element_ref
.as_accessible_proxy(&conn)
.await
.map_err(|e| dbus_err(e, "open element"))?;
let proxies = element
.proxies()
.await
.map_err(|e| dbus_err(e, "enumerate element interfaces"))?;
let role = role_name(element.get_role().await.unwrap_or(Role::Unknown));
match act {
ElementAct::Press => press_element(&proxies).await,
ElementAct::SetValue { text } => set_element_value(&proxies, &role, &text).await,
}
}
async fn raw_input(
&self,
target: &TargetSpec,
input: RawInput,
) -> Result<Impact, anyhow::Error> {
core::check_raw_input(&input)?;
if wayland() {
return Err(core::taxonomy_error(
core::ERR_DEGRADED,
"this is a Wayland session, which cannot receive synthesized input: XTEST would \
reach only the XWayland windows, never the native ones on screen — use ref-based \
accessibility actions where the app exposes them",
));
}
let mut budget = core::Budget::new(core::LIST_BUDGET, "surface lookup", target.describe());
if core::pinned_input(target, &input) {
let conn = self.conn().await?;
let (window_ref, _app) = resolve_window(&conn, target, &budget).await?;
raise_window(&conn, &window_ref, target).await?;
}
let geo = match &input {
RawInput::Click { .. }
| RawInput::Scroll { point: Some(_), .. }
| RawInput::Drag { .. } => Some(self.surface_geometry_with(target, &mut budget).await?),
_ => None,
};
with_block_in_place(|| raw_input_blocking(input, geo.as_ref()))
}
async fn cursor_position(&self) -> Result<(f64, f64), anyhow::Error> {
if wayland() {
return Err(core::taxonomy_error(
core::ERR_DEGRADED,
"global pointer position is not exposed on Wayland",
));
}
with_block_in_place(|| {
let enigo =
Enigo::new(&Settings::default()).map_err(|e| anyhow!("enigo init failed: {e}"))?;
let (x, y) = enigo.location().map_err(input_err)?;
Ok((f64::from(x), f64::from(y)))
})
}
async fn capture(
&self,
target: &TargetSpec,
budget: &CaptureBudget,
) -> Result<Capture, anyhow::Error> {
match target {
TargetSpec::Screen => Ok(self.capture_screen_channel(budget).await?.0),
other => {
let conn = self.conn().await?;
let mut lookup =
core::Budget::new(core::LIST_BUDGET, "surface lookup", other.describe());
let window = resolve_surface(&conn, other, &mut lookup).await?;
let (screen_cap, screen_geo) = self.capture_screen_channel(budget).await?;
let lookup = core::Budget::new(
core::LIST_BUDGET,
"surface lookup",
lookup.target().to_string(),
);
window_still(
&conn,
&window.surface,
&screen_cap,
&screen_geo,
budget,
&lookup,
)
.await
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn role_name_covers_key_roles_and_falls_back() {
assert_eq!(role_name(Role::Button), "pushbutton");
assert_eq!(role_name(Role::Frame), "frame");
assert_eq!(role_name(Role::Window), "window");
assert_eq!(role_name(Role::Dialog), "dialog");
assert_eq!(role_name(Role::CheckBox), "checkbox");
assert_eq!(role_name(Role::PageTab), "tab");
assert_eq!(role_name(Role::Entry), "textfield");
assert_eq!(role_name(Role::Unknown), "unknown");
assert_eq!(role_name(Role::Canvas), "canvas");
}
#[test]
fn surface_union_handles_negative_origins() {
let rects = [
(0, 0, 1920, 1080),
(1920, 0, 1920, 1080),
(0, -100, 800, 600),
];
let geo = surface_from_rects(&rects);
assert_eq!(
(geo.x, geo.y, geo.width, geo.height),
(0.0, -100.0, 3840.0, 1180.0)
);
let (px, py) = core::normalized_to_surface(500.0, 500.0, &geo).unwrap();
assert!((px - 1920.0).abs() < 1e-9, "x = {px}");
assert!((py - 490.0).abs() < 1e-9, "y = {py}");
}
}