use crate::atspi_tree::{
accessibility_chrome_context, focused_element_summary_in_app, list_accessible_apps,
object_ref_application_pid, object_ref_owner_pid,
perform_action as invoke_accessibility_action, perform_named_action, probe_focused_element,
set_element_value, snapshot_accessibility_tree, AccessibilityAction, AccessibilityNode,
AccessibleAppSummary, Bounds, EditableTextSetOutcome, FocusProbe, FocusedElementSummary,
FocusedTextInsert, ValueSetInvocation,
};
use crate::diagnostics::{
doctor_report, setup_accessibility_report, DoctorReport, ReadinessReport, SetupReport,
WindowingReport,
};
use crate::diagnostics_cache::DiagnosticsCache;
use crate::gnome_extension::{setup_window_targeting_report, WindowTargetingSetupReport};
use crate::remote_desktop::{
click as portal_click, drag as portal_drag, keysyms_for_text, press_key_chord,
press_keycode_chord, scroll as portal_scroll, start_portal_keyboard_session,
start_portal_pointer_session, type_text_with_keysyms, InputOperationGuard, PointerButton,
PortalKey, PortalKeyboardSession, PortalPointerSession, ScrollDirection,
};
use crate::screenshot::{
capture_screenshot_raw, prepare_screenshot_payload, RawScreenshotCapture, ScreenshotCapture,
ScreenshotOutputFormat, ScreenshotPayloadOptions,
};
use crate::terminal::{terminal_paste_shortcut, TerminalPasteShortcut};
use crate::tree_encoding::{is_placeholder_text, NodeDetail, TreeNodes};
use crate::windowing::backends::kwin::{
KwinElementClickPoint, KwinElementClickTarget, KwinElementFrame,
};
use crate::windowing::registry;
use crate::windows::{
focus_window_target, focused_window, focused_window_with_backend, list_windows,
resolve_window_target, window_permission_hint, WindowFocusResult, WindowInfo, WindowTarget,
GNOME_SHELL_EXTENSION_BACKEND, GNOME_SHELL_INTROSPECT_BACKEND, KWIN_BACKEND,
UNAVAILABLE_BACKEND,
};
use crate::ydotool;
use anyhow::{Context, Result};
use rmcp::{
handler::server::wrapper::{Json, Parameters},
model::{CallToolResult, ContentBlock},
schemars::JsonSchema,
tool, tool_handler, tool_router, ErrorData, ServerHandler, ServiceExt,
};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::{
collections::BTreeMap,
env,
ffi::OsString,
future::Future,
path::Path,
process::{Command, Output, Stdio},
sync::{Arc, Mutex},
time::Duration,
};
use tokio::{
process::Command as TokioCommand,
time::{sleep, timeout},
};
use xkeysym::key as xkey;
use zbus::{Connection as ZbusConnection, Proxy as ZbusProxy};
const INPUT_COMMAND_TIMEOUT: Duration = Duration::from_secs(10);
// Linux limits each exec argument (including its NUL) to 32 pages, at least
// 128 KiB on supported platforms. Reject oversized text before sending input.
const WTYPE_MAX_TEXT_BYTES: usize = 128 * 1024 - 1;
const WTYPE_STDIN_MAX_CHARS: usize = 100;
const WTYPE_FOCUS_TIMEOUT: Duration = Duration::from_millis(1500);
const MAX_SCROLL_PAGES: f64 = 100.0;
const YDOTOOL_TYPE_CHARS_PER_SECOND: u64 = 20;
const KDE_CLIPBOARD_DBUS_TIMEOUT: Duration = Duration::from_secs(3);
const KDE_KLIPPER_SERVICE: &str = "org.kde.klipper";
const KDE_KLIPPER_PATH: &str = "/klipper";
const KDE_KLIPPER_INTERFACE: &str = "org.kde.klipper.klipper";
const SHELL_ENABLE_ENV: &str = "COMPUTER_USE_LINUX_ENABLE_SHELL";
const COMPLETION_ENABLE_ENV: &str = "COMPUTER_USE_LINUX_NOTIFY_ON_COMPLETE";
const SHELL_DEFAULT_TIMEOUT_SECS: u64 = 30;
const SHELL_MAX_TIMEOUT_SECS: u64 = 120;
const SHELL_MAX_COMMAND_BYTES: usize = 64 * 1024;
const SHELL_MAX_CWD_BYTES: usize = 4096;
const SHELL_MAX_ENV_ENTRIES: usize = 64;
const SHELL_MAX_ENV_BYTES: usize = 64 * 1024;
const SHELL_RESPONSE_STREAM_BYTES: usize = 512 * 1024;
const UNMATCHABLE_ACCESSIBILITY_APP_FILTER: &str = "\0";
/// Nodes, their app/window ownership, and their recovery scope must come from
/// the same snapshot.
#[derive(Default)]
struct CachedAccessibilitySnapshot {
/// The snapshot_id results return for this tree; empty before the first
/// store. Every store gets a new one.
id: String,
nodes: Vec<AccessibilityNode>,
pid: Option<u32>,
window_id: Option<u64>,
window_backend: Option<String>,
recovery_scope: Option<AccessibilityRecoveryScope>,
/// Encoding of the result the nodes were returned in; get_tree_page pages
/// them in the same one.
detail: NodeDetail,
/// The AT-SPI window the tree was narrowed to, when the app exposes
/// several windows under one root. A post-action tree with no window
/// target of its own keeps the same scope.
window_scope: Option<String>,
}
/// The app or window an element action names, as the cached-node target
/// check sees it: the target pid when one is known, and the resolved window
/// with its backend when the target named a window.
#[derive(Debug, Clone, Default, PartialEq)]
struct ElementOwnerTarget {
pid: Option<u32>,
window: Option<(u64, String)>,
/// The target names exactly this window: a `window_id`, or an app_id,
/// wm_class or title that matches one window. An `app_id` shared by
/// several windows resolves to the first, which the caller did not name.
exact_window: bool,
}
impl ElementOwnerTarget {
fn pid(pid: Option<u32>) -> Self {
Self {
pid,
..Self::default()
}
}
/// A resolved window target. The window's own pid wins over `pid`, the
/// one the caller passed.
fn window(window: &WindowInfo, pid: Option<u32>, exact_window: bool) -> Self {
Self {
pid: window.pid.or(pid),
window: Some((window.window_id, window.backend.clone())),
exact_window,
}
}
}
/// The previous snapshot's window scope, pid and window, for a post-action
/// tree that narrows through that scope.
#[derive(Debug, Clone)]
struct CachedTreeWindow {
scope: String,
pid: Option<u32>,
window: Option<(u64, String)>,
}
/// The cached pid and window scope a post-action tree may reuse, taken
/// under the action's input lock so a later snapshot cannot change it.
#[derive(Debug, Clone, Default)]
struct PostActionScope {
pid: Option<u32>,
tree: Option<CachedTreeWindow>,
/// Bound to a dispatched element's owner before the action. The tree
/// keeps this scope on stdio too, and the owner may match zero or
/// several compositor windows without the desktop having changed.
bound: bool,
}
impl PostActionScope {
/// The scope of an element's verified owner: its window range only,
/// never the latest caller's snapshot.
fn bound_to(target: &ElementObservationTarget) -> Self {
Self {
pid: target.owner.pid,
tree: target.tree_window.clone(),
bound: true,
}
}
}
#[derive(Debug, Clone, Default)]
struct ElementObservationTarget {
owner: WindowTarget,
// The observed tree's range, not the popup or dialog's own window identity.
tree_window: Option<CachedTreeWindow>,
}
impl ElementObservationTarget {
fn inherit_tree_window(&mut self, previous: &Self) {
// Recovery clears the snapshot before retrying the same application.
if self.tree_window.is_none()
&& self.owner.pid.is_some()
&& self.owner.pid == previous.owner.pid
{
self.tree_window = previous
.tree_window
.clone()
.filter(|window| window.pid == self.owner.pid);
}
}
}
/// How a cached tree was returned: its node encoding, which get_tree_page
/// pages in, and the AT-SPI window it was narrowed to.
#[derive(Debug, Clone, Default)]
struct TreeShape {
detail: NodeDetail,
window_scope: Option<String>,
}
/// Whether an action brought its target window to the front, so a
/// post-action `screenshot_after: "window"` crop shows that window.
/// perform_action and set_value act through AT-SPI and leave it where it is.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum TargetRaised {
Yes,
No,
}
/// Snapshot ids: a random prefix per server and a serial. The prefix keeps an
/// id from an earlier server process from naming a snapshot of this one.
struct SnapshotIds {
prefix: String,
serial: std::sync::atomic::AtomicU64,
}
impl Default for SnapshotIds {
fn default() -> Self {
Self {
prefix: format!(
"{:08x}",
getrandom::u32().unwrap_or_else(|_| std::process::id())
),
serial: std::sync::atomic::AtomicU64::new(0),
}
}
}
impl SnapshotIds {
fn next(&self) -> String {
let serial = self
.serial
.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
+ 1;
format!("{}-{serial}", self.prefix)
}
}
/// Refuse a `snapshot_id` that does not name the cached snapshot `current`.
/// Indices and selectors resolve only against the cached tree, so once a later
/// get_app_state, include_tree or perform_action recovery replaced the named
/// one, its indices may name other elements. `prefix` is this server's id
/// prefix. An empty or blank id counts as absent, as other optional strings do.
fn check_snapshot_id(
requested: Option<&str>,
current: &str,
prefix: &str,
) -> std::result::Result<(), String> {
let Some(requested) = trimmed_nonempty(requested) else {
return Ok(());
};
if requested == current {
return Ok(());
}
let reason = if requested
.split_once('-')
.is_none_or(|(requested_prefix, _)| requested_prefix != prefix)
{
"this server did not issue it; it may come from a server process that has since exited"
} else if current.is_empty() {
"no tree is cached now, since the last tree read failed or perform_action recovery cleared it"
} else {
"a later get_app_state, include_tree or perform_action recovery replaced it"
};
Err(format!(
"snapshot_id {requested:?} is not the cached snapshot: {reason}, and its element indices could name other elements now. Call get_app_state and use the element_index values and snapshot_id it returns."
))
}
#[derive(Clone)]
struct AccessibilityRecoveryScope {
params: GetAppStateParams,
target_pid: Option<u32>,
}
struct RefreshedAccessibilitySnapshot {
snapshot: crate::atspi_tree_impl::AccessibilitySnapshot,
target_pid: Option<u32>,
window: Option<WindowInfo>,
}
#[derive(Clone, Default)]
pub struct ComputerUseLinux {
last_snapshot: Arc<Mutex<CachedAccessibilitySnapshot>>,
snapshot_ids: Arc<SnapshotIds>,
portal_pointer_session: Arc<Mutex<Option<PortalPointerSession>>>,
portal_keyboard_session: Arc<Mutex<Option<PortalKeyboardSession>>>,
/// Lazily-created uinput absolute pointer (preferred coordinate backend).
abs_pointer: Arc<Mutex<Option<crate::abs_pointer::AbsPointer>>>,
portal_session_init_lock: Arc<tokio::sync::Mutex<()>>,
input_operation_lock: Arc<tokio::sync::Mutex<()>>,
kde_clipboard_lock: Arc<tokio::sync::Mutex<()>>,
/// Cached physical desktop size from the most recent full-frame capture;
/// used for off-screen warnings and portal logical-coordinate mapping.
desktop_size: Arc<Mutex<Option<(u32, u32)>>>,
/// Monitor layout for that capture, read once per capture for the
/// off-screen note. Every new capture resets it.
capture_layout: Arc<Mutex<CaptureLayout>>,
/// On-screen activity overlay (opt-out via `COMPUTER_USE_LINUX_INDICATOR=0`).
indicator: Arc<crate::indicator::Indicator>,
/// Private files for trees too large to return inline; removed on exit.
tree_spiller: Arc<crate::tree_spill::TreeSpiller>,
/// Diagnostics behind get_app_state's readiness block. Failed observations,
/// failed actions, and setup tools invalidate it; the doctor tool bypasses
/// it.
diagnostics: Arc<DiagnosticsCache<DoctorReport>>,
accessibility_check: AccessibilityCheck,
/// Set for HTTP: clients share the cached tree, so element calls must name it.
require_snapshot_ids: bool,
/// Set for HTTP: a remote client gets no file writes on the host, so
/// screenshot_file_path is refused.
http_transport: bool,
}
/// The monitor layout of the cached capture. `generation` counts captures,
/// so a layout read during a newer capture is not stored for it.
#[derive(Debug, Default)]
struct CaptureLayout {
generation: u64,
monitors: CaptureMonitors,
}
/// Monitor rectangles (x, y, width, height), tagged with their space.
#[derive(Debug, Clone, Default, PartialEq)]
enum CaptureMonitors {
#[default]
Unread,
Unknown,
/// Wayland logical pixels. The capture spans their bounding box.
Logical(Vec<(i32, i32, i32, i32)>),
/// X11 root-window pixels, the space of an X11 capture, which also
/// holds framebuffer area no monitor shows.
Root(Vec<(i32, i32, i32, i32)>),
}
/// The route a coordinate click, scroll or drag took, for the off-screen note.
/// Each one treats a point on no monitor differently.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum PointerRoute {
/// xdotool, or ydotool on X11. The X server keeps the pointer on the
/// screen, then moves it to the monitor it came from or lets it sit in
/// the dead space.
X11,
/// The uinput absolute pointer. The device clamps to the capture and the
/// clamp note says so; the compositor places a point in a monitor gap.
Uinput,
/// The remote desktop portal refuses a point on no shared monitor.
Portal,
}
/// The live check get_app_state compares a cached report's
/// `can_build_accessibility_tree` against. Tests swap it to agree with their
/// fixture report instead of reading the desktop running them.
#[derive(Clone, Copy)]
struct AccessibilityCheck(fn() -> bool);
impl Default for AccessibilityCheck {
fn default() -> Self {
Self(crate::diagnostics_impl::can_build_accessibility_tree_now)
}
}
fn sanitize_unsigned_integer_formats(value: &mut serde_json::Value) {
let serde_json::Value::Object(object) = value else {
return;
};
let has_unsigned_format = matches!(
object.get("format").and_then(serde_json::Value::as_str),
Some("uint" | "uint8" | "uint16" | "uint32" | "uint64" | "usize")
);
if has_unsigned_format {
object.remove("format");
}
for nested in object.values_mut() {
match nested {
serde_json::Value::Object(_) => sanitize_unsigned_integer_formats(nested),
serde_json::Value::Array(items) => {
for item in items {
sanitize_unsigned_integer_formats(item);
}
}
_ => {}
}
}
}
impl ComputerUseLinux {
fn mcp_tool_router(&self) -> rmcp::handler::server::router::tool::ToolRouter<Self> {
self.router_with_completion(env::var(COMPLETION_ENABLE_ENV).as_deref() == Ok("1"))
}
fn router_with_completion(
&self,
enabled: bool,
) -> rmcp::handler::server::router::tool::ToolRouter<Self> {
let mut router = Self::tool_router();
if !enabled {
router.map.remove("complete_interaction");
}
if !shell_execution_enabled() {
router.map.remove("run_shell");
}
for route in router.map.values_mut() {
let input_schema = Arc::make_mut(&mut route.attr.input_schema);
for value in input_schema.values_mut() {
sanitize_unsigned_integer_formats(value);
}
if let Some(output_schema) = route.attr.output_schema.as_mut() {
for value in Arc::make_mut(output_schema).values_mut() {
sanitize_unsigned_integer_formats(value);
}
}
}
router
}
}
#[tool_router]
impl ComputerUseLinux {
#[tool(
name = "complete_interaction",
description = "Send a desktop notification that this agent has finished its interaction. Available only with COMPUTER_USE_LINUX_NOTIFY_ON_COMPLETE=1. This cue does not acquire or release an exclusive desktop lock. Delivery is best effort and bounded; it may be suppressed by desktop notification settings.",
annotations(
read_only_hint = false,
destructive_hint = false,
idempotent_hint = false,
open_world_hint = true
)
)]
async fn complete_interaction(&self) -> Json<CompletionOutput> {
if env::var(COMPLETION_ENABLE_ENV).as_deref() != Ok("1") {
return Json(CompletionOutput {
ok: true,
cue: "skipped".into(),
note: "Completion notifications are disabled.".into(),
});
}
Json(send_completion_notification(Path::new("notify-send"), Duration::from_secs(2)).await)
}
#[tool(
name = "doctor",
description = "Report Linux Computer Use desktop integration readiness.",
annotations(
read_only_hint = true,
destructive_hint = false,
idempotent_hint = true,
open_world_hint = false
)
)]
async fn doctor(&self) -> Json<DoctorReport> {
Json(
tokio::task::spawn_blocking(doctor_report)
.await
.expect("diagnostics task panicked"),
)
}
#[tool(
name = "setup_accessibility",
description = "Enable GNOME accessibility so Linux Computer Use can read AT-SPI trees: sets and reads back the toolkit-accessibility gsettings key when it is not already true, and requests runtime org.a11y.Status IsEnabled when the tree cannot be built or the key does not read back.",
annotations(
read_only_hint = false,
destructive_hint = false,
idempotent_hint = true,
open_world_hint = false
)
)]
async fn setup_accessibility(&self) -> Json<SetupReport> {
let report = tokio::task::spawn_blocking(setup_accessibility_report)
.await
.expect("accessibility setup task panicked");
self.diagnostics.invalidate();
Json(report)
}
#[tool(
name = "setup_window_targeting",
description = "Install and enable the bundled GNOME Shell extension. GNOME Wayland needs it for exact window focus, move_window and resize_window, since Introspect cannot focus, move or resize windows (and recent GNOME blocks its window list too); every GNOME session needs it for the on-screen indicator. Ask the user to log out and back in when the result says a shell reload is required.",
annotations(
read_only_hint = false,
destructive_hint = false,
idempotent_hint = true,
open_world_hint = false
)
)]
async fn setup_window_targeting(&self) -> Json<WindowTargetingSetupReport> {
let report = setup_window_targeting_report().await;
self.diagnostics.invalidate();
Json(report)
}
#[tool(
name = "list_apps",
description = "List running Linux desktop app candidates visible to the Computer Use backend.",
annotations(
read_only_hint = true,
destructive_hint = false,
idempotent_hint = true,
open_world_hint = true
)
)]
async fn list_apps(&self) -> Json<ListAppsOutput> {
let (accessible_apps, accessibility_error) = match list_accessible_apps(50).await {
Ok(apps) => (apps, None),
Err(error) => {
self.diagnostics.invalidate();
(Vec::new(), Some(format!("{error:#}")))
}
};
Json(ListAppsOutput {
apps: list_process_apps(),
accessible_apps,
accessibility_error,
note: "Linux Computer Use lists process candidates plus AT-SPI application roots when accessibility is enabled.".to_string(),
})
}
#[tool(
name = "get_tree_page",
description = "Return a page of the accessibility tree from the latest get_app_state or include_tree snapshot, starting at node offset (default 0). Use it when a tree was too large to return inline (accessibility_tree_spill is set) and the host cannot read the spill file; page with next_offset until it is null. Element indices match the snapshot, so element_index actions work on these nodes. Pass the snapshot's snapshot_id to refuse a page from a tree a later snapshot replaced. It does not read the desktop again.",
annotations(
read_only_hint = true,
destructive_hint = false,
idempotent_hint = true,
open_world_hint = true
)
)]
async fn get_tree_page(
&self,
Parameters(params): Parameters<GetTreePageParams>,
) -> Json<GetTreePageOutput> {
let (id, nodes, detail) = self
.last_snapshot
.lock()
.map(|snapshot| (snapshot.id.clone(), snapshot.nodes.clone(), snapshot.detail))
.unwrap_or_default();
let offset = params.offset.unwrap_or(0);
let checked = self
.require_snapshot_id("get_tree_page", params.snapshot_id.as_deref())
.and_then(|()| {
check_snapshot_id(
params.snapshot_id.as_deref(),
&id,
&self.snapshot_ids.prefix,
)
});
if let Err(message) = checked {
return Json(GetTreePageOutput {
snapshot_id: None,
nodes: TreeNodes::new(Vec::new(), detail),
offset,
next_offset: None,
node_count: 0,
message,
});
}
let mut page = tree_page(&nodes, detail, offset);
page.snapshot_id = (!id.is_empty()).then_some(id);
Json(page)
}
#[tool(
name = "list_windows",
description = "List compositor windows with title, app id, class, focus state, client type, and known bounds.",
annotations(
read_only_hint = true,
destructive_hint = false,
idempotent_hint = true,
open_world_hint = true
)
)]
async fn list_windows(&self) -> Json<ListWindowsOutput> {
let output = window_list_output().await;
if output.error.is_some() {
self.diagnostics.invalidate();
}
Json(output)
}
#[tool(
name = "focused_window",
description = "Return the compositor window that currently has keyboard focus.",
annotations(
read_only_hint = true,
destructive_hint = false,
idempotent_hint = true,
open_world_hint = true
)
)]
async fn focused_window(&self) -> Json<FocusedWindowOutput> {
match focused_window_with_backend().await {
Ok((backend, window)) => {
Json(FocusedWindowOutput {
backend,
focused_window: window,
error: None,
permissions_hint: None,
message:
"Focused window query completed through the available compositor window backend."
.to_string(),
})
}
Err(error) => {
self.diagnostics.invalidate();
let error = format!("{error:#}");
Json(FocusedWindowOutput {
backend: UNAVAILABLE_BACKEND.to_string(),
focused_window: None,
permissions_hint: window_permission_hint(&error),
error: Some(error),
message: "Focused window query failed; targeted keyboard input is unavailable until window introspection works.".to_string(),
})
}
}
}
#[tool(
name = "activate_window",
output_schema = rmcp::handler::server::tool::schema_for_output::<ActivateWindowToolOutput>(),
description = "Focus a Linux desktop window by window_id, pid, app_id, wm_class, title, or terminal selectors when the compositor permits it. Pass screenshot_after (true, \"screen\", or \"window\") or include_tree=true to return post-action observation in the same result.",
annotations(
read_only_hint = false,
destructive_hint = false,
idempotent_hint = true,
open_world_hint = true
)
)]
async fn activate_window(
&self,
Parameters(params): Parameters<ActivateWindowParams>,
) -> Result<CallToolResult, ErrorData> {
let capture = params.capture.clone();
let target = params.clone().into_target();
let Json(output) = self.dispatch_activate_window(params).await;
self.finish_activate_window_output(output, &capture, &target)
.await
}
async fn dispatch_activate_window(
&self,
params: ActivateWindowParams,
) -> Json<ActivateWindowOutput> {
let received = Some(serde_json::json!(params.clone()));
let target = params.into_target();
self.indicator.action("activate_window").await;
match focus_window_target(&target).await {
Ok(focus) => {
let ok = focus_satisfies_target(&focus, &target);
Json(ActivateWindowOutput {
ok,
implemented: true,
backend: focus.backend.clone(),
focus: Some(focus),
error: None,
permissions_hint: None,
received,
})
}
Err(error) => {
let error = format!("{error:#}");
Json(ActivateWindowOutput {
ok: false,
implemented: true,
backend: UNAVAILABLE_BACKEND.to_string(),
focus: None,
permissions_hint: window_permission_hint(&error),
error: Some(error),
received,
})
}
}
}
#[tool(
name = "get_app_state",
output_schema = rmcp::handler::server::tool::schema_for_output::<GetAppStateOutput>(),
description = "Get a size-bounded screenshot and accessibility state for a Linux app. Scope the accessibility tree with app_name_or_bundle_identifier or a window_id/pid/app_id/wm_class/title target; omitting a target returns the whole desktop tree and can flood context. Nodes are compact (empty fields left out, actions by name; verbose=true returns every field). Pass interactive_only=true for only actionable nodes, headings and status regions, with the same element_index values. A title or window_id target narrows an app with several windows under one AT-SPI root (Chromium) to that window, keeping the app's popups and dialogs. A window-targeted screenshot needs that window visible and already focused (outside Hyprland background mode); otherwise screenshot_error is set and the tree still returns, so use the screenshot tool to raise and capture it. A tree too large to return inline is written whole to the private file in accessibility_tree_spill. The result's snapshot_id names the tree: element_index and selector actions that pass it are refused once a later tree replaced it. Screenshot results always include coordinate_width, coordinate_height, scale, and format, plus quality for jpeg; callers can request jpeg/quality for compression before resizing. Pass screenshot_file_path (an absolute path to a new file, stdio only; an existing path is refused) to save the screenshot there instead of returning image content; screenshot.file_path then names it.",
annotations(
read_only_hint = true,
destructive_hint = false,
idempotent_hint = true,
open_world_hint = true
)
)]
async fn get_app_state(
&self,
Parameters(params): Parameters<GetAppStateParams>,
) -> Result<CallToolResult, ErrorData> {
let captures = params.include_screenshot.unwrap_or(true);
let result = self.app_state(params).await;
if captures {
self.indicator.action("get_app_state").await;
}
result
}
async fn app_state(&self, params: GetAppStateParams) -> Result<CallToolResult, ErrorData> {
let verbose = params.verbose.unwrap_or(false);
// A clone: this call's screenshot result is recorded on it, not on the
// cached report.
let (mut diagnostics, mut probed) = self.diagnostics.report_and_probed(verbose).await;
// The tree gate below must see accessibility switched on or off within
// the TTL.
if !probed {
let can_build_now = tokio::task::spawn_blocking(self.accessibility_check.0)
.await
.unwrap_or(false);
if can_build_now != diagnostics.readiness.can_build_accessibility_tree {
diagnostics = self.diagnostics.report(true).await;
probed = true;
}
}
let (
window_context,
window_error,
window_permissions_hint,
window_enumerated,
failed_window_backends,
) = self.resolve_window_context(¶ms).await;
let (max_nodes, max_depth) =
crate::atspi_tree::snapshot_limits(params.max_nodes, params.max_depth);
let include_screenshot = params.include_screenshot.unwrap_or(true);
let screenshot_options = params.screenshot_options();
let screenshot_target_requested = params.window_target().has_target();
// Whether the caller asked for any scope at all. `tree_scoped` can still
// come back false when a target matched no AT-SPI root, and that case
// needs different advice than "pass a target".
let accessibility_target_requested = screenshot_target_requested
|| params
.app_name_or_bundle_identifier
.as_deref()
.is_some_and(|name| !name.trim().is_empty());
let app_filter = self
.resolve_accessibility_app_filter(¶ms, window_context.as_ref(), window_enumerated)
.await;
let mut screenshot_captured = None;
let mut capture_note = None;
// A path that cannot be written is reported without a capture.
let file_refusal = params
.screenshot_file_path
.as_deref()
.filter(|_| include_screenshot)
.and_then(|path| {
crate::screenshot_file::validate_screenshot_file_path(
path,
params.format.unwrap_or(ScreenshotOutputFormat::Png),
self.http_transport,
)
.err()
})
.map(|error| format!("{error:#}"));
let (screenshot, screenshot_error) = if include_screenshot && file_refusal.is_none() {
let result: Result<ScreenshotCapture> = async {
let _pipeline = crate::screenshot_impl::screenshot_pipeline_permit().await?;
if let Some(window) = window_context.as_ref() {
if let Some(capture) = self.hyprland_background_capture(window).await {
let (raw, note) = capture?;
screenshot_captured = Some(true);
capture_note = note;
return crate::screenshot_impl::run_image_task(move || {
prepare_screenshot_payload(raw, screenshot_options)
})
.await;
}
// Refuse an unfocused target before capturing: the call must
// not grab the desktop (and raise a portal prompt) only to
// discard it.
ensure_readonly_screenshot_target_is_visible(window)?;
}
let raw = {
let _hold = self.indicator.hold_for_capture().await?;
capture_screenshot_raw().await
};
screenshot_captured = Some(raw.is_ok());
let raw = raw?;
self.cache_desktop_size(raw.width, raw.height);
let crop = if let Some(window) = window_context.as_ref() {
Some(self.window_crop_rect_for_capture(window, &raw).await?)
} else {
None
};
crate::screenshot_impl::run_image_task(move || {
prepare_app_state_screenshot(
raw,
crop,
screenshot_target_requested,
screenshot_options,
)
})
.await
}
.await;
match result {
Ok(capture) => (Some(capture), None),
Err(error) => (None, Some(format!("{error:#}"))),
}
} else {
(None, None)
};
// A failed file write is reported like a failed capture, but it says
// nothing about the desktop's screenshot readiness.
let screenshot_capture_failed = screenshot_error.is_some();
let mut screenshot_file = None;
let (screenshot, screenshot_error) =
match (screenshot, params.screenshot_file_path.as_deref()) {
(Some(capture), Some(path)) => {
match self.save_screenshot_file(&capture, path).await {
Ok(saved) => {
screenshot_file = Some(saved);
(Some(capture), None)
}
Err(error) => (None, Some(format!("{error:#}"))),
}
}
(screenshot, _) => (screenshot, screenshot_error.or(file_refusal)),
};
let mut tree_scoped = false;
let mut tree_root_pid = None;
let mut accessibility_tree_truncated = false;
let mut shell_only_note = None;
let mut unmatched_pid_note = None;
let mut window_scope = None;
let mut interactive_note = None;
let detail = NodeDetail::from_verbose(verbose);
let (accessibility_tree, accessibility_tree_raw_count, accessibility_error, tree_read) =
if diagnostics.readiness.can_build_accessibility_tree {
let target_pid = accessibility_snapshot_target_pid(
app_filter.as_deref(),
window_context
.as_ref()
.and_then(|window| window.pid)
.or(params.pid.filter(|_| params.window_id.is_none())),
);
let window_title = accessibility_window_title(¶ms, window_context.as_ref());
// A listed window is alive even when its pid owns no AT-SPI
// root (a Flatpak app behind xdg-dbus-proxy); its title finds it.
let window_pid = window_context.as_ref().and_then(|window| window.pid);
let pid_fallback_name = accessibility_filter_candidates(window_context.as_ref())
.into_iter()
.next()
.filter(|_| window_pid.is_some() && window_pid == target_pid);
match snapshot_accessibility_tree(
app_filter.as_deref(),
target_pid,
window_title.as_deref(),
pid_fallback_name.as_deref(),
max_nodes,
max_depth,
)
.await
{
Ok(snapshot) => {
unmatched_pid_note = unmatched_accessibility_pid_note(
target_pid,
snapshot.root_pid,
snapshot.app_filter_matched,
);
tree_scoped = snapshot.scoped;
tree_root_pid = snapshot.root_pid;
accessibility_tree_truncated = snapshot.truncated;
window_scope = snapshot.window_scope;
let raw_count = snapshot.nodes.len();
let (nodes, note) = compact_accessibility_tree_with_note(
snapshot.nodes,
tree_scoped,
accessibility_tree_truncated,
);
shell_only_note = note;
let nodes = if params.interactive_only.unwrap_or(false) {
let compacted = nodes.len();
let nodes = interactive_accessibility_tree(nodes);
interactive_note = Some(format!(
"interactive_only kept {} of {compacted} compacted nodes; element_index values match the full tree.",
nodes.len()
));
nodes
} else {
nodes
};
let read = if raw_count == 0 {
TreeRead::Failed
} else {
TreeRead::Nodes
};
(nodes, raw_count, None, read)
}
Err(error) => (Vec::new(), 0, Some(format!("{error:#}")), TreeRead::Failed),
}
} else {
(
Vec::new(),
0,
Some(
"GNOME accessibility is disabled; call setup_accessibility first."
.to_string(),
),
TreeRead::AccessibilityOff,
)
};
let observed = AppStateObservation {
screenshot_failed: screenshot_capture_failed,
window: match (&window_context, &window_error) {
(_, Some(_)) => WindowLookup::Failed,
(Some(_), None) => WindowLookup::Resolved,
(None, None) => WindowLookup::NotRequested,
},
failed_window_backends: &failed_window_backends,
tree: tree_read,
};
if let Some(contradiction) =
readiness_contradiction(&diagnostics.readiness, &diagnostics.windowing, observed)
{
// Readiness is what explains this observation, so it must describe
// the desktop as this call found it, not a cached report from
// before.
if !probed {
diagnostics = self.diagnostics.report(true).await;
}
if contradiction == Contradiction::Failure {
// After the refresh, so the next call probes again.
self.diagnostics.invalidate();
}
}
if let Some(captured) = screenshot_captured {
diagnostics.readiness.record_screenshot_result(captured);
}
let snapshot_id = if accessibility_error.is_none() {
// Record a pid only when the tree's roots were matched by it. An
// explicit app-name filter can match another process when this pid
// has no AT-SPI root; recording the requested pid then would let
// that app's index pass the target check instead of taking the
// per-node owner lookup.
let id = self
.commit_app_state_snapshot(
&accessibility_tree,
tree_root_pid,
window_context.as_ref(),
¶ms,
tree_scoped,
TreeShape {
detail,
window_scope: window_scope.clone(),
},
)
.await;
Some(id)
} else {
self.commit_cleared_snapshot().await;
None
};
let compacted_count = accessibility_tree.len();
let mut message = if let Some(error) = &accessibility_error {
format!("MCP registration is working, but AT-SPI tree extraction failed: {error}")
} else if let Some(note) = &unmatched_pid_note {
let screenshot_status = if let Some(capture) = &screenshot {
format!("Captured a screenshot through {}.", capture.source)
} else if let Some(error) = &screenshot_error {
format!("Screenshot capture failed: {error}")
} else {
"Screenshot capture was not requested.".to_string()
};
format!("MCP registration is working. {note} {screenshot_status}")
} else if let Some(capture) = &screenshot {
format!(
"MCP registration, screenshot capture, and AT-SPI tree extraction are working. Captured {} accessibility nodes (compacted from {}) and a screenshot through {}.",
compacted_count,
accessibility_tree_raw_count,
capture.source
)
} else if let Some(error) = &screenshot_error {
format!(
"MCP registration and AT-SPI tree extraction are working. Captured {} accessibility nodes (compacted from {}). Screenshot capture failed: {error}",
compacted_count,
accessibility_tree_raw_count,
)
} else {
format!(
"MCP registration and AT-SPI tree extraction are working. Captured {} accessibility nodes (compacted from {}). Screenshot capture was not requested.",
compacted_count,
accessibility_tree_raw_count,
)
};
if let Some(window) = &window_context {
message.push_str(&format!(
" Window target resolved to window_id {}.",
window.window_id
));
} else if let Some(error) = &window_error {
message.push_str(&format!(" Window target resolution failed: {error}"));
}
if let Some(note) = capture_note {
message.push(' ');
message.push_str(¬e);
}
if let Some(path) = &screenshot_file {
message.push_str(&format!(" The screenshot was saved to {path}."));
}
if let Some(warning) = unscoped_accessibility_tree_warning(
tree_scoped,
accessibility_error.is_none(),
accessibility_target_requested,
) {
message.push(' ');
message.push_str(warning);
}
if let Some(note) = truncated_accessibility_tree_note(accessibility_tree_truncated) {
message.push(' ');
message.push_str(note);
}
if let Some(note) = shell_only_note {
message.push(' ');
message.push_str(note);
}
if let Some(window) = window_scope {
message.push_str(&format!(
" The app exposes several windows; the tree covers only the window named {window:?}."
));
}
if let Some(note) = interactive_note {
message.push(' ');
message.push_str(¬e);
}
// Full diagnostics are huge (portal/process dumps); emit them only on
// request. The compact readiness block always travels, and failures get
// a pointer to verbose=true instead of an automatic dump.
let readiness = diagnostics.readiness.clone();
let include_full = verbose;
if !include_full
&& (accessibility_error.is_some()
|| screenshot_error.is_some()
|| window_error.is_some())
{
message.push_str(" Pass verbose=true for full diagnostics.");
}
let mut output = GetAppStateOutput {
app_name_or_bundle_identifier: params.app_name_or_bundle_identifier,
window_context,
window_error,
window_permissions_hint,
backend: "linux-atspi".to_string(),
screenshot: screenshot.as_ref().map(|capture| ScreenshotSummary {
file_path: screenshot_file.clone(),
..ScreenshotSummary::from(capture)
}),
screenshot_error,
snapshot_id,
accessibility_tree: TreeNodes::new(Vec::new(), detail),
accessibility_tree_raw_count,
accessibility_tree_spill: None,
tree_scoped,
accessibility_tree_truncated,
accessibility_error,
readiness,
diagnostics: include_full.then_some(diagnostics),
message,
};
// The tree gets whatever the rest of the result leaves of the budget.
let rest = serde_json::to_string(&output).map_or(0, |json| json.len());
let mut accessibility_tree = TreeNodes::new(accessibility_tree, detail);
let (spill, note) = fit_tree(
&self.tree_spiller,
&mut accessibility_tree,
tree_budget(rest),
);
output.accessibility_tree = accessibility_tree;
output.accessibility_tree_spill = spill;
if let Some(note) = note {
output.message.push(' ');
output.message.push_str(¬e);
}
// A saved image is not returned inline.
app_state_result(output, screenshot.filter(|_| screenshot_file.is_none()))
}
#[tool(
name = "screenshot",
description = "Capture the screen and return it as a viewable, size-bounded image. Optionally target a window (window_id/pid/wm_class/title/app_id): the window is raised to the front and the image is cropped before any resize; outside Hyprland background mode, a target without known bounds is refused, so pass full_screen=true there. Returns the image plus a JSON caption with returned and coordinate dimensions, scale, format, quality, source, cropped_to_window, and window_title. The crop origin is not reported; use relative=true on click or scroll for window-relative input. Callers can request jpeg/quality for compression before resizing. Pass screenshot_file_path (an absolute path to a new file, stdio only; an existing path is refused) to save the image there instead: the result is then only the caption, with file_path.",
annotations(
read_only_hint = false,
destructive_hint = false,
idempotent_hint = false,
open_world_hint = true
)
)]
async fn screenshot(&self, Parameters(params): Parameters<ScreenshotParams>) -> CallToolResult {
let result = match self.capture_screenshot(params).await {
Ok(result) => result,
Err(error) => {
self.diagnostics.invalidate();
CallToolResult::error(vec![ContentBlock::text(format!("{error:#}"))])
}
};
self.indicator.action("screenshot").await;
result
}
/// Background mode for the screenshot tool: an unfocused Hyprland target is
/// captured with `grim -T` and never raised.
async fn hyprland_background_screenshot(
&self,
target: &WindowTarget,
params: &ScreenshotParams,
) -> Option<Result<CallToolResult>> {
if !crate::hyprland_input::background_enabled() {
return None;
}
let windows = list_windows().await.ok()?;
let window = resolve_window_target(&windows, target).ok()?.clone();
let (raw, note) = match self.hyprland_background_capture(&window).await? {
Ok(capture) => capture,
Err(error) => return Some(Err(error.context("targeted screenshot failed"))),
};
let options = params.screenshot_options();
let capture = match crate::screenshot_impl::run_image_task(move || {
prepare_screenshot_payload(raw, options)
})
.await
{
Ok(capture) => capture,
Err(error) => return Some(Err(error.context("screenshot resize failed"))),
};
let mut caption = serde_json::json!({
"width": capture.width,
"height": capture.height,
"coordinate_width": capture.coordinate_width,
"coordinate_height": capture.coordinate_height,
"scale": capture.scale,
"resized": capture.resized,
"bytes": capture.bytes,
"original_bytes": capture.original_bytes,
"max_bytes": capture.max_bytes,
"format": capture.format,
"quality": capture.quality,
"source": capture.source,
"cropped_to_window": true,
"window_title": window.title,
"window_raised": false,
});
if let Some(note) = note {
caption["hidden_workspace_note"] = serde_json::json!(note);
}
Some(
self.screenshot_tool_result(capture, caption, params.screenshot_file_path.as_deref())
.await,
)
}
/// The screenshot tool's result: the image and its JSON caption, or with
/// screenshot_file_path only the caption, naming the saved file.
async fn screenshot_tool_result(
&self,
capture: ScreenshotCapture,
mut caption: serde_json::Value,
file_path: Option<&str>,
) -> Result<CallToolResult> {
if let Some(path) = file_path {
caption["file_path"] =
serde_json::json!(self.save_screenshot_file(&capture, path).await?);
return Ok(CallToolResult::success(vec![ContentBlock::text(
caption.to_string(),
)]));
}
Ok(CallToolResult::success(vec![
ContentBlock::image(data_url_payload(&capture.data_url), capture.mime_type),
ContentBlock::text(caption.to_string()),
]))
}
async fn capture_screenshot(&self, params: ScreenshotParams) -> Result<CallToolResult> {
// Refuse a bad path before anything is raised or captured.
if let Some(path) = params.screenshot_file_path.as_deref() {
crate::screenshot_file::validate_screenshot_file_path(
path,
params.format.unwrap_or(ScreenshotOutputFormat::Png),
self.http_transport,
)?;
}
let _pipeline = crate::screenshot_impl::screenshot_pipeline_permit().await?;
let target = params.window_target();
if let (Some(target), false) = (target.as_ref(), params.full_screen.unwrap_or(false)) {
if let Some(result) = self.hyprland_background_screenshot(target, ¶ms).await {
return result;
}
}
let target_window = match target.as_ref() {
Some(target) => Some(
self.resolve_screenshot_window(target, params.raise_window.unwrap_or(true))
.await
.context("targeted screenshot failed")?,
),
None => None,
};
let crop_window = (!params.full_screen.unwrap_or(false))
.then_some(target_window.as_ref())
.flatten();
let window_label = target_window
.as_ref()
.and_then(|window| window.title.clone());
let raw_capture = {
let _hold = self.indicator.hold_for_capture().await?;
capture_screenshot_raw()
.await
.context("screenshot failed")?
};
self.cache_desktop_size(raw_capture.width, raw_capture.height);
// Warn when the target window extends past the visible desktop: the
// portal only captures on-screen pixels, so the crop silently loses the
// off-screen region while coordinate metadata still claims full size.
let off_screen_note = match crop_window.and_then(|window| window.bounds.as_ref()) {
Some(bounds) => self.off_screen_note_for_bounds(bounds).await,
None => None,
};
let (capture, cropped) = match crop_window {
Some(window) => {
let (x, y, width, height) = self
.window_crop_rect_for_capture(window, &raw_capture)
.await
.context("targeted screenshot crop failed")?;
let (raw_capture, bytes, width, height) =
crate::screenshot_impl::run_image_task(move || {
let (bytes, width, height) =
crop_png(&raw_capture.bytes, x, y, width, height).map_err(|error| {
anyhow::anyhow!("targeted screenshot crop failed: {error}")
})?;
Ok((raw_capture, bytes, width, height))
})
.await?;
(
RawScreenshotCapture {
mime_type: raw_capture.mime_type,
bytes,
source: raw_capture.source,
width,
height,
},
true,
)
}
None => (raw_capture, false),
};
let options = params.screenshot_options();
let capture = crate::screenshot_impl::run_image_task(move || {
prepare_screenshot_payload(capture, options)
})
.await
.context("screenshot resize failed")?;
let mut caption = serde_json::json!({
"width": capture.width,
"height": capture.height,
"coordinate_width": capture.coordinate_width,
"coordinate_height": capture.coordinate_height,
"scale": capture.scale,
"resized": capture.resized,
"bytes": capture.bytes,
"original_bytes": capture.original_bytes,
"max_bytes": capture.max_bytes,
"format": capture.format,
"quality": capture.quality,
"source": capture.source,
"cropped_to_window": cropped,
"window_title": window_label,
});
if let Some(note) = off_screen_note {
caption["window_off_screen"] = serde_json::json!(true);
caption["off_screen_note"] = serde_json::json!(note);
}
self.screenshot_tool_result(capture, caption, params.screenshot_file_path.as_deref())
.await
}
/// Lazily create the uinput absolute pointer, sizing its ABS range to the
/// full capture's pixel dimensions. This preserves the coordinate space
/// used by click/scroll/drag, independently of logical monitor scaling.
/// Explicit backend overrides skip this device, including a cached one.
/// A device sized for another capture is replaced: the compositor spreads
/// its range across the current layout, so it would scale every point.
/// With the device matching the capture, an off-capture point is clamped
/// and the clamp note reports where it landed. A failed replacement is an
/// error, as on the KWin path: the caller reports it and sends nothing.
async fn ensure_abs_pointer(&self) -> std::result::Result<bool, AbsPointerFailure> {
if self.is_cosmic_wayland_session()
|| (native_x11_xdotool_pointer_session(
env::var("XDG_SESSION_TYPE").ok().as_deref(),
env::var("WAYLAND_DISPLAY").ok().as_deref(),
) && env_var_non_empty("DISPLAY"))
|| env_flag_enabled("CU_DISABLE_ABS_POINTER")
|| env_flag_enabled("COMPUTER_USE_LINUX_FORCE_YDOTOOL_POINTER")
|| (env_flag_enabled("COMPUTER_USE_LINUX_FORCE_PORTAL_POINTER")
&& self.is_wayland_session())
{
return Ok(false);
}
let desktop_size = self.desktop_size.lock().ok().and_then(|guard| *guard);
let stale_size = match self.abs_pointer.lock() {
Ok(mut guard) => match (guard.as_ref(), desktop_size) {
(Some(pointer), Some((width, height)))
if !pointer.matches_dimensions(width as i32, height as i32) =>
{
*guard = None;
Some((width, height))
}
(Some(_), _) => return Ok(true),
(None, _) => None,
},
Err(_) => return Ok(false),
};
// The cached size is the latest raw capture, the space the caller's
// coordinates come from, so a replacement needs no new capture.
let ((width, height), stale) = match stale_size {
Some(size) => (size, true),
None => {
let Ok(cap) = capture_screenshot_raw().await else {
return Ok(false);
};
self.cache_desktop_size(cap.width, cap.height);
((cap.width, cap.height), false)
}
};
self.create_abs_pointer_with(width, height, stale, crate::abs_pointer::AbsPointer::create)
.await
}
async fn ensure_abs_pointer_for_kwin_capture(
&self,
width: u32,
height: u32,
) -> std::result::Result<bool, AbsPointerFailure> {
if self.is_cosmic_wayland_session()
|| (native_x11_xdotool_pointer_session(
env::var("XDG_SESSION_TYPE").ok().as_deref(),
env::var("WAYLAND_DISPLAY").ok().as_deref(),
) && env_var_non_empty("DISPLAY"))
|| env_flag_enabled("CU_DISABLE_ABS_POINTER")
|| env_flag_enabled("COMPUTER_USE_LINUX_FORCE_YDOTOOL_POINTER")
|| (env_flag_enabled("COMPUTER_USE_LINUX_FORCE_PORTAL_POINTER")
&& self.is_wayland_session())
{
return Ok(false);
}
let stale = {
let mut guard = self
.abs_pointer
.lock()
.map_err(|_| AbsPointerFailure::lock_poisoned())?;
match guard.as_ref() {
Some(pointer) if pointer.matches_dimensions(width as i32, height as i32) => {
return Ok(true)
}
Some(_) => {
*guard = None;
true
}
None => false,
}
};
self.create_abs_pointer_with(width, height, stale, crate::abs_pointer::AbsPointer::create)
.await
}
/// The uinput pointer for a KWin element click, sized to the capture
/// KWin mapped the element onto. A device that cannot be replaced fails
/// the click with a device message, not a mapping failure.
async fn kwin_abs_pointer_ready(
&self,
kwin_click: PreparedKwinElementClick,
received: &Option<serde_json::Value>,
) -> std::result::Result<bool, ActionOutput> {
let (width, height) = kwin_click.capture_size;
match self
.ensure_abs_pointer_for_kwin_capture(width, height)
.await
{
Ok(ready) => Ok(ready),
Err(failure) => Err(kwin_abs_pointer_not_sent(
kwin_click.element_index,
&failure,
self.pointer_retry_route().await,
received.clone(),
)),
}
}
async fn create_abs_pointer_with<F>(
&self,
width: u32,
height: u32,
stale: bool,
create: F,
) -> std::result::Result<bool, AbsPointerFailure>
where
F: FnOnce(i32, i32) -> Result<crate::abs_pointer::AbsPointer> + Send + 'static,
{
let created =
tokio::task::spawn_blocking(move || create(width as i32, height as i32)).await;
let detail = match created {
Ok(Ok(pointer)) => {
let mut guard = self
.abs_pointer
.lock()
.map_err(|_| AbsPointerFailure::lock_poisoned())?;
*guard = Some(pointer);
return Ok(true);
}
Ok(Err(error)) if stale => format!(
"cached absolute-pointer geometry was stale for the {width}x{height} raw capture and the device could not be recreated: {error:#}"
),
Err(error) if stale => format!(
"cached absolute-pointer geometry was stale for the {width}x{height} raw capture and the device recreation task failed: {error}"
),
_ => return Ok(false),
};
Err(AbsPointerFailure {
detail,
device_dropped: true,
})
}
/// Where a coordinate action goes once no uinput pointer can be made:
/// the routes click, scroll and drag try after the uinput block.
async fn pointer_retry_route(&self) -> &'static str {
if self.cached_portal_pointer_session().is_some()
|| self.should_prefer_portal_pointer_backend().await
{
"it goes through the remote desktop portal"
} else if self.is_wayland_session() {
"it is not sent, since ydotool cannot position the pointer accurately on Wayland"
} else {
"it goes through ydotool"
}
}
#[tool(
name = "click",
output_schema = rmcp::handler::server::tool::schema_for_output::<ActionToolOutput>(),
description = "Click an element by index, semantic selector, or desktop coordinate pixels from screenshot metadata. A plain left click on an element whose first AT-SPI action is click, press, or toggle invokes that action, avoiding toolkit coordinate scaling. Entry activate and slider jump actions are not substituted for pointer clicks. Explicit coordinates, right clicks, and multi-clicks retain pointer semantics. Explicit x/y default to raw screenshot capture pixels; coordinate_space=\"atspi\" with element_index or a semantic selector maps a global AT-SPI screen point through the element's KWin window and output geometry on KDE Wayland. A window target is raised and focused first (on GNOME Wayland, raising needs the Shell extension); nothing is sent when focus cannot be verified. Pass screenshot_after (true, \"screen\", or \"window\") or include_tree=true to return post-action observation in the same result.",
annotations(
read_only_hint = false,
destructive_hint = true,
idempotent_hint = false,
open_world_hint = true
)
)]
async fn click(
&self,
Parameters(params): Parameters<ClickParams>,
) -> Result<CallToolResult, ErrorData> {
let capture = params.capture.clone();
let target = params.window_target();
let (Json(output), scope) = self.dispatch_click(params).await;
self.finish_action_output(output, &capture, target.as_ref(), TargetRaised::Yes, scope)
.await
}
async fn dispatch_click(&self, params: ClickParams) -> (Json<ActionOutput>, PostActionScope) {
let received = Some(serde_json::json!(params.clone()));
let input_guard = Arc::clone(&self.input_operation_lock).lock_owned().await;
// Before the window is raised: a refused call changes nothing.
match self.check_element_snapshot(
"click",
params.snapshot_id.as_deref(),
click_resolves_cached_element(¶ms),
) {
Ok(scope) => (
self.dispatch_checked_click(params, received, input_guard)
.await,
scope,
),
Err(message) => (
Json(ActionOutput {
ok: false,
implemented: true,
action: "click".to_string(),
message,
received,
}),
PostActionScope::default(),
),
}
}
/// The rest of click, under the input lock the caller took.
async fn dispatch_checked_click(
&self,
mut params: ClickParams,
received: Option<serde_json::Value>,
input_guard: tokio::sync::OwnedMutexGuard<()>,
) -> Json<ActionOutput> {
let mut portal_target_point = None;
let mut input_focus = None;
let mut owner_target = ElementOwnerTarget::pid(params.pid);
if let Err(message) = validate_click_coordinate_space(¶ms) {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "click".to_string(),
message,
received,
});
}
if params.coordinate_space == ClickCoordinateSpace::Atspi
&& !self.should_map_kwin_element_click(None)
{
return Json(ActionOutput {
ok: false,
implemented: true,
action: "click".to_string(),
message: "AT-SPI coordinate clicks require a KWin Wayland window backend. Use screenshot capture coordinates on this session.".to_string(),
received,
});
}
// Raise the target window first (if specified) so the click lands on the
// intended app rather than whatever is stacked on top at that pixel.
let mut window_target = params.window_target();
if params.relative == Some(true) && window_target.is_none() {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "click".to_string(),
message: "Relative coordinate clicks require a window target.".to_string(),
received,
});
}
if let Some(target) = window_target.as_ref() {
let focus = match self.focus_target_for_input(target).await {
Ok(focus) => focus,
Err(message) => {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "click".to_string(),
message,
received,
});
}
};
owner_target = focus_owner_target(focus.as_ref(), params.pid, params.window_id);
tokio::time::sleep(Duration::from_millis(120)).await;
// Window-relative coordinates: translate by the window's top-left so
// the agent can click the pixel it saw in a window-cropped screenshot.
if params.relative == Some(true) {
let Some(focus) = focus.as_ref() else {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "click".to_string(),
message: "Relative coordinate clicks require verified target-window focus."
.to_string(),
received,
});
};
let coordinate_map = match self.focused_window_coordinate_map(focus).await {
Ok(mapping) => mapping,
Err(message) => {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "click".to_string(),
message,
received,
});
}
};
if let Err(message) = apply_window_relative_click_coordinates(
&mut params,
coordinate_map.capture_rect,
) {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "click".to_string(),
message,
received,
});
}
portal_target_point = params
.x
.zip(params.y)
.and_then(|(x, y)| coordinate_map.portal_point(x, y));
}
if let Some(focus) = focus.as_ref() {
window_target = Some(portal_focus_target(target, &focus.requested_window));
}
input_focus = focus;
}
if params.x.zip(params.y).is_none()
|| params.coordinate_space == ClickCoordinateSpace::Atspi
{
if let Err(message) = self
.check_cached_element_target(
params.element_index,
¶ms.selector(),
ElementResolvePurpose::Click,
&owner_target,
)
.await
{
return Json(ActionOutput {
ok: false,
implemented: true,
action: "click".to_string(),
message,
received,
});
}
}
let target = match self.resolve_click_target(¶ms) {
Ok(target) => target,
Err(message) => {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "click".to_string(),
message,
received,
});
}
};
if let ClickTarget::PrimaryAction {
object_ref,
action_name,
action_index,
} = target
{
self.indicator.action("click").await;
let invocation = if let Some(name) = action_name
.as_deref()
.filter(|name| !name.trim().is_empty())
{
perform_named_action(&object_ref, name).await
} else {
invoke_accessibility_action(&object_ref, Some(&action_index.to_string())).await
};
return match invocation {
Ok(invocation) => Json(ActionOutput {
ok: invocation.ok,
implemented: true,
action: "click".to_string(),
message: if invocation.ok {
format!(
"Invoked the primary AT-SPI action{} without pointer-coordinate conversion.",
action_name
.as_deref()
.filter(|name| !name.is_empty())
.map(|name| format!(" ({name})"))
.unwrap_or_default()
)
} else {
format!(
"The primary AT-SPI action{} returned false.",
action_name
.as_deref()
.filter(|name| !name.is_empty())
.map(|name| format!(" ({name})"))
.unwrap_or_default()
)
},
received,
}),
Err(error) => Json(ActionOutput {
ok: false,
implemented: true,
action: "click".to_string(),
message: error.to_string(),
received,
}),
};
}
let element_bounds = match &target {
ClickTarget::ElementBounds { node, .. } | ClickTarget::ElementPoint { node, .. } => {
node.bounds.clone()
}
_ => None,
};
let element_click = element_bounds.is_some();
let global_point = matches!(&target, ClickTarget::ElementPoint { .. });
let (x, y, kwin_click) = match target {
ClickTarget::Coordinates(x, y) => (x, y, None),
ClickTarget::ElementBounds { node, center }
| ClickTarget::ElementPoint {
node,
point: center,
} => {
if self.should_map_kwin_element_click(input_focus.as_ref()) {
match self
.prepare_kwin_element_click(
&node,
center,
global_point,
input_focus.as_ref(),
)
.await
{
Ok(prepared) => {
portal_target_point = Some(prepared.portal_point());
let (x, y) = prepared.uinput_point();
(x, y, Some(prepared))
}
Err(message) => {
return self.kwin_element_click_failure(node.index, &message, received);
}
}
} else if global_point {
return self.kwin_element_click_failure(
node.index,
"AT-SPI coordinate clicks require a KWin Wayland window backend; the focused window uses a different backend",
received,
);
} else {
(center.0, center.1, None)
}
}
ClickTarget::PrimaryAction { .. } => unreachable!(),
};
self.indicator
.pointer("click", (x, y), self.indicator_space())
.await;
let button = mouse_button_code(params.button.as_deref());
let click_count = params.click_count.unwrap_or(1).clamp(1, 10).to_string();
let input_guard = Arc::new(input_guard);
if let Some(output) = self
.try_pixelflux_click(
x,
y,
// KWin already mapped the element into capture pixels.
element_click && kwin_click.is_none(),
element_bounds.as_ref(),
params.button.as_deref(),
params.click_count.unwrap_or(1),
received.clone(),
Arc::clone(&input_guard),
)
.await
{
return Json(output);
}
let input_guard = Arc::try_unwrap(input_guard)
.expect("pixelflux fallback must not retain dispatched input");
if self.should_prefer_xdotool_pointer() {
if let Some(xdotool_args) = xdotool_pointer_click_args(
x,
y,
params.click_count.unwrap_or(1).clamp(1, 10),
params.button.as_deref(),
) {
let (input_guard, result) = run_cancellation_safe_input(input_guard, async move {
run_xdotool_pointer_or_fallback(Path::new("xdotool"), &xdotool_args, || async {
Err("Native X11 pointer input requires xdotool; no system-wide input fallback was used.".to_string())
})
.await
})
.await;
let _input_guard = input_guard;
let used_xdotool = result
.as_ref()
.is_ok_and(|result| result.backend == KeyboardCommandBackend::Xdotool);
let mut output =
action_result("click", result.map(|result| result.outputs), received);
if output.ok && used_xdotool {
output.message = "Action sent through xdotool (X11 XTEST).".to_string();
}
// Nothing was sent when the dispatch failed.
let off_screen_note = if output.ok {
self.off_screen_note_for_point(x, y, PointerRoute::X11)
.await
} else {
None
};
return Json(with_notes(output, off_screen_note));
}
return Json(ActionOutput {
ok: false,
implemented: true,
action: "click".to_string(),
message: "X11 click supports left, middle and right buttons.".to_string(),
received,
});
}
// Preferred backend: the uinput absolute pointer. Unlike ydotool's
// relative-only device (faked `--absolute` via pin-to-corner + relative
// move, which acceleration + fractional scaling distort) and unlike the
// portal (per-monitor coordinate scaling + an approval dialog), the
// absolute pointer uses screenshot-pixel coordinates directly and
// reports the point it emitted after desktop-edge clamping.
if let Some(abs_button) =
crate::abs_pointer::PointerButton::from_name(params.button.as_deref())
{
let abs_pointer_ready = if let Some(kwin_click) = kwin_click {
match self.kwin_abs_pointer_ready(kwin_click, &received).await {
Ok(ready) => ready,
Err(output) => return Json(output),
}
} else {
match self.ensure_abs_pointer().await {
Ok(ready) => ready,
Err(failure) => {
return Json(abs_pointer_not_sent(
"click",
&failure,
self.pointer_retry_route().await,
received,
))
}
}
};
if abs_pointer_ready {
let pointer = Arc::clone(&self.abs_pointer);
let count = params.click_count.unwrap_or(1).clamp(1, 10);
let (input_guard, result) = run_cancellation_safe_input(input_guard, async move {
run_abs_pointer(pointer, move |pointer| {
pointer.click(x, y, abs_button, count)
})
.await
})
.await;
let _input_guard = input_guard;
// KWin mapped an element point onto one of its outputs.
let off_screen_note = match kwin_click {
Some(_) => None,
None => self.uinput_off_screen_note(&result).await,
};
return Json(kwin_element_click_dispatch_output(
with_notes(
abs_pointer_result("click", result, received),
off_screen_note,
),
kwin_click,
));
}
}
// A portal release guard holds a clone, so a cancelled action keeps
// the lock until its button is up; this one keeps it until a failed
// session is cleared.
let input_guard = InputOperationGuard::new(input_guard);
if let Some(session) = self.cached_portal_pointer_session() {
// KWin mapped an element point onto one of its outputs.
let portal_off_screen_note = self
.portal_off_screen_note(kwin_click.is_none().then_some((x, y)))
.await;
let Some((portal_x, portal_y)) =
portal_target_point.or_else(|| self.logical_portal_point(&session, x, y))
else {
self.clear_portal_pointer_session(&session);
return Json(with_notes(
portal_coordinate_error("click", received),
portal_off_screen_note.clone(),
));
};
match portal_click(
&session,
portal_x,
portal_y,
PointerButton::from_name(params.button.as_deref()),
params.click_count.unwrap_or(1).clamp(1, 10),
input_guard.clone(),
)
.await
{
Ok(()) => {
return Json(kwin_element_click_dispatch_output(
with_notes(
ActionOutput {
ok: true,
implemented: true,
action: "click".to_string(),
message: "Action sent through the remote desktop portal."
.to_string(),
received,
},
portal_off_screen_note.clone(),
),
kwin_click,
));
}
Err(error) => {
self.clear_portal_pointer_session(&session);
return Json(with_notes(
portal_action_error("click", error, received),
portal_off_screen_note.clone(),
));
}
}
} else if self.should_prefer_portal_pointer_backend().await {
match self.ensure_portal_pointer_session().await {
Ok(Some(session)) => {
if let Some(target) = window_target.as_ref() {
if let Err(message) = self.focus_target_for_input(target).await {
return Json(action_result("click", Err(message), received));
}
}
let portal_off_screen_note = self
.portal_off_screen_note(kwin_click.is_none().then_some((x, y)))
.await;
let Some((portal_x, portal_y)) =
portal_target_point.or_else(|| self.logical_portal_point(&session, x, y))
else {
self.clear_portal_pointer_session(&session);
return Json(with_notes(
portal_coordinate_error("click", received),
portal_off_screen_note.clone(),
));
};
match portal_click(
&session,
portal_x,
portal_y,
PointerButton::from_name(params.button.as_deref()),
params.click_count.unwrap_or(1).clamp(1, 10),
input_guard.clone(),
)
.await
{
Ok(()) => {
return Json(kwin_element_click_dispatch_output(
with_notes(
ActionOutput {
ok: true,
implemented: true,
action: "click".to_string(),
message: "Action sent through the remote desktop portal."
.to_string(),
received,
},
portal_off_screen_note.clone(),
),
kwin_click,
));
}
Err(error) => {
self.clear_portal_pointer_session(&session);
return Json(with_notes(
portal_action_error("click", error, received),
portal_off_screen_note.clone(),
));
}
}
}
Ok(None) => {}
Err(error) => return Json(portal_start_error("click", error, received)),
}
}
if self.is_wayland_session() {
return Json(unsafe_pointer_fallback("click", received));
}
let commands = vec![
absolute_mousemove_args(x, y),
vec![
"click".to_string(),
"--repeat".to_string(),
click_count,
button,
],
];
let (input_guard, result) = run_cancellation_safe_input(input_guard, async move {
run_ydotool_sequence(&commands).await
})
.await;
let _input_guard = input_guard;
let output = action_result("click", result, received);
let off_screen_note = if output.ok {
self.off_screen_note_for_point(x, y, PointerRoute::X11)
.await
} else {
None
};
Json(with_notes(output, off_screen_note))
}
#[tool(
name = "perform_action",
output_schema = rmcp::handler::server::tool::schema_for_output::<ActionToolOutput>(),
description = "Invoke an accessibility action exposed by an element selected by index, identifier, or semantic selector. Defaults to the primary action unless action is provided. A stale semantic target can recover once with verified process/window ownership. Recovery invalidates cached indices; call get_app_state before further element_index actions. Pass the target app (pid, window_id, app_id, wm_class, or window_title, as for click) to refuse an element_index or selector from another app's or window's get_app_state snapshot, and an element_identifier owned by another pid; the window is not focused. Pass screenshot_after (true, \"screen\", or \"window\") or include_tree=true to return post-action observation in the same result.",
annotations(
read_only_hint = false,
destructive_hint = true,
idempotent_hint = false,
open_world_hint = true
)
)]
async fn perform_action(
&self,
Parameters(params): Parameters<ActionParams>,
) -> Result<CallToolResult, ErrorData> {
let capture = params.capture.clone();
let requested_action = requested_or_primary_action(params.action.as_deref());
let screenshot_target = params.target.window_target();
let (Json(output), target) = self
.perform_element_action(¶ms, Some(requested_action))
.await;
let (observation, screenshot, notes) = if output.ok {
self.capture_element_observation(&capture, screenshot_target.as_ref(), target.as_ref())
.await
} else {
(PostActionObservation::default(), None, Vec::new())
};
self.action_tool_result(
ActionToolOutput {
action: with_notes(output, notes),
observation,
},
screenshot,
)
}
#[tool(
name = "set_value",
output_schema = rmcp::handler::server::tool::schema_for_output::<SetValueToolOutput>(),
description = "Set the value of a settable accessibility element selected by index, identifier, or semantic selector. Editable-text writes are verified by exact readback; password fields are explicitly unverifiable. Pass the target app (pid, window_id, app_id, wm_class, or window_title, as for click) to refuse an element_index or selector from another app's or window's get_app_state snapshot, and an element_identifier owned by another pid; the window is not focused. Pass screenshot_after (true, \"screen\", or \"window\") or include_tree=true to return post-action observation in the same result.",
annotations(
read_only_hint = false,
destructive_hint = true,
idempotent_hint = false,
open_world_hint = true
)
)]
async fn set_value(
&self,
Parameters(params): Parameters<SetValueParams>,
) -> Result<CallToolResult, ErrorData> {
let capture = params.capture.clone();
let screenshot_target = params.target.window_target();
let (output, verified, target) = self.dispatch_set_value(params).await;
self.finish_set_value_output(
output,
verified,
&capture,
screenshot_target.as_ref(),
target.as_ref(),
)
.await
}
async fn dispatch_set_value(
&self,
params: SetValueParams,
) -> (
ActionOutput,
Option<bool>,
std::result::Result<ElementObservationTarget, String>,
) {
let _input_guard = Arc::clone(&self.input_operation_lock).lock_owned().await;
let received = Some(serde_json::json!(params.clone()));
// An element_identifier names the object itself, not a snapshot node.
match self.check_element_snapshot(
"set_value",
params.snapshot_id.as_deref(),
trimmed_nonempty(params.element_identifier.as_deref()).is_none()
&& (params.element_index.is_some() || !params.selector().is_empty()),
) {
// The post-action tree follows the element's bound owner instead.
Ok(_) => self.dispatch_checked_set_value(params, received).await,
Err(message) => (
ActionOutput {
ok: false,
implemented: true,
action: "set_value".to_string(),
message,
received,
},
None,
Ok(ElementObservationTarget::default()),
),
}
}
/// The rest of set_value; the caller holds the input lock.
async fn dispatch_checked_set_value(
&self,
params: SetValueParams,
received: Option<serde_json::Value>,
) -> (
ActionOutput,
Option<bool>,
std::result::Result<ElementObservationTarget, String>,
) {
if let Err(message) = self
.check_element_action_target(
¶ms.target,
params.element_index,
params.element_identifier.as_deref(),
¶ms.selector(),
ElementResolvePurpose::SetValue,
)
.await
{
return (
ActionOutput {
ok: false,
implemented: true,
action: "set_value".to_string(),
message,
received,
},
None,
Ok(ElementObservationTarget::default()),
);
}
let object_ref = match self.resolve_object_ref(
params.element_index,
params.element_identifier.as_deref(),
¶ms.selector(),
ElementResolvePurpose::SetValue,
) {
Ok(object_ref) => object_ref,
Err(message) => {
return (
ActionOutput {
ok: false,
implemented: true,
action: "set_value".to_string(),
message,
received,
},
None,
Ok(ElementObservationTarget::default()),
);
}
};
let target = if params.capture.include_tree.unwrap_or(false) {
self.element_observation_target(
¶ms.target,
&object_ref,
params.snapshot_id.as_deref(),
)
.await
} else {
Ok(ElementObservationTarget::default())
};
self.indicator.text("set_value", ¶ms.value, false).await;
let (output, verified) = match set_element_value(&object_ref, ¶ms.value).await {
Ok(ValueSetInvocation::Numeric { value }) => (
ActionOutput {
ok: true,
implemented: true,
action: "set_value".to_string(),
message: format!("AT-SPI numeric value set to {value}."),
received,
},
None,
),
Ok(ValueSetInvocation::EditableText { outcome }) => {
editable_text_action_result(outcome, received)
}
Err(error) => (
ActionOutput {
ok: false,
implemented: true,
action: "set_value".to_string(),
message: error.to_string(),
received,
},
None,
),
};
(output, verified, target)
}
#[tool(
name = "scroll",
output_schema = rmcp::handler::server::tool::schema_for_output::<ActionToolOutput>(),
description = "Scroll an element, window, or point in a direction by a number of pages (5 wheel notches per page). With a window target and no x/y/element_index, scrolls at the centre of the targeted window; with no point or target, at the current pointer. A window target is raised and focused first; nothing is sent when focus cannot be verified. Pass screenshot_after (true, \"screen\", or \"window\") or include_tree=true to return post-action observation in the same result.",
annotations(
read_only_hint = false,
destructive_hint = false,
idempotent_hint = false,
open_world_hint = true
)
)]
async fn scroll(
&self,
Parameters(params): Parameters<ScrollParams>,
) -> Result<CallToolResult, ErrorData> {
let capture = params.capture.clone();
let target = params.window_target();
let (Json(output), scope) = self.dispatch_scroll(params).await;
self.finish_action_output(output, &capture, target.as_ref(), TargetRaised::Yes, scope)
.await
}
async fn dispatch_scroll(&self, params: ScrollParams) -> (Json<ActionOutput>, PostActionScope) {
let received = Some(serde_json::json!(params.clone()));
let refused = |message| {
(
Json(ActionOutput {
ok: false,
implemented: true,
action: "scroll".to_string(),
message,
received: received.clone(),
}),
PostActionScope::default(),
)
};
let units = match scroll_units(params.pages) {
Ok(units) => units,
Err(message) => return refused(message),
};
let input_guard = Arc::clone(&self.input_operation_lock).lock_owned().await;
match self.check_element_snapshot(
"scroll",
params.snapshot_id.as_deref(),
scroll_resolves_cached_element(¶ms),
) {
Ok(scope) => (
self.dispatch_checked_scroll(params, units, received.clone(), input_guard)
.await,
scope,
),
Err(message) => refused(message),
}
}
/// The rest of scroll, under the input lock the caller took.
async fn dispatch_checked_scroll(
&self,
mut params: ScrollParams,
units: i32,
received: Option<serde_json::Value>,
input_guard: tokio::sync::OwnedMutexGuard<()>,
) -> Json<ActionOutput> {
let mut portal_target_point = None;
let mut owner_target = ElementOwnerTarget::pid(params.pid);
// Raise/focus the target window first (parity with click) so wheel
// events land on the intended app.
let mut window_target = params.window_target();
if params.relative == Some(true) && window_target.is_none() {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "scroll".to_string(),
message: "Relative scroll coordinates require a window target.".to_string(),
received,
});
}
if let Some(target) = window_target.as_ref() {
let focus = match self.focus_target_for_input(target).await {
Ok(focus) => focus,
Err(message) => {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "scroll".to_string(),
message,
received,
});
}
};
owner_target = focus_owner_target(focus.as_ref(), params.pid, params.window_id);
tokio::time::sleep(Duration::from_millis(120)).await;
if params.relative == Some(true) {
let Some(focus) = focus.as_ref() else {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "scroll".to_string(),
message:
"Relative scroll coordinates require verified target-window focus."
.to_string(),
received,
});
};
let coordinate_map = match self.focused_window_coordinate_map(focus).await {
Ok(mapping) => mapping,
Err(message) => {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "scroll".to_string(),
message,
received,
});
}
};
if let Err(message) = apply_window_relative_scroll_coordinates(
&mut params,
coordinate_map.capture_rect,
) {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "scroll".to_string(),
message,
received,
});
}
portal_target_point = params
.x
.zip(params.y)
.and_then(|(x, y)| coordinate_map.portal_point(x, y));
} else if params.x.is_none() && params.y.is_none() && params.element_index.is_none() {
// A window target without a point would otherwise scroll
// whatever happens to sit under the pointer: focusing does not
// move the cursor, and the wheel path never repositions it.
// Default to the centre of the resolved target window.
let Some(focus) = focus.as_ref() else {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "scroll".to_string(),
message: "Window-targeted scroll requires verified target-window focus."
.to_string(),
received,
});
};
let coordinate_map = match self.focused_window_coordinate_map(focus).await {
Ok(mapping) => mapping,
Err(message) => {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "scroll".to_string(),
message,
received,
});
}
};
if let Err(message) =
apply_window_center_scroll_point(&mut params, coordinate_map.capture_rect)
{
return Json(ActionOutput {
ok: false,
implemented: true,
action: "scroll".to_string(),
message,
received,
});
}
portal_target_point = params
.x
.zip(params.y)
.and_then(|(x, y)| coordinate_map.portal_point(x, y));
}
if let Some(focus) = focus.as_ref() {
window_target = Some(portal_focus_target(target, &focus.requested_window));
}
}
if scroll_resolves_cached_element(¶ms) {
if let Err(message) = self
.check_cached_element_target(
params.element_index,
&ElementSelector::default(),
ElementResolvePurpose::Click,
&owner_target,
)
.await
{
return Json(ActionOutput {
ok: false,
implemented: true,
action: "scroll".to_string(),
message,
received,
});
}
}
let target_point =
match self.resolve_optional_target_point(params.x, params.y, params.element_index) {
Ok(point) => point,
Err(message) => {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "scroll".to_string(),
message,
received,
});
}
};
let direction = match params.direction.to_ascii_lowercase().as_str() {
"up" => ScrollDirection::Up,
"down" => ScrollDirection::Down,
"left" => ScrollDirection::Left,
"right" => ScrollDirection::Right,
_ => {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "scroll".to_string(),
message: "Unsupported scroll direction; expected up, down, left, or right."
.to_string(),
received,
});
}
};
match target_point {
Some(point) => {
self.indicator
.pointer("scroll", point, self.indicator_space())
.await
}
None => self.indicator.action("scroll").await,
}
let (dx, dy) = wheel_deltas(direction, units);
// Element centres are AT-SPI logical points; pixelflux wants capture pixels.
let pixelflux_point = match (params.x.zip(params.y), params.element_index) {
(None, Some(index)) if crate::pixelflux::PixelfluxConfig::from_env().is_some() => {
let bounds = self.cached_node_bounds(index);
match self.pixelflux_element_point(bounds.as_ref()).await {
Some(point) => Some(point),
None => {
if let Some(output) =
self.pixelflux_unmapped("scroll", received.clone()).await
{
return Json(output);
}
None
}
}
}
_ => target_point,
};
let input_guard = Arc::new(input_guard);
if let Some(output) = self
.pixelflux_send(
"scroll",
crate::pixelflux::PixelfluxAction::Scroll {
point: pixelflux_point,
direction,
amount: u32::try_from(units.max(1)).unwrap_or(1),
},
received.clone(),
Arc::clone(&input_guard),
)
.await
{
return Json(output);
}
let input_guard = Arc::try_unwrap(input_guard)
.expect("pixelflux fallback must not retain dispatched input");
if self.should_prefer_xdotool_pointer() {
let repeat = u32::try_from(units.max(1)).unwrap_or(1);
let xdotool_args = xdotool_pointer_scroll_args(target_point, repeat, direction);
let command_timeout = xdotool_scroll_timeout(repeat, XDOTOOL_SCROLL_CLICK_DELAY_MS);
let (input_guard, result) = run_cancellation_safe_input(input_guard, async move {
run_xdotool_pointer_or_fallback_with_timeout(
Path::new("xdotool"),
&xdotool_args,
command_timeout,
|| async { Err("Native X11 pointer input requires xdotool; no system-wide input fallback was used.".to_string()) },
)
.await
})
.await;
let _input_guard = input_guard;
let used_xdotool = result
.as_ref()
.is_ok_and(|result| result.backend == KeyboardCommandBackend::Xdotool);
let mut output = action_result("scroll", result.map(|result| result.outputs), received);
if output.ok && used_xdotool {
output.message = "Action sent through xdotool (X11 XTEST).".to_string();
}
let off_screen_note = match target_point {
Some((x, y)) if output.ok => {
self.off_screen_note_for_point(x, y, PointerRoute::X11)
.await
}
_ => None,
};
return Json(with_notes(output, off_screen_note));
}
// Coordinate scroll needs the same accurate positioning as click. Keep
// the move and wheel event under one cancellation-safe input guard.
// The device sends both, so this route needs no ydotool daemon.
if let Some((x, y)) = target_point {
let abs_pointer_ready = if self.is_wayland_session() {
match self.ensure_abs_pointer().await {
Ok(ready) => ready,
Err(failure) => {
return Json(abs_pointer_not_sent(
"scroll",
&failure,
self.pointer_retry_route().await,
received,
))
}
}
} else {
false
};
if abs_pointer_ready {
let pointer = Arc::clone(&self.abs_pointer);
let (input_guard, result) = run_cancellation_safe_input(input_guard, async move {
// Move and wheel must come from the same device: a wheel
// event sent through ydotool right after this device jumps
// the cursor can be routed with a stale pointer position
// and dropped, so the window never scrolls.
run_abs_pointer(pointer, move |pointer| {
let landing = pointer.move_to(x, y)?;
// spawn_blocking context: a blocking sleep is fine, and
// it keeps the settle delay that the old ydotool path
// had between motion and wheel.
std::thread::sleep(Duration::from_millis(35));
pointer.wheel(dx, dy)?;
Ok(landing)
})
.await
})
.await;
let _input_guard = input_guard;
let off_screen_note = self.uinput_off_screen_note(&result).await;
return Json(with_notes(
abs_pointer_result("scroll", result, received),
off_screen_note,
));
}
}
let mut portal_start_failure = None;
if let Some(session) = self.cached_portal_pointer_session() {
let portal_off_screen_note = self.portal_off_screen_note(target_point).await;
let mapped_target = match (portal_target_point, target_point) {
(Some(point), _) => Some(Some(point)),
(None, Some((x, y))) => self.logical_portal_point(&session, x, y).map(Some),
(None, None) => Some(None),
};
let Some(portal_target_point) = mapped_target else {
self.clear_portal_pointer_session(&session);
return Json(with_notes(
portal_coordinate_error("scroll", received),
portal_off_screen_note.clone(),
));
};
let session_for_input = session.clone();
let (input_guard, result) = run_cancellation_safe_input(input_guard, async move {
portal_scroll(&session_for_input, portal_target_point, direction, units)
.await
.map_err(|error| format!("{error:#}"))
})
.await;
let _input_guard = input_guard;
match result {
Ok(()) => {
return Json(with_notes(
ActionOutput {
ok: true,
implemented: true,
action: "scroll".to_string(),
message: "Action sent through the remote desktop portal.".to_string(),
received,
},
portal_off_screen_note.clone(),
));
}
Err(error) => {
self.clear_portal_pointer_session(&session);
return Json(with_notes(
portal_action_error("scroll", anyhow::anyhow!(error), received),
portal_off_screen_note.clone(),
));
}
}
} else if self.should_prefer_portal_pointer_backend().await {
match self.ensure_portal_pointer_session().await {
Ok(Some(session)) => {
if let Some(target) = window_target.as_ref() {
if let Err(message) = self.focus_target_for_input(target).await {
return Json(action_result("scroll", Err(message), received));
}
}
let portal_off_screen_note = self.portal_off_screen_note(target_point).await;
let mapped_target = match (portal_target_point, target_point) {
(Some(point), _) => Some(Some(point)),
(None, Some((x, y))) => self.logical_portal_point(&session, x, y).map(Some),
(None, None) => Some(None),
};
let Some(portal_target_point) = mapped_target else {
self.clear_portal_pointer_session(&session);
return Json(with_notes(
portal_coordinate_error("scroll", received),
portal_off_screen_note.clone(),
));
};
let session_for_input = session.clone();
let (input_guard, result) =
run_cancellation_safe_input(input_guard, async move {
portal_scroll(&session_for_input, portal_target_point, direction, units)
.await
.map_err(|error| format!("{error:#}"))
})
.await;
let _input_guard = input_guard;
match result {
Ok(()) => {
return Json(with_notes(
ActionOutput {
ok: true,
implemented: true,
action: "scroll".to_string(),
message: "Action sent through the remote desktop portal."
.to_string(),
received,
},
portal_off_screen_note.clone(),
));
}
Err(error) => {
self.clear_portal_pointer_session(&session);
return Json(with_notes(
portal_action_error("scroll", anyhow::anyhow!(error), received),
portal_off_screen_note.clone(),
));
}
}
}
Ok(None) => {}
Err(error)
if target_point.is_some()
|| env_flag_enabled("COMPUTER_USE_LINUX_FORCE_PORTAL_POINTER") =>
{
return Json(portal_start_error("scroll", error, received));
}
Err(error) => portal_start_failure = Some(error),
}
}
if target_point.is_some() && self.is_wayland_session() {
return Json(unsafe_pointer_fallback("scroll", received));
}
let mut sequence = Vec::new();
if let Some((x, y)) = target_point {
sequence.push(absolute_mousemove_args(x, y));
}
sequence.push(wheel_mousemove_args(dx, dy));
// X11: ydotool's absolute move warps through (0, 0), so the pointer
// re-enters the window. GTK 3 resets its XI2 scroll valuators on that
// enter, and the one wheel event only re-baselines them (delta 0).
// The call reported success while the window did not move (#184).
// XTEST wheel buttons have no valuators, so GTK takes the button path.
let (input_guard, result) = run_cancellation_safe_input(input_guard, async move {
run_ydotool_sequence(&sequence).await
})
.await;
let _input_guard = input_guard;
// Outside COSMIC the portal is preferred only when ydotool looked
// unavailable, so this fallback usually fails too. Keep the portal
// error then: it may be a denied consent.
let off_screen_note = match target_point {
Some((x, y)) if result.is_ok() => {
self.off_screen_note_for_point(x, y, PointerRoute::X11)
.await
}
_ => None,
};
let portal_note = portal_start_failure
.filter(|_| result.is_err())
.map(|error| format!("The scroll fell back to ydotool because the remote desktop portal could not start: {error:#}"));
Json(with_notes(
action_result("scroll", result, received),
portal_note.into_iter().chain(off_screen_note),
))
}
#[tool(
name = "drag",
output_schema = rmcp::handler::server::tool::schema_for_output::<ActionToolOutput>(),
description = "Drag with the left button from one point to another in screenshot capture pixels. Takes no window target or relative coordinates. Pass screenshot_after (true, \"screen\", or \"window\") or include_tree=true to return post-action observation in the same result.",
annotations(
read_only_hint = false,
destructive_hint = true,
idempotent_hint = false,
open_world_hint = true
)
)]
async fn drag(
&self,
Parameters(params): Parameters<DragParams>,
) -> Result<CallToolResult, ErrorData> {
let capture = params.capture.clone();
let Json(output) = self.dispatch_drag(params).await;
self.finish_action_output(
output,
&capture,
None,
TargetRaised::Yes,
PostActionScope::default(),
)
.await
}
async fn dispatch_drag(&self, params: DragParams) -> Json<ActionOutput> {
let received = Some(serde_json::json!(params));
let input_guard = Arc::clone(&self.input_operation_lock).lock_owned().await;
let space = self.indicator_space();
self.indicator
.pointer("drag", (params.start_x, params.start_y), space)
.await;
// Lead the overlay cursor to the drop point while the drag runs.
self.indicator
.follow("drag", (params.end_x, params.end_y), space)
.await;
let input_guard = Arc::new(input_guard);
if let Some(output) = self
.pixelflux_send(
"drag",
crate::pixelflux::PixelfluxAction::Drag {
from: (params.start_x, params.start_y),
to: (params.end_x, params.end_y),
},
received.clone(),
Arc::clone(&input_guard),
)
.await
{
return Json(output);
}
let input_guard = Arc::try_unwrap(input_guard)
.expect("pixelflux fallback must not retain dispatched input");
let start = (params.start_x, params.start_y);
let end = (params.end_x, params.end_y);
if self.should_prefer_xdotool_pointer() {
let (input_guard, result) = run_cancellation_safe_input(input_guard, async move {
run_xdotool_drag(Path::new("xdotool"), start, end).await
})
.await;
let _input_guard = input_guard;
let mut output = action_result("drag", result, received);
// A failed drag gets no note, as a failed click does.
if !output.ok {
return Json(output);
}
output.message =
"Drag dispatched through xdotool (X11 XTEST); verify the drop target.".to_string();
let notes = self
.drag_off_screen_notes(start, end, PointerRoute::X11)
.await;
return Json(with_notes(output, notes));
}
// Preferred backend: the uinput absolute pointer (accurate landing).
let abs_pointer_ready = match self.ensure_abs_pointer().await {
Ok(ready) => ready,
Err(failure) => {
return Json(abs_pointer_not_sent(
"drag",
&failure,
self.pointer_retry_route().await,
received,
))
}
};
if abs_pointer_ready {
let pointer = Arc::clone(&self.abs_pointer);
let (input_guard, result) = run_cancellation_safe_input(input_guard, async move {
run_abs_pointer(pointer, move |pointer| {
pointer.drag(start, end, crate::abs_pointer::PointerButton::Left)
})
.await
})
.await;
let _input_guard = input_guard;
let notes = self.uinput_drag_notes(&result).await;
return Json(match result {
Ok(_) => with_notes(
ActionOutput {
ok: true,
implemented: true,
action: "drag".to_string(),
message: "Action sent through the uinput absolute pointer.".to_string(),
received,
},
notes,
),
Err(error) => abs_pointer_error("drag", error, received),
});
}
// A portal release guard holds a clone, so a cancelled action keeps
// the lock until its button is up; this one keeps it until a failed
// session is cleared.
let input_guard = InputOperationGuard::new(input_guard);
if let Some(session) = self.cached_portal_pointer_session() {
let _ = self.capture_space_rect().await;
// The portal sends nothing unless both endpoints are on a shared
// monitor, so the notes say which endpoint stopped it.
let portal_notes = self
.drag_off_screen_notes(start, end, PointerRoute::Portal)
.await;
let Some((start_x, start_y)) = self.logical_portal_point(&session, start.0, start.1)
else {
self.clear_portal_pointer_session(&session);
return Json(with_notes(
portal_coordinate_error("drag", received),
portal_notes,
));
};
let Some((end_x, end_y)) = self.logical_portal_point(&session, end.0, end.1) else {
self.clear_portal_pointer_session(&session);
return Json(with_notes(
portal_coordinate_error("drag", received),
portal_notes,
));
};
match portal_drag(
&session,
start_x,
start_y,
end_x,
end_y,
input_guard.clone(),
)
.await
{
Ok(()) => {
return Json(with_notes(
ActionOutput {
ok: true,
implemented: true,
action: "drag".to_string(),
message: "Action sent through the remote desktop portal.".to_string(),
received,
},
portal_notes,
));
}
Err(error) => {
self.clear_portal_pointer_session(&session);
return Json(with_notes(
portal_action_error("drag", error, received),
portal_notes,
));
}
}
} else if self.should_prefer_portal_pointer_backend().await {
let _ = self.capture_space_rect().await;
match self.ensure_portal_pointer_session().await {
Ok(Some(session)) => {
let portal_notes = self
.drag_off_screen_notes(start, end, PointerRoute::Portal)
.await;
let Some((start_x, start_y)) =
self.logical_portal_point(&session, start.0, start.1)
else {
self.clear_portal_pointer_session(&session);
return Json(with_notes(
portal_coordinate_error("drag", received),
portal_notes,
));
};
let Some((end_x, end_y)) = self.logical_portal_point(&session, end.0, end.1)
else {
self.clear_portal_pointer_session(&session);
return Json(with_notes(
portal_coordinate_error("drag", received),
portal_notes,
));
};
match portal_drag(
&session,
start_x,
start_y,
end_x,
end_y,
input_guard.clone(),
)
.await
{
Ok(()) => {
return Json(with_notes(
ActionOutput {
ok: true,
implemented: true,
action: "drag".to_string(),
message: "Action sent through the remote desktop portal."
.to_string(),
received,
},
portal_notes,
));
}
Err(error) => {
self.clear_portal_pointer_session(&session);
return Json(with_notes(
portal_action_error("drag", error, received),
portal_notes,
));
}
}
}
Ok(None) => {}
Err(error) => return Json(portal_start_error("drag", error, received)),
}
}
if self.is_wayland_session() {
return Json(unsafe_pointer_fallback("drag", received));
}
let (input_guard, result) = run_cancellation_safe_input(input_guard, async move {
run_ydotool_drag(start.0, start.1, end.0, end.1).await
})
.await;
let _input_guard = input_guard;
let output = action_result("drag", result, received);
// Not Wayland, so this is the X11 route, as for click.
let notes = if output.ok {
self.drag_off_screen_notes(start, end, PointerRoute::X11)
.await
} else {
Vec::new()
};
Json(with_notes(output, notes))
}
#[tool(
name = "press_key",
output_schema = rmcp::handler::server::tool::schema_for_output::<ActionToolOutput>(),
description = "Press a key or key-combination on the keyboard, optionally after focusing a target window or terminal selector. Key grammar (case-insensitive; hyphens/spaces ignored): combos join with '+', e.g. Ctrl+L or Ctrl+Shift+T. Modifiers: ctrl/control, alt/option, shift, meta/super/cmd/command. Named keys: enter/return, escape/esc, tab, backspace, delete/del, space, home, end, pageup, pagedown, arrowleft/left, arrowright/right, arrowup/up, arrowdown/down, f1-f12. Plus single US letters a-z and digits 0-9. Anything else returns an error (never silently dropped). With PIXELFLUX_CU and PIXELFLUX_CU_TOKEN set, chords go to pixelflux. An unreachable pixelflux falls through to the routes below; a pixelflux error does not. On X11, chords go through xdotool (XTEST), or ydotool when xdotool is missing or cannot start. On Wayland, chords are sent through an active remote desktop portal keyboard session when one is available (or when ydotool is absent), falling back to ydotool otherwise. When ydotool is absent and the session has no RemoteDesktop portal (for example wlroots compositors), chords go through wtype instead. Portal chords send modifiers and named keys as keysyms so remapped keys (e.g. Caps Lock swapped with Control) follow the active keymap; letters and digits are physical US positions. On KDE Plasma every key in a portal chord is a physical keycode. Note: compositor-level shortcuts (e.g. Super+Up) may be consumed by GNOME before reaching the app. Pass screenshot_after (true, \"screen\", or \"window\") or include_tree=true to return post-action observation in the same result.",
annotations(
read_only_hint = false,
destructive_hint = true,
idempotent_hint = false,
open_world_hint = true
)
)]
async fn press_key(
&self,
Parameters(params): Parameters<PressKeyParams>,
) -> Result<CallToolResult, ErrorData> {
let capture = params.capture.clone();
let target = params.window_target();
let Json(output) = self.dispatch_press_key(params).await;
let raised = keyboard_target_raised(&target);
self.finish_action_output(
output,
&capture,
Some(&target),
raised,
PostActionScope::default(),
)
.await
}
async fn dispatch_press_key(&self, params: PressKeyParams) -> Json<ActionOutput> {
let received = Some(serde_json::json!(params.clone()));
let input_guard =
InputOperationGuard::new(Arc::clone(&self.input_operation_lock).lock_owned().await);
if let Some(output) = self
.hyprland_background_input(
"press_key",
¶ms.window_target(),
¶ms.key,
// The shared press_key grammar decides what is a key, so
// background mode accepts exactly what every backend accepts.
key_chord(¶ms.key)
.and_then(|_| crate::hyprland_input::chord_to_key_press(¶ms.key))
.map(|press| vec![press])
.ok_or_else(|| {
"Unsupported key. Use names like Enter, Escape, Tab, ArrowLeft, Super, Ctrl+L, or a single US keyboard letter/digit.".to_string()
}),
received.clone(),
)
.await
{
return Json(output);
}
let focus = match self.focus_target_for_input(¶ms.window_target()).await {
Ok(focus) => focus,
Err(message) => {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "press_key".to_string(),
message,
received,
});
}
};
let Some((chord_modifiers, chord_key)) = key_chord(¶ms.key) else {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "press_key".to_string(),
message: "Unsupported key. Use names like Enter, Escape, Tab, ArrowLeft, Super, Ctrl+L, or a single US keyboard letter/digit.".to_string(),
received,
});
};
let secret = self.typing_into_secret(focus.as_ref()).await;
self.indicator.keys(¶ms.key, secret).await;
let input_guard = Arc::new(input_guard);
if let Some(chord) = xdotool_key_spec(¶ms.key) {
if let Some(output) = self
.pixelflux_send(
"press_key",
crate::pixelflux::PixelfluxAction::Key { chord },
received.clone(),
Arc::clone(&input_guard),
)
.await
{
return Json(output);
}
}
let input_guard = Arc::try_unwrap(input_guard)
.expect("pixelflux fallback must not retain dispatched input");
let route = self.chord_route().await;
if route == ChordRoute::Wtype {
let Some(args) = wtype_key_args(¶ms.key) else {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "press_key".to_string(),
message: "Unsupported key. Use names like Enter, Escape, Tab, ArrowLeft, Super, Ctrl+L, or a single US keyboard letter/digit.".to_string(),
received,
});
};
let (input_guard, result) =
run_cancellation_safe_input(input_guard, async move { run_wtype_key(&args).await })
.await;
let _input_guard = input_guard;
let output = match result {
Ok(()) => successful_action_with_focus(
"press_key",
"Action sent through wtype (Wayland virtual-keyboard protocol).",
received,
focus.clone(),
),
Err(error) => {
action_result_with_focus("press_key", Err(error), received, focus.clone())
}
};
if output.ok && focus.is_some() {
let notes = self.input_landing_notes(focus.as_ref(), false).await;
return Json(with_notes(output, notes));
}
return Json(output);
}
if route == ChordRoute::Portal {
let portal_target = match self
.portal_keyboard_target(¶ms.window_target(), focus.as_ref())
.await
{
Ok(target) => target,
Err(error) => {
return Json(action_result_with_focus(
"press_key",
Err(format!("{error:#}")),
received,
focus,
))
}
};
match self.ensure_portal_keyboard_session().await {
Ok(Some(session)) => {
let focus = match self.focus_target_for_input(&portal_target).await {
Ok(focus) => focus,
Err(message) => {
return Json(action_result("press_key", Err(message), received))
}
};
// KDE keeps physical keycodes: its keysym path is untested
// here, and #191 was reported against mutter.
let (modifiers, key) = portal_key_chord(¶ms.key)
.filter(|_| !self.is_kde_wayland_session())
.unwrap_or_else(|| {
let modifiers = chord_modifiers
.iter()
.map(|m| PortalKey::Keycode(i32::from(*m)))
.collect();
(modifiers, PortalKey::Keycode(i32::from(chord_key)))
});
// The release cleanup holds a clone, so a cancelled chord
// keeps the lock until its keys are up; this one keeps it
// through the landing notes.
match press_key_chord(&session, &modifiers, key, input_guard.clone()).await {
Ok(()) => {
let notes = self.input_landing_notes(focus.as_ref(), false).await;
return Json(with_notes(
successful_action_with_focus(
"press_key",
"Action sent through the remote desktop portal.",
received,
focus,
),
notes,
));
}
Err(error) => {
self.clear_portal_keyboard_session(&session);
return Json(action_result_with_focus(
"press_key",
Err(format!("{error:#}")),
received,
focus,
));
}
}
}
Ok(None) => {}
Err(error) => return Json(portal_start_error("press_key", error, received)),
}
}
let Some(key_events) = key_sequence(¶ms.key) else {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "press_key".to_string(),
message: "Unsupported key. Use names like Enter, Escape, Tab, ArrowLeft, Super, Ctrl+L, or a single US keyboard letter/digit.".to_string(),
received,
});
};
// X11: prefer xdotool/XTEST. ydotool's raw evdev scancodes get
// re-mapped by the active XKB layout on X11, so named keys and chords
// arrive as stray glyphs instead of real key events (issue #58).
if self.should_prefer_xdotool_keyboard() {
if let Some(spec) = xdotool_key_spec(¶ms.key) {
let xdotool_args = vec!["key".to_string(), "--clearmodifiers".to_string(), spec];
let ydotool_args =
ydotool_key_args(key_events.clone(), !chord_modifiers.is_empty());
let (input_guard, result) = run_cancellation_safe_input(input_guard, async move {
run_xdotool_or_fallback(Path::new("xdotool"), &xdotool_args, || {
run_ydotool(&ydotool_args)
})
.await
})
.await;
let _input_guard = input_guard;
let used_xdotool = result
.as_ref()
.is_ok_and(|result| result.backend == KeyboardCommandBackend::Xdotool);
let mut output = action_result_with_focus(
"press_key",
result.map(|result| vec![result.output]),
received,
focus.clone(),
);
if used_xdotool {
output.message = "Action sent through xdotool (X11 XTEST).".to_string();
}
if output.ok && focus.is_some() {
let notes = self.input_landing_notes(focus.as_ref(), false).await;
output = with_notes(output, notes);
}
return Json(output);
}
}
let args = ydotool_key_args(key_events, !chord_modifiers.is_empty());
let (input_guard, result) = run_cancellation_safe_input(input_guard, async move {
run_ydotool(&args).await.map(|output| vec![output])
})
.await;
let _input_guard = input_guard;
let mut output = action_result_with_focus("press_key", result, received, focus.clone());
if output.ok && focus.is_some() {
let notes = self.input_landing_notes(focus.as_ref(), false).await;
output = with_notes(output, notes);
}
Json(output)
}
#[tool(
name = "run_shell",
description = "Execute one explicitly approved /bin/sh command with same-user host authority. This tool is absent unless the server operator starts computer-use-linux with COMPUTER_USE_LINUX_ENABLE_SHELL=1. It is not sandboxed: the command can read or modify files and use the network with the server user's permissions. The inherited environment is cleared to a small desktop/runtime allowlist; pass any additional variables explicitly. Execution time and output are bounded: returned streams are truncated to 512 KiB, while a stream exceeding the 8 MiB collection ceiling fails the call without returning partial output. An audit digest is written to server stderr.",
annotations(
read_only_hint = false,
destructive_hint = true,
idempotent_hint = false,
open_world_hint = true
)
)]
async fn run_shell(
&self,
Parameters(params): Parameters<RunShellParams>,
) -> Json<RunShellOutput> {
Json(execute_shell(params).await)
}
#[tool(
name = "type_text",
output_schema = rmcp::handler::server::tool::schema_for_output::<ActionToolOutput>(),
description = "Type literal text using keyboard input, optionally after focusing a target window or terminal selector. Keyboard success confirms dispatch, not field-content verification; re-observe the target before retrying. On GNOME, when a configured keyboard layout group cannot type a character of the text, the text goes into the focused editable field of the verified target window through AT-SPI EditableText instead: inserted at the caret, replacing a selection, and reported as ok only when the field reads it back with the rest unchanged. Password fields, text with a newline, tab or other control character, and focus outside an editable field of that active window keep the refusal. X11 xdotool can silently substitute symbols under non-US or changing XKB layouts; inspect the text before pressing Enter or submitting it. The wtype backend rejects NUL and text over 131071 UTF-8 bytes before focus or indicators. On Hyprland, confirmed password fields are rejected, and ambiguous fields and terminals use private stdin for at most 100 Unicode characters per call. Pass screenshot_after (true, \"screen\", or \"window\") or include_tree=true to return post-action observation in the same result.",
annotations(
read_only_hint = false,
destructive_hint = true,
idempotent_hint = false,
open_world_hint = true
)
)]
async fn type_text(
&self,
Parameters(params): Parameters<TypeTextParams>,
) -> Result<CallToolResult, ErrorData> {
let capture = params.capture.clone();
let target = params.window_target();
let Json(mut output) = self.dispatch_type_text(params).await;
if output.ok && !output.message.starts_with(TYPE_TEXT_ATSPI_ROUTE) {
output.message.push_str(
" Field contents were not verified; re-observe the target before retrying.",
);
}
let raised = keyboard_target_raised(&target);
self.finish_action_output(
output,
&capture,
Some(&target),
raised,
PostActionScope::default(),
)
.await
}
/// GNOME's portal route refused the text because a layout group lacks a
/// character. Insert it into the target's focused field through AT-SPI
/// EditableText instead, and succeed only when the field reads it back.
/// Every other case keeps the portal refusal.
async fn type_text_through_atspi(
&self,
text: &str,
portal_target: &WindowTarget,
missing: crate::keyboard_keymap::MissingLayoutKeysym,
refusal: String,
received: Option<serde_json::Value>,
input_guard: InputOperationGuard,
) -> ActionOutput {
if let Some(key) = key_press_character(text) {
return action_result(
"type_text",
Err(type_text_control_character_refusal(&refusal, key)),
received,
);
}
let focus = match self.focus_target_for_input(portal_target).await {
Ok(focus) => focus,
Err(message) => {
return action_result(
"type_text",
Err(format!("{refusal} No AT-SPI text fallback: {message}")),
received,
)
}
};
// The window the compositor verified as focused: the field must be in it.
let window = focus.as_ref().map(|focus| {
focus
.focused_window
.as_ref()
.unwrap_or(&focus.requested_window)
});
let title = window.and_then(|window| window.title.clone());
let Some(pid) = window.and_then(|window| window.pid) else {
return action_result_with_focus(
"type_text",
Err(format!(
"{refusal} No AT-SPI text fallback: the target window has no process id to find its focused field."
)),
received,
focus,
);
};
// A selection replace is DeleteText then InsertText. Finish both even
// if this call is cancelled, and hold the input lock until they end.
let owned_text = text.to_string();
let (_guard, insert) = run_cancellation_safe_guarded(input_guard, async move {
crate::atspi_tree::insert_text_at_focus(pid, title.as_deref(), &owned_text).await
})
.await;
let insert = insert.unwrap_or_else(|error| {
Ok(FocusedTextInsert::EditFailed {
reason: error,
may_have_changed: true,
})
});
match type_text_atspi_result(&refusal, missing, text, insert) {
Ok(message) => successful_action_with_focus("type_text", &message, received, focus),
Err(message) => action_result_with_focus("type_text", Err(message), received, focus),
}
}
async fn dispatch_type_text(&self, params: TypeTextParams) -> Json<ActionOutput> {
let received = Some(serde_json::json!(params.clone()));
// Reject before focus changes or any indicator can display the text.
let wtype_mode = self.wtype_text_mode();
if let Err(message) = preflight_type_text(¶ms.text, wtype_mode) {
return Json(action_result("type_text", Err(message), received));
}
let input_guard =
InputOperationGuard::new(Arc::clone(&self.input_operation_lock).lock_owned().await);
let window_target = params.window_target();
if let Some(output) = self
.hyprland_background_input(
"type_text",
&window_target,
¶ms.text,
crate::hyprland_input::text_to_key_presses(¶ms.text),
received.clone(),
)
.await
{
return Json(output);
}
let focus = match self.focus_target_for_input(&window_target).await {
Ok(focus) => focus,
Err(message) => {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "type_text".to_string(),
message,
received,
});
}
};
let hyprland_argument_mode = wtype_mode == Some(WtypeTextMode::Argument);
let (wtype_mode, secret) = if hyprland_argument_mode {
match hyprland_wtype_delivery(¶ms.text, self.input_focus_probe(focus.as_ref()))
.await
{
Ok(mode) => (Some(mode), mode == WtypeTextMode::Stdin),
Err(message) => {
return Json(action_result_with_focus(
"type_text",
Err(message),
received,
focus,
))
}
}
} else {
(wtype_mode, self.typing_into_secret(focus.as_ref()).await)
};
if let Some(mode) = wtype_mode {
if let Err(message) = validate_wtype_privacy(mode, secret) {
return Json(action_result_with_focus(
"type_text",
Err(message),
received,
focus,
));
}
}
self.indicator.text("type_text", ¶ms.text, secret).await;
let input_guard = Arc::new(input_guard);
if let Some(output) = self
.pixelflux_send(
"type_text",
crate::pixelflux::PixelfluxAction::Type {
text: params.text.clone(),
},
received.clone(),
Arc::clone(&input_guard),
)
.await
{
return Json(output);
}
let mut input_guard = Arc::try_unwrap(input_guard)
.expect("pixelflux fallback must not retain dispatched input");
if self.should_prefer_kde_clipboard_text_backend() {
let portal_target = match self
.portal_keyboard_target(&window_target, focus.as_ref())
.await
{
Ok(target) => target,
Err(error) => {
return Json(action_result_with_focus(
"type_text",
Err(format!("{error:#}")),
received,
focus,
))
}
};
match self.ensure_portal_keyboard_session().await {
Ok(Some(session)) => {
let kde_focus = match self.focus_target_for_input(&portal_target).await {
Ok(focus) => focus,
Err(message) => {
return Json(action_result("type_text", Err(message), received))
}
};
let paste_shortcut =
kde_clipboard_paste_shortcut(&window_target, kde_focus.as_ref()).await;
let clipboard_guard = Arc::clone(&self.kde_clipboard_lock).lock_owned().await;
let session_for_input = session.clone();
let text = params.text.clone();
let portal_operation_guard = input_guard.clone();
// The paste puts the user's clipboard back after the
// shortcut. Run it to the end even if this call is
// cancelled, and hold both locks until it has.
let (guards, guarded_result) =
run_cancellation_safe_guarded((input_guard, clipboard_guard), async move {
run_kde_clipboard_paste_text(
&session_for_input,
&text,
paste_shortcut,
portal_operation_guard,
)
.await
})
.await;
let (returned_input_guard, clipboard_guard) = match guards {
Some(guards) => guards,
None => {
return Json(action_result_with_focus(
"type_text",
Err(guarded_result.unwrap_err()),
received,
kde_focus,
));
}
};
input_guard = returned_input_guard;
drop(clipboard_guard);
let result = guarded_result.expect("guarded task returned its guards");
match result {
Ok(message) => {
let notes = self.input_landing_notes(kde_focus.as_ref(), true).await;
return Json(with_notes(
successful_action_with_focus(
"type_text",
&message,
received,
kde_focus,
),
notes,
));
}
Err(error) => {
if error.clear_portal_keyboard_session {
self.clear_portal_keyboard_session(&session);
}
if !error.can_fallback_to_ydotool {
return Json(action_result_with_focus(
"type_text",
Err(error.message),
received,
kde_focus,
));
}
}
}
}
Ok(None) => {}
Err(error) => return Json(portal_start_error("type_text", error, received)),
}
}
if self.should_prefer_portal_text_backend() {
let mut focus = focus;
let mut layout_refusal = None;
let result = async {
let portal_target = self
.portal_keyboard_target(&window_target, focus.as_ref())
.await
.map_err(|error| format!("{error:#}"))?;
let keysyms = keysyms_for_text(¶ms.text)
.map_err(|error| format!("Literal text was not sent: {error:#}"))?;
if self.is_gnome_wayland_session() {
if let Err(error) =
crate::keyboard_keymap::validate_gnome_text(¶ms.text, &keysyms).await
{
let message = format!("Literal text was not sent: {error:#}");
if let Some(missing) = atspi_fallback_trigger(&error) {
layout_refusal = Some((portal_target, missing, message.clone()));
}
return Err(message);
}
}
let session = self
.ensure_portal_keyboard_session()
.await
.map_err(|error| format!("Portal text input could not start: {error:#}"))?
.ok_or_else(|| {
"Portal keyboard input is unavailable; no text was sent.".to_string()
})?;
focus = self.focus_target_for_input(&portal_target).await?;
type_text_with_keysyms(&session, &keysyms, input_guard.clone())
.await
.map_err(|error| {
self.clear_portal_keyboard_session(&session);
format!("{error:#}")
})
}
.await;
if let Some((portal_target, missing, refusal)) = layout_refusal {
return Json(
self.type_text_through_atspi(
¶ms.text,
&portal_target,
missing,
refusal,
received,
input_guard,
)
.await,
);
}
return Json(match result {
Ok(()) => {
let notes = self.input_landing_notes(focus.as_ref(), true).await;
with_notes(
successful_action_with_focus(
"type_text",
"Action sent through the remote desktop portal.",
received,
focus,
),
notes,
)
}
Err(error) => action_result_with_focus("type_text", Err(error), received, focus),
});
}
// X11: xdotool type resolves keysyms against the live XKB layout.
// ydotool's raw scancodes get re-mapped by X11 and mangle symbols and
// digits (`_` → `%`, `1` → `+`) even on a plain US layout (issue #58).
if self.should_prefer_xdotool_keyboard() {
let delay_ms = xdotool_type_delay_ms();
let args = xdotool_type_args_with_delay(¶ms.text, delay_ms);
let command_timeout = xdotool_type_timeout(¶ms.text, delay_ms);
let text = params.text.clone();
let (input_guard, result) = run_cancellation_safe_input(input_guard, async move {
run_xdotool_type_with_input_check(
Path::new("xdotool"),
&args,
command_timeout,
|| run_ydotool_type_text(&text),
)
.await
})
.await;
let _input_guard = input_guard;
let used_xdotool = result
.as_ref()
.is_ok_and(|(result, _)| result.backend == KeyboardCommandBackend::Xdotool);
let layout_warning = result
.as_ref()
.ok()
.and_then(|(_, warning)| warning.clone());
let mut output = action_result_with_focus(
"type_text",
result.map(|(result, _)| vec![result.output]),
received,
focus.clone(),
);
if used_xdotool {
output.message = "Action sent through xdotool (X11 XTEST). WARNING: X11 keyboard layout fidelity is unverified. xdotool can silently substitute symbols under non-US XKB layouts (reported with br/ABNT2) or changing keymap state. Re-observe the text before pressing Enter, submitting it, or treating it as delivered; do not retry blindly.".to_string();
}
if let Some(warning) = layout_warning {
output = with_notes(output, vec![warning]);
}
if output.ok && focus.is_some() {
let notes = self.input_landing_notes(focus.as_ref(), true).await;
output = with_notes(output, notes);
}
return Json(output);
}
if let Some(mode) = wtype_mode {
let text = params.text.clone();
let (input_guard, result) = run_cancellation_safe_input(input_guard, async move {
run_wtype_type_text_or_fallback(Path::new("wtype"), &text, mode, secret, || {
run_ydotool_type_text(&text)
})
.await
})
.await;
let _input_guard = input_guard;
let used_wtype = result
.as_ref()
.is_ok_and(|result| result.backend == KeyboardCommandBackend::Wtype);
let mut output = action_result_with_focus(
"type_text",
result.map(|result| vec![result.output]),
received,
focus.clone(),
);
if used_wtype {
output.message =
"Action sent through wtype (Wayland virtual-keyboard protocol).".to_string();
if hyprland_argument_mode && mode == WtypeTextMode::Stdin {
output.message.push_str(" AT-SPI could not classify the field; text was sent through stdin, limited to 100 Unicode characters per call.");
}
}
if output.ok && focus.is_some() {
let notes = self.input_landing_notes(focus.as_ref(), true).await;
output = with_notes(output, notes);
}
return Json(output);
}
let text = params.text.clone();
let (input_guard, result) = run_cancellation_safe_input(input_guard, async move {
run_ydotool_type_text(&text)
.await
.map(|output| vec![output])
})
.await;
let _input_guard = input_guard;
let mut output = action_result_with_focus("type_text", result, received, focus.clone());
if output.ok && focus.is_some() {
let notes = self.input_landing_notes(focus.as_ref(), true).await;
output = with_notes(output, notes);
}
Json(output)
}
#[tool(
name = "move_window",
output_schema = rmcp::handler::server::tool::schema_for_output::<WindowGeometryToolOutput>(),
description = "Move a window to a new desktop position (frame top-left in desktop coordinates). Useful to recover windows that are partially off-screen. Works only through the computer-use-linux GNOME Shell extension or a generic X11/EWMH window manager (wmctrl); other window backends refuse it. Pass screenshot_after (true, \"screen\", or \"window\") or include_tree=true to return post-action observation in the same result.",
annotations(
read_only_hint = false,
destructive_hint = false,
idempotent_hint = true,
open_world_hint = true
)
)]
async fn move_window(
&self,
Parameters(params): Parameters<MoveWindowParams>,
) -> Result<CallToolResult, ErrorData> {
let capture = params.target.capture.clone();
let received = Some(serde_json::json!(params.clone()));
let target = params.target.into_target();
self.indicator.action("move_window").await;
let Json(output) = self
.window_geometry_op(received, &target, |window| async move {
registry::move_window(&window, params.x, params.y).await
})
.await;
self.finish_window_geometry_output(output, &capture, &target)
.await
}
#[tool(
name = "resize_window",
output_schema = rmcp::handler::server::tool::schema_for_output::<WindowGeometryToolOutput>(),
description = "Resize a window in desktop pixels, unmaximizing it first if needed. Useful to fit a window fully on-screen. Through the computer-use-linux GNOME Shell extension, width and height set the frame; through a generic X11/EWMH window manager (wmctrl), they set the client area, and decorations add to it. Other window backends refuse it. Pass screenshot_after (true, \"screen\", or \"window\") or include_tree=true to return post-action observation in the same result.",
annotations(
read_only_hint = false,
destructive_hint = false,
idempotent_hint = true,
open_world_hint = true
)
)]
async fn resize_window(
&self,
Parameters(params): Parameters<ResizeWindowParams>,
) -> Result<CallToolResult, ErrorData> {
let capture = params.target.capture.clone();
let received = Some(serde_json::json!(params.clone()));
let target = params.target.into_target();
self.indicator.action("resize_window").await;
let Json(output) = self
.window_geometry_op(received, &target, |window| async move {
registry::resize_window(&window, params.width, params.height).await
})
.await;
self.finish_window_geometry_output(output, &capture, &target)
.await
}
}
#[tool_handler(
router = self.mcp_tool_router(),
name = "computer-use-linux",
// NOTE: keep in lockstep with Cargo.toml + package.json on every release.
// The rmcp tool_handler macro only accepts a string literal here, so this
// can't be env!("CARGO_PKG_VERSION"); the MCP safety check (CI) fails the
// build if it drifts from the Cargo version.
version = "0.7.52",
instructions = "Begin every turn that uses Computer Use by calling get_app_state. If diagnostics report disabled GNOME accessibility, call setup_accessibility before asking the user to retry. Use list_windows/focused_window before targeted keyboard input. If readiness reports can_query_windows or can_focus_windows false on GNOME, call setup_window_targeting to install the GNOME Shell extension, then ask the user to log out and back in if the setup report says a shell reload is required. This Linux backend captures size-bounded screenshots (pixelflux when configured, GNOME Shell, the XDG portal, native X11, gnome-screenshot; grim in Hyprland background mode), reads AT-SPI trees with action/value metadata, invokes native AT-SPI actions, sets AT-SPI values or editable text, lists and focuses compositor windows through registered Linux window backends when the session permits it, and attaches best-effort terminal tty/process metadata to terminal windows. Click, scroll and drag try pixelflux when configured, xdotool on native X11, the uinput pointer, the RemoteDesktop portal (an open portal session, or when ydotool is absent, on COSMIC, or when forced), then ydotool outside Wayland. On Wayland a point never goes through ydotool: without uinput or the portal a coordinate action is refused; a coordinate scroll sends the move and the wheel through the uinput pointer, and an untargeted scroll sends the wheel through ydotool, or the portal without its daemon. The uinput pointer is sized to the latest full screenshot; take a new one after the desktop size changes. type_text sends literal text through pixelflux when configured (an unreachable pixelflux falls through), KDE clipboard paste on Plasma Wayland (raw ydotool when Klipper cannot be read), wtype on compatible Wayland compositors, portal keysyms on other Wayland sessions, or xdotool on X11. Portal text startup or conversion failures return an error without replaying through ydotool. GNOME preflights the complete text against every configured keyboard layout group because the background reader cannot establish the active group; rejected text goes into the target's focused editable field through AT-SPI EditableText and must read back, and otherwise can use set_value on an editable field. Raw ydotool text is limited to printable ASCII, tab and newline, uses US physical key positions, and requires checking the resulting field contents under the active layout. Screenshot results include width/height for the returned image plus coordinate_width/coordinate_height and scale for desktop coordinate conversion; request more detail with max_width, max_height, max_bytes, format=jpeg, quality, or a smaller target/crop instead of relying on unbounded screenshots. Mutating tools (click, type_text, press_key, scroll, drag, perform_action, set_value, activate_window, move_window, resize_window) accept optional screenshot_after (true, \"screen\", or \"window\") with format/quality/scale/max_width/max_height/max_bytes, optional screenshot_file_path (an absolute path to a new file, stdio only, refused over HTTP) to save the screenshot there instead of returning image content, and optional include_tree=true to return post-action observation in the same tool call. A failed action returns no observation, except set_value with verified=false. Tools with readOnlyHint=false may mutate local desktop or application state; hosts should require approval for actions that can submit, delete, send, purchase, or overwrite data. For element-targeted actions, prefer element_index from the latest get_app_state result; click, perform_action, and set_value can also use semantic role/name/text/states selectors when the target is unique. Pass that result's snapshot_id with the index or selector: each get_app_state or include_tree tree replaces the cached one, and a call naming a replaced snapshot is refused before any input. type_text and press_key accept optional window_id, pid, app_id, wm_class, title, tty, terminal_pid, terminal_command, or terminal_cwd selectors and refuse targeted input if focus cannot be verified. After targeted keyboard input, results append focused-element feedback from AT-SPI (role, name, editable); type_text also warns when a concrete non-terminal focus is not editable. Exact window focus alone does not suppress a missing-focus warning; only a complete scoped tree containing window/menu chrome without content makes it informational. Terminals accept keyboard input without an EditableText interface. On Hyprland, wtype passes classified nonsecret text as one literal argument after -- to avoid stdin keymap-refresh corruption; that text is visible in process arguments, so confirmed password fields are rejected (use set_value), and terminals and ambiguous fields, with or without AT-SPI, use private stdin for at most 100 Unicode characters per call (split larger input or expose accessibility). Input classification has its own 1500 ms budget, independent of the 250 ms cosmetic indicator timeout. Other compatible compositors retain stdin. The wtype path rejects NUL and more than 131071 UTF-8 bytes before focus or indicators. X11 xdotool can silently substitute symbols under non-US XKB layouts (reported with br/ABNT2) or changing keymap state; inspect the text before pressing Enter or submitting it. A successful type_text result through a keyboard backend confirms dispatch only and leaves field contents unverified; only the GNOME AT-SPI fallback reads the field back. Re-observe the target before retrying. Screenshot, keyboard and move/resize results warn when the target window is partly or fully off-screen; click, scroll and drag results warn when a coordinate is off the screenshot or between monitors (not through pixelflux, and not for KWin element clicks, which KWin maps onto an output). move_window and resize_window, which bring a window back on-screen, work only through the GNOME Shell extension or X11/EWMH (wmctrl). scroll accepts the same window targeting and relative coordinates as click; on both, a window target is raised and focused first, and nothing is sent when focus cannot be verified. get_app_state returns a compact readiness block and compact tree nodes (null, empty and default fields left out, actions listed by name) by default; pass verbose=true for the full diagnostics dump and full nodes. Pass interactive_only=true to get only nodes with an action, editable text or a value, plus headings, status/live regions, windows and dialogs; containers are collapsed and element_index values match the full tree. A title or window_id target narrows an app that exposes several windows under one AT-SPI application root (Chromium) to that window's subtree, keeping the app's popups and dialogs. Scope get_app_state with app_name_or_bundle_identifier or a window target (window_id, pid, app_id, wm_class, title); without one it returns the whole desktop AT-SPI tree, reports tree_scoped=false, and warns in message, which can flood context. accessibility_tree_truncated=true means the node, depth, read, or time budget stopped traversal with unread elements left; recover by scoping to a narrower app or window target and raising max_nodes or max_depth (hard caps 10000 and 64), not by lowering max_nodes. A compacted tree too large to return inline comes back as its leading nodes plus accessibility_tree_spill.path, a private JSON file with the whole tree and the same element_index values; search that file instead of requesting the tree again, or page through it with get_tree_page when files are not readable. Electron apps with only window/menu chrome may need --force-renderer-accessibility to expose renderer content."
)]
impl ServerHandler for ComputerUseLinux {
async fn call_tool(
&self,
request: rmcp::model::CallToolRequestParams,
context: rmcp::service::RequestContext<rmcp::RoleServer>,
) -> Result<rmcp::model::CallToolResponse, rmcp::ErrorData> {
// Each HTTP client names itself per request; a shared field would show the last to initialize.
let agent = crate::indicator::request_agent_name(&context.meta, context.client_info());
let router = self.mcp_tool_router();
// A bad screenshot_file_path is refused before any capture or input,
// so a mutating tool never acts and then drops its observation.
let accepts_file = router.get(&request.name).is_some_and(|tool| {
tool.input_schema
.get("properties")
.and_then(serde_json::Value::as_object)
.is_some_and(|properties| properties.contains_key("screenshot_file_path"))
});
if let Some(refusal) = screenshot_file_refusal(
&request.name,
request.arguments.as_ref(),
accepts_file,
self.http_transport,
) {
return Ok(CallToolResult::error(vec![ContentBlock::text(refusal)]).into());
}
let call = rmcp::handler::server::tool::ToolCallContext::new(self, request, context);
crate::indicator::with_request_agent(agent, router.call(call)).await
}
}
pub(crate) fn shell_execution_enabled() -> bool {
shell_execution_enabled_value(env::var(SHELL_ENABLE_ENV).ok().as_deref())
}
#[derive(Debug, Serialize, JsonSchema)]
struct CompletionOutput {
ok: bool,
cue: String,
note: String,
}
async fn send_completion_notification(program: &Path, limit: Duration) -> CompletionOutput {
let mut command = TokioCommand::new(program);
command.args([
"--app-name=computer-use-linux",
"--expire-time=3000",
"Desktop interaction finished",
"The agent has finished its desktop interaction.",
]);
let sent =
crate::command_runner::output_with_timeout(command, "send completion notification", limit)
.await;
match sent {
Ok(output) if output.status.success() => CompletionOutput {
ok: true,
cue: "notification".into(),
note: "Notification submitted; desktop settings control whether it is displayed."
.into(),
},
_ => CompletionOutput {
ok: true,
cue: "skipped".into(),
note:
"Notification unavailable, failed, or timed out. The interaction is still complete."
.into(),
},
}
}
fn shell_execution_enabled_value(value: Option<&str>) -> bool {
value == Some("1")
}
fn valid_environment_name(name: &str) -> bool {
let mut chars = name.chars();
chars
.next()
.is_some_and(|first| first == '_' || first.is_ascii_alphabetic())
&& chars.all(|character| character == '_' || character.is_ascii_alphanumeric())
}
fn inherited_shell_environment_allowed(name: &str) -> bool {
matches!(
name,
"PATH"
| "HOME"
| "USER"
| "LOGNAME"
| "LANG"
| "TERM"
| "XDG_RUNTIME_DIR"
| "DISPLAY"
| "WAYLAND_DISPLAY"
| "DBUS_SESSION_BUS_ADDRESS"
) || name.starts_with("LC_")
}
fn inherited_shell_environment(
variables: impl IntoIterator<Item = (OsString, OsString)>,
) -> Vec<(String, String)> {
variables
.into_iter()
.filter_map(|(name, value)| {
let (Ok(name), Ok(value)) = (name.into_string(), value.into_string()) else {
return None;
};
inherited_shell_environment_allowed(&name).then_some((name, value))
})
.collect()
}
fn shell_command_sha256(command: &str) -> String {
Sha256::digest(command.as_bytes())
.iter()
.map(|byte| format!("{byte:02x}"))
.collect()
}
fn bounded_shell_stream(bytes: &[u8]) -> (String, bool) {
let truncated = bytes.len() > SHELL_RESPONSE_STREAM_BYTES;
let visible = if truncated {
&bytes[..SHELL_RESPONSE_STREAM_BYTES]
} else {
bytes
};
(String::from_utf8_lossy(visible).into_owned(), truncated)
}
fn shell_error_output(
command_sha256: String,
cwd: String,
timeout_seconds: u64,
error: impl Into<String>,
) -> RunShellOutput {
RunShellOutput {
ok: false,
command_sha256,
cwd,
timeout_seconds,
exit_code: None,
stdout: String::new(),
stderr: String::new(),
stdout_truncated: false,
stderr_truncated: false,
error: Some(error.into()),
}
}
async fn execute_shell(params: RunShellParams) -> RunShellOutput {
let command_sha256 = shell_command_sha256(¶ms.command);
let timeout_seconds = params.timeout_seconds.unwrap_or(SHELL_DEFAULT_TIMEOUT_SECS);
let requested_cwd = params.cwd.as_deref().unwrap_or(".").to_string();
if !shell_execution_enabled() {
return shell_error_output(
command_sha256,
requested_cwd,
timeout_seconds,
format!(
"shell execution is disabled; restart the MCP server with {SHELL_ENABLE_ENV}=1 to opt in"
),
);
}
if params.command.trim().is_empty() {
return shell_error_output(
command_sha256,
requested_cwd,
timeout_seconds,
"command must not be empty",
);
}
if params.command.len() > SHELL_MAX_COMMAND_BYTES {
return shell_error_output(
command_sha256,
requested_cwd,
timeout_seconds,
format!("command exceeds the {SHELL_MAX_COMMAND_BYTES}-byte limit"),
);
}
if requested_cwd.len() > SHELL_MAX_CWD_BYTES {
return shell_error_output(
command_sha256,
"<rejected: cwd too long>".to_string(),
timeout_seconds,
format!("cwd exceeds the {SHELL_MAX_CWD_BYTES}-byte limit"),
);
}
if !(1..=SHELL_MAX_TIMEOUT_SECS).contains(&timeout_seconds) {
return shell_error_output(
command_sha256,
requested_cwd,
timeout_seconds,
format!("timeout_seconds must be between 1 and {SHELL_MAX_TIMEOUT_SECS}"),
);
}
if params.env.len() > SHELL_MAX_ENV_ENTRIES {
return shell_error_output(
command_sha256,
requested_cwd,
timeout_seconds,
format!("env contains more than {SHELL_MAX_ENV_ENTRIES} entries"),
);
}
let env_bytes = params
.env
.iter()
.map(|(name, value)| name.len().saturating_add(value.len()))
.sum::<usize>();
if env_bytes > SHELL_MAX_ENV_BYTES {
return shell_error_output(
command_sha256,
requested_cwd,
timeout_seconds,
format!("env exceeds the {SHELL_MAX_ENV_BYTES}-byte limit"),
);
}
if let Some(invalid) = params.env.keys().find(|name| !valid_environment_name(name)) {
return shell_error_output(
command_sha256,
requested_cwd,
timeout_seconds,
format!("invalid environment variable name: {invalid}"),
);
}
let cwd = match std::fs::canonicalize(&requested_cwd) {
Ok(path) if path.is_dir() => path,
Ok(_) => {
return shell_error_output(
command_sha256,
requested_cwd,
timeout_seconds,
"cwd is not a directory",
)
}
Err(error) => {
return shell_error_output(
command_sha256,
requested_cwd,
timeout_seconds,
format!("failed to resolve cwd: {error}"),
)
}
};
let cwd_display = cwd.display().to_string();
let mut child = TokioCommand::new("/bin/sh");
// Do not use a login shell: profile scripts could reintroduce credentials
// after env_clear() and contaminate or prevent the requested command.
child.args(["-c", ¶ms.command]);
child.current_dir(&cwd);
child.env_clear();
let mut inherited_path = false;
for (name, value) in inherited_shell_environment(env::vars_os()) {
inherited_path |= name == "PATH";
child.env(name, value);
}
if !inherited_path {
child.env("PATH", "/usr/local/bin:/usr/bin:/bin");
}
child.envs(¶ms.env);
eprintln!(
"[computer-use-linux] run_shell start sha256={command_sha256} cwd={cwd_display:?} timeout_seconds={timeout_seconds}"
);
match crate::command_runner::output_with_timeout(
child,
"run approved shell command",
Duration::from_secs(timeout_seconds),
)
.await
{
Ok(output) => {
let exit_code = output.status.code();
let (stdout, stdout_truncated) = bounded_shell_stream(&output.stdout);
let (stderr, stderr_truncated) = bounded_shell_stream(&output.stderr);
eprintln!(
"[computer-use-linux] run_shell finish sha256={command_sha256} exit_code={exit_code:?} stdout_bytes={} stderr_bytes={} stdout_truncated={stdout_truncated} stderr_truncated={stderr_truncated}",
output.stdout.len(),
output.stderr.len()
);
RunShellOutput {
ok: output.status.success(),
command_sha256,
cwd: cwd_display,
timeout_seconds,
exit_code,
stdout,
stderr,
stdout_truncated,
stderr_truncated,
error: None,
}
}
Err(error) => {
let error = format!("{error:#}");
eprintln!(
"[computer-use-linux] run_shell error sha256={command_sha256} error={error:?}"
);
shell_error_output(command_sha256, cwd_display, timeout_seconds, error)
}
}
}
pub async fn serve_mcp(listen: Option<crate::http_listen::ListenConfig>) -> Result<()> {
let mut server = ComputerUseLinux::default();
let tree_spiller = Arc::clone(&server.tree_spiller);
tree_spiller.sweep();
if let Some(config) = listen {
server.require_snapshot_ids = true;
server.http_transport = true;
let signal = crate::http_listen::serve_http(server, config).await?;
tree_spiller.remove_current();
std::process::exit(128 + signal);
}
let indicator = Arc::clone(&server.indicator);
let running = server.serve(rmcp::transport::stdio()).await?;
// Legacy initialization supplies the client name through the peer.
// Discovery-only hosts use the indicator's configured name or Agent fallback.
indicator.attach(running.peer().clone());
// Hosts stop servers by closing stdin or with SIGTERM/SIGINT; a spilled
// tree is removed either way.
let mut terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?;
let mut interrupt = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::interrupt())?;
let signal = tokio::select! {
result = running.waiting() => {
tree_spiller.remove_current();
result?;
return Ok(());
}
_ = terminate.recv() => libc::SIGTERM,
_ = interrupt.recv() => libc::SIGINT,
};
tree_spiller.remove_current();
// Exit now: returning would wait forever for the stdin reader thread.
std::process::exit(128 + signal);
}
#[derive(Debug, Clone, Serialize, JsonSchema)]
struct ListAppsOutput {
apps: Vec<AppCandidate>,
accessible_apps: Vec<AccessibleAppSummary>,
accessibility_error: Option<String>,
note: String,
}
#[derive(Debug, Clone, Serialize, JsonSchema)]
struct ListWindowsOutput {
backend: String,
windows: Vec<WindowInfo>,
error: Option<String>,
permissions_hint: Option<String>,
note: String,
}
#[derive(Debug, Clone, Serialize, JsonSchema)]
struct FocusedWindowOutput {
backend: String,
focused_window: Option<WindowInfo>,
error: Option<String>,
permissions_hint: Option<String>,
message: String,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize, JsonSchema)]
#[serde(rename_all = "lowercase")]
enum ScreenshotAfterScope {
Window,
Screen,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
#[serde(untagged)]
enum ScreenshotAfter {
Enabled(bool),
Scope(ScreenshotAfterScope),
}
impl ScreenshotAfter {
fn resolve(self) -> Option<ScreenshotAfterScope> {
match self {
Self::Enabled(false) => None,
Self::Enabled(true) => Some(ScreenshotAfterScope::Screen),
Self::Scope(scope) => Some(scope),
}
}
}
fn optional_screenshot_after_schema(_gen: &mut schemars::SchemaGenerator) -> schemars::Schema {
schemars::json_schema!({
"description": "Capture and return a post-action screenshot in the same result: true or \"screen\" captures the full desktop; \"window\" crops to the action's target window (or the focused window when untargeted); false or null disables capture.",
"anyOf": [
{ "type": "boolean" },
{ "type": "string", "enum": ["window", "screen"] },
{ "type": "null" }
]
})
}
#[derive(Debug, Clone, Default, Deserialize, Serialize, JsonSchema)]
struct PostActionCaptureParams {
/// Capture and return a post-action screenshot in the same result: `true` or
/// `"screen"` captures the full desktop; `"window"` crops to the action's
/// target window (or the focused window when untargeted); `false` disables capture.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(schema_with = "optional_screenshot_after_schema")]
screenshot_after: Option<ScreenshotAfter>,
/// Maximum returned post-action screenshot width in pixels (default 1920, hard-capped).
#[serde(default, skip_serializing_if = "Option::is_none")]
max_width: Option<u32>,
/// Maximum returned post-action screenshot height in pixels (default 1920, hard-capped).
#[serde(default, skip_serializing_if = "Option::is_none")]
max_height: Option<u32>,
/// Maximum returned post-action screenshot image bytes before base64 (default 2 MiB, hard-capped).
#[serde(default, skip_serializing_if = "Option::is_none")]
max_bytes: Option<usize>,
/// Additional downscale factor from 0.0 to 1.0, applied before max dimensions.
#[serde(default, skip_serializing_if = "Option::is_none")]
scale: Option<f32>,
/// Post-action screenshot image format (default png). Use jpeg with quality to trade exact pixels for smaller payloads.
#[serde(default, skip_serializing_if = "Option::is_none")]
format: Option<ScreenshotOutputFormat>,
/// JPEG quality from 1 to 95 (default 80) for post-action screenshots. Ignored for png.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(range(min = 1, max = 95))]
quality: Option<u8>,
/// Save the post-action screenshot to this absolute path instead of
/// returning it as image content; needs screenshot_after. The parent
/// directory must exist, and the extension must match format (.png, or
/// .jpg/.jpeg for jpeg). The file must not exist: file output creates a
/// new file and never replaces one. Refused over HTTP (mcp --listen). A
/// failed write is reported in screenshot_error, and the action's ok stays
/// true.
#[serde(default, skip_serializing_if = "Option::is_none")]
screenshot_file_path: Option<String>,
/// Include a fresh compacted AT-SPI accessibility tree snapshot after the action (default false). Also refreshes cached element_index values.
/// It reads at most 1000 nodes and depth 32; when accessibility_tree_truncated
/// is true, call get_app_state with a higher max_nodes.
#[serde(default, skip_serializing_if = "Option::is_none")]
include_tree: Option<bool>,
}
/// The fields of a tool call that decide whether its screenshot_file_path
/// can be used.
#[derive(Deserialize)]
struct ScreenshotFileRequest {
#[serde(default)]
screenshot_file_path: Option<String>,
#[serde(default)]
format: Option<ScreenshotOutputFormat>,
#[serde(default)]
include_screenshot: Option<bool>,
#[serde(default)]
screenshot_after: Option<ScreenshotAfter>,
}
/// The refusal for a call whose screenshot_file_path cannot be used: over
/// HTTP, without a capture to save, or with a path that fails
/// `validate_screenshot_file_path`. `accepts_file` says whether the tool takes
/// the parameter at all. Malformed arguments are left to the tool's own
/// parameter errors.
fn screenshot_file_refusal(
tool: &str,
arguments: Option<&rmcp::model::JsonObject>,
accepts_file: bool,
remote: bool,
) -> Option<String> {
let arguments = arguments?;
if !accepts_file
|| arguments
.get("screenshot_file_path")
.is_none_or(serde_json::Value::is_null)
{
return None;
}
let request: ScreenshotFileRequest =
serde_json::from_value(serde_json::Value::Object(arguments.clone())).ok()?;
let path = request.screenshot_file_path?;
if remote {
return Some(format!(
"{tool} was not run: {}",
crate::screenshot_file::REMOTE_REFUSAL
));
}
let missing_capture = match tool {
"screenshot" => None,
"get_app_state" => (!request.include_screenshot.unwrap_or(true))
.then_some("screenshot_file_path needs include_screenshot=true"),
_ => request
.screenshot_after
.and_then(ScreenshotAfter::resolve)
.is_none()
.then_some("screenshot_file_path needs screenshot_after"),
};
if let Some(reason) = missing_capture {
return Some(format!("{tool} was not run: {reason}"));
}
crate::screenshot_file::validate_screenshot_file_path(
&path,
request.format.unwrap_or(ScreenshotOutputFormat::Png),
remote,
)
.err()
.map(|error| format!("{tool} was not run: {error:#}"))
}
impl PostActionCaptureParams {
fn screenshot_scope(&self) -> Option<ScreenshotAfterScope> {
self.screenshot_after.and_then(ScreenshotAfter::resolve)
}
fn screenshot_options(&self) -> ScreenshotPayloadOptions {
ScreenshotPayloadOptions {
max_width: self.max_width,
max_height: self.max_height,
max_bytes: self.max_bytes,
scale: self.scale,
format: self.format,
quality: self.quality,
}
}
}
#[derive(Debug, Clone, Default, Serialize, JsonSchema)]
struct PostActionObservation {
/// Post-action screenshot metadata when `screenshot_after` was requested; the image itself is the first `image` content block, or the file at `file_path` when `screenshot_file_path` was given.
#[serde(skip_serializing_if = "Option::is_none")]
screenshot: Option<ScreenshotSummary>,
/// Post-action screenshot capture error when a requested `screenshot_after` capture failed.
#[serde(skip_serializing_if = "Option::is_none")]
screenshot_error: Option<String>,
/// Names the post-action tree, which replaced the action's snapshot. Pass
/// it with element_index actions on that tree.
#[serde(skip_serializing_if = "Option::is_none")]
snapshot_id: Option<String>,
/// Post-action compacted AT-SPI accessibility tree when `include_tree=true` was requested.
#[serde(skip_serializing_if = "Option::is_none")]
accessibility_tree: Option<TreeNodes>,
/// Raw AT-SPI node count before compaction when `include_tree=true` was requested.
#[serde(skip_serializing_if = "Option::is_none")]
accessibility_tree_raw_count: Option<usize>,
/// Set when the post-action tree was too large to return inline: the whole
/// tree is in this private file, with the same element_index values.
#[serde(skip_serializing_if = "Option::is_none")]
accessibility_tree_spill: Option<crate::tree_spill::TreeSpill>,
/// True when the post-action accessibility tree was scoped to the target app/window.
#[serde(skip_serializing_if = "Option::is_none")]
tree_scoped: Option<bool>,
/// True when the post-action accessibility tree hit traversal limits.
#[serde(skip_serializing_if = "Option::is_none")]
accessibility_tree_truncated: Option<bool>,
/// Post-action AT-SPI error when a requested `include_tree=true` capture failed.
#[serde(skip_serializing_if = "Option::is_none")]
accessibility_error: Option<String>,
}
#[derive(Debug, Clone, Default, Deserialize, Serialize, JsonSchema)]
struct ActivateWindowParams {
/// Window id from a recent list_windows. The first target field set
/// decides, in this order: window_id, terminal fields, pid, app_id,
/// wm_class, title. With a terminal field, pid, app_id, wm_class and title
/// only narrow the terminal match, which must name one window.
#[serde(
default,
deserialize_with = "crate::windowing::types::deserialize_window_id"
)]
#[schemars(range(max = crate::windowing::types::MAX_WINDOW_ID))]
window_id: Option<u64>,
/// Process id. It must match exactly one window.
#[serde(default)]
pid: Option<u32>,
/// Terminal tty: /dev/pts/N, pts/N or N.
#[serde(default)]
tty: Option<String>,
/// Pid of the shell or foreground process on a terminal's tty, not the
/// terminal emulator (use pid for that).
#[serde(default)]
terminal_pid: Option<u32>,
/// Substring of the command name or command line of a terminal's shell
/// or foreground process, ignoring ASCII case.
#[serde(default)]
terminal_command: Option<String>,
/// Working directory of a terminal's shell or foreground process: an
/// absolute path, or a trailing path such as project/src.
#[serde(default)]
terminal_cwd: Option<String>,
/// Application id, ignoring ASCII case: the full desktop id, or its last
/// dot-separated segment outside a terminal match. A segment that matches several apps is
/// rejected.
#[serde(default)]
app_id: Option<String>,
/// Window manager class, an exact match ignoring ASCII case. When several
/// windows match outside a terminal match, the first wins.
#[serde(default)]
wm_class: Option<String>,
/// Window title substring, ignoring ASCII case. When several windows
/// match outside a terminal match, the first wins.
#[serde(default)]
title: Option<String>,
#[serde(flatten)]
capture: PostActionCaptureParams,
}
impl ActivateWindowParams {
fn into_target(self) -> WindowTarget {
WindowTarget {
window_id: self.window_id,
pid: self.pid,
tty: self.tty,
terminal_pid: self.terminal_pid,
terminal_command: self.terminal_command,
terminal_cwd: self.terminal_cwd,
app_id: self.app_id,
wm_class: self.wm_class,
title: self.title,
}
}
}
#[derive(Debug, Clone, Serialize, JsonSchema)]
struct ActivateWindowOutput {
ok: bool,
implemented: bool,
backend: String,
focus: Option<WindowFocusResult>,
error: Option<String>,
permissions_hint: Option<String>,
// Echo of the request for debugging. `serde_json::Value` has no fixed JSON
// schema, which strict MCP clients (Claude Code) reject in `outputSchema`,
// and one invalid tool fails the whole tool list. Keep it in the runtime
// response (serde) but omit it from the generated schema.
#[schemars(skip)]
received: Option<serde_json::Value>,
}
#[derive(Debug, Clone, Serialize, JsonSchema)]
struct ActivateWindowToolOutput {
#[serde(flatten)]
output: ActivateWindowOutput,
/// Post-action observation notes (off-screen warnings, tree warnings, or capture errors) when present.
#[serde(skip_serializing_if = "Option::is_none")]
message: Option<String>,
#[serde(flatten)]
observation: PostActionObservation,
}
#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)]
struct MoveWindowParams {
#[serde(flatten)]
target: ActivateWindowParams,
/// New frame-left in desktop coordinates.
x: i32,
/// New frame-top in desktop coordinates.
y: i32,
}
#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)]
struct ResizeWindowParams {
#[serde(flatten)]
target: ActivateWindowParams,
/// New width in desktop pixels: the frame through the GNOME Shell
/// extension, the client area on X11/EWMH.
width: i32,
/// New height in desktop pixels: the frame through the GNOME Shell
/// extension, the client area on X11/EWMH.
height: i32,
}
#[derive(Debug, Clone, Serialize, JsonSchema)]
struct WindowGeometryOutput {
ok: bool,
implemented: bool,
backend: String,
/// Post-operation window info (compositor-final geometry).
window: Option<WindowInfo>,
message: String,
permissions_hint: Option<String>,
#[schemars(skip)]
received: Option<serde_json::Value>,
}
#[derive(Debug, Clone, Serialize, JsonSchema)]
struct WindowGeometryToolOutput {
#[serde(flatten)]
output: WindowGeometryOutput,
#[serde(flatten)]
observation: PostActionObservation,
}
#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)]
struct RunShellParams {
/// Shell program text passed to `/bin/sh -c` without loading login profiles.
command: String,
/// Existing directory in which to start the shell. Symlinks are resolved.
#[serde(default)]
cwd: Option<String>,
/// Additional environment entries. The ambient process environment is not inherited wholesale.
#[serde(default)]
env: BTreeMap<String, String>,
/// Wall-clock timeout in seconds (default 30, maximum 120).
#[serde(default)]
timeout_seconds: Option<u64>,
}
#[derive(Debug, Clone, Serialize, JsonSchema)]
struct RunShellOutput {
ok: bool,
command_sha256: String,
cwd: String,
timeout_seconds: u64,
exit_code: Option<i32>,
stdout: String,
stderr: String,
stdout_truncated: bool,
stderr_truncated: bool,
error: Option<String>,
}
#[derive(Debug, Clone, Serialize, JsonSchema)]
struct AppCandidate {
name: String,
pid: u32,
command: String,
}
#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)]
struct GetAppStateParams {
/// App name or AT-SPI id that limits the accessibility tree. Omit only when
/// you need the whole desktop tree; unscoped results can flood context.
#[serde(default)]
app_name_or_bundle_identifier: Option<String>,
/// Compositor window id. Also scopes the accessibility tree to that window's
/// application when possible.
#[serde(
default,
deserialize_with = "crate::windowing::types::deserialize_window_id"
)]
#[schemars(range(max = crate::windowing::types::MAX_WINDOW_ID))]
window_id: Option<u64>,
/// Process id. Also scopes the accessibility tree to that process when it
/// exposes AT-SPI.
#[serde(default)]
pid: Option<u32>,
/// Terminal tty device (for example /dev/pts/3). Resolves a window target
/// and scopes the tree when possible.
#[serde(default)]
tty: Option<String>,
/// Pid of the shell or foreground process on a terminal's tty, not the
/// terminal emulator (use pid for that). Resolves a window target and
/// scopes the tree when possible.
#[serde(default)]
terminal_pid: Option<u32>,
/// Terminal command substring, ignoring ASCII case. Resolves a window
/// target and scopes the tree when possible.
#[serde(default)]
terminal_command: Option<String>,
/// Terminal working directory. Resolves a window target and scopes the tree
/// when possible.
#[serde(default)]
terminal_cwd: Option<String>,
/// Application id. Also scopes the accessibility tree when it matches an
/// AT-SPI root.
#[serde(default)]
app_id: Option<String>,
/// Window manager class. Also scopes the accessibility tree when it matches
/// an AT-SPI root.
#[serde(default)]
wm_class: Option<String>,
/// Window title substring. Also scopes the accessibility tree when it
/// matches an AT-SPI root.
#[serde(default)]
title: Option<String>,
/// Maximum raw AT-SPI nodes to inspect before compaction (default 1000, hard max 10000).
#[serde(default)]
max_nodes: Option<usize>,
/// Maximum AT-SPI traversal depth (default 32, hard max 64).
#[serde(default)]
max_depth: Option<u32>,
/// Include a size-bounded screenshot (default true). Set false when the
/// accessibility tree is enough.
#[serde(default)]
include_screenshot: Option<bool>,
/// Maximum returned screenshot width in pixels (default 1920, hard-capped).
#[serde(default)]
max_width: Option<u32>,
/// Maximum returned screenshot height in pixels (default 1920, hard-capped).
#[serde(default)]
max_height: Option<u32>,
/// Maximum returned screenshot image bytes before base64 (default 2 MiB, hard-capped).
#[serde(default)]
max_bytes: Option<usize>,
/// Additional downscale factor from 0.0 to 1.0, applied before max dimensions.
#[serde(default)]
scale: Option<f32>,
/// Output image format (default png). Use jpeg with quality to trade exact pixels for smaller payloads.
#[serde(default)]
format: Option<ScreenshotOutputFormat>,
/// JPEG quality from 1 to 95 (default 80). Ignored for png.
#[serde(default)]
#[schemars(range(min = 1, max = 95))]
quality: Option<u8>,
/// Save the screenshot to this absolute path instead of returning it as
/// image content; screenshot.file_path then names it. The parent directory
/// must exist, and the extension must match format (.png, or .jpg/.jpeg for
/// jpeg). The file must not exist: file output creates a new file and
/// never replaces one. Refused over HTTP (mcp --listen). A failed write is
/// reported in screenshot_error.
#[serde(default)]
screenshot_file_path: Option<String>,
/// Include the full diagnostics report (large) and return tree nodes in the
/// full encoding, with null and empty fields and action descriptions and
/// keybindings. Default false: only the compact readiness block, and
/// compact nodes.
#[serde(default)]
verbose: Option<bool>,
/// Return only nodes that can be acted on (an action, editable text, or a
/// value), plus headings, status/live regions, windows and dialogs.
/// Containers are collapsed: each node's parent_index points at its
/// nearest kept ancestor. element_index values are the same as in the full
/// tree, so they work with click, perform_action and set_value.
#[serde(default)]
interactive_only: Option<bool>,
}
impl GetAppStateParams {
fn window_target(&self) -> WindowTarget {
WindowTarget {
window_id: self.window_id,
pid: self.pid,
tty: self.tty.clone(),
terminal_pid: self.terminal_pid,
terminal_command: self.terminal_command.clone(),
terminal_cwd: self.terminal_cwd.clone(),
app_id: self.app_id.clone(),
wm_class: self.wm_class.clone(),
title: self.title.clone(),
}
}
fn screenshot_options(&self) -> ScreenshotPayloadOptions {
ScreenshotPayloadOptions {
max_width: self.max_width,
max_height: self.max_height,
max_bytes: self.max_bytes,
scale: self.scale,
format: self.format,
quality: self.quality,
}
}
}
#[derive(Debug, Clone, Default, Deserialize, Serialize, JsonSchema)]
struct ScreenshotParams {
/// Window id from a recent list_windows. The first target field set
/// decides, in this order: window_id, pid, app_id, wm_class, title.
#[serde(
default,
deserialize_with = "crate::windowing::types::deserialize_window_id"
)]
#[schemars(range(max = crate::windowing::types::MAX_WINDOW_ID))]
window_id: Option<u64>,
/// Process id. It must match exactly one window.
#[serde(default)]
pid: Option<u32>,
/// Application id, ignoring ASCII case: the full desktop id, or its last
/// dot-separated segment. A segment that matches several apps is
/// rejected.
#[serde(default)]
app_id: Option<String>,
/// Window manager class, an exact match ignoring ASCII case. When several
/// windows match, the first wins.
#[serde(default)]
wm_class: Option<String>,
/// Window title substring, ignoring ASCII case. When several windows
/// match, the first wins.
#[serde(default)]
title: Option<String>,
/// Raise the targeted window before capture (default true). Ignored without
/// a window target.
#[serde(default)]
raise_window: Option<bool>,
/// Capture the whole desktop even when a window is targeted (default false).
#[serde(default)]
full_screen: Option<bool>,
/// Maximum returned screenshot width in pixels (default 1920, hard-capped).
#[serde(default)]
max_width: Option<u32>,
/// Maximum returned screenshot height in pixels (default 1920, hard-capped).
#[serde(default)]
max_height: Option<u32>,
/// Maximum returned screenshot image bytes before base64 (default 2 MiB, hard-capped).
#[serde(default)]
max_bytes: Option<usize>,
/// Additional downscale factor from 0.0 to 1.0, applied before max dimensions.
#[serde(default)]
scale: Option<f32>,
/// Output image format (default png). Use jpeg with quality to trade exact pixels for smaller payloads.
#[serde(default)]
format: Option<ScreenshotOutputFormat>,
/// JPEG quality from 1 to 95 (default 80). Ignored for png.
#[serde(default)]
#[schemars(range(min = 1, max = 95))]
quality: Option<u8>,
/// Save the image to this absolute path instead of returning it as image
/// content; the caption's file_path then names it. The parent directory
/// must exist, and the extension must match format (.png, or .jpg/.jpeg for
/// jpeg). The file must not exist: file output creates a new file and
/// never replaces one. Refused over HTTP (mcp --listen).
#[serde(default)]
screenshot_file_path: Option<String>,
}
impl ScreenshotParams {
fn window_target(&self) -> Option<WindowTarget> {
if self.window_id.is_none()
&& self.pid.is_none()
&& self.app_id.is_none()
&& self.wm_class.is_none()
&& self.title.is_none()
{
return None;
}
Some(WindowTarget {
window_id: self.window_id,
pid: self.pid,
tty: None,
terminal_pid: None,
terminal_command: None,
terminal_cwd: None,
app_id: self.app_id.clone(),
wm_class: self.wm_class.clone(),
title: self.title.clone(),
})
}
fn screenshot_options(&self) -> ScreenshotPayloadOptions {
ScreenshotPayloadOptions {
max_width: self.max_width,
max_height: self.max_height,
max_bytes: self.max_bytes,
scale: self.scale,
format: self.format,
quality: self.quality,
}
}
}
/// Screenshot metadata for `get_app_state` and post-action captures. The image
/// bytes travel as a separate `image` content block (issue #145), not inside
/// this JSON, or are saved to `file_path` (issue #446).
#[derive(Debug, Clone, Serialize, JsonSchema)]
struct ScreenshotSummary {
mime_type: String,
source: String,
/// Width of the returned image payload.
width: u32,
/// Height of the returned image payload.
height: u32,
/// Coordinate-space width before payload downscaling.
coordinate_width: u32,
/// Coordinate-space height before payload downscaling.
coordinate_height: u32,
/// Returned pixels per coordinate-space pixel.
scale: f32,
resized: bool,
bytes: usize,
original_bytes: usize,
max_bytes: usize,
format: ScreenshotOutputFormat,
quality: Option<u8>,
/// True when a post-action screenshot was cropped to a target or focused window.
#[serde(skip_serializing_if = "Option::is_none")]
cropped_to_window: Option<bool>,
/// Title of the window when cropped to a window.
#[serde(skip_serializing_if = "Option::is_none")]
window_title: Option<String>,
/// The file the image was saved to when screenshot_file_path was given;
/// no image content block is returned then. bytes is its size.
#[serde(skip_serializing_if = "Option::is_none")]
file_path: Option<String>,
}
impl From<&ScreenshotCapture> for ScreenshotSummary {
fn from(capture: &ScreenshotCapture) -> Self {
Self {
mime_type: capture.mime_type.clone(),
source: capture.source.clone(),
width: capture.width,
height: capture.height,
coordinate_width: capture.coordinate_width,
coordinate_height: capture.coordinate_height,
scale: capture.scale,
resized: capture.resized,
bytes: capture.bytes,
original_bytes: capture.original_bytes,
max_bytes: capture.max_bytes,
format: capture.format,
quality: capture.quality,
cropped_to_window: None,
window_title: None,
file_path: None,
}
}
}
/// Assemble a structured tool result: the screenshot (when present) as a
/// structured image block first, then the JSON report as text and as
/// `structured_content`. Inlining the base64 in the JSON made hosts treat a
/// 100KB+ image as plain text tokens (issue #145).
fn structured_tool_result<T: Serialize>(
output: T,
screenshot: Option<ScreenshotCapture>,
) -> Result<CallToolResult, ErrorData> {
let value = serde_json::to_value(&output).map_err(|error| {
ErrorData::internal_error(format!("failed to serialize tool output: {error}"), None)
})?;
let is_error = value.get("ok").and_then(serde_json::Value::as_bool) == Some(false);
let mut result = CallToolResult::structured(value);
result.is_error = Some(is_error);
if let Some(capture) = screenshot {
result.content.insert(
0,
ContentBlock::image(data_url_payload(&capture.data_url), capture.mime_type),
);
}
Ok(result)
}
fn app_state_result(
output: GetAppStateOutput,
screenshot: Option<ScreenshotCapture>,
) -> Result<CallToolResult, ErrorData> {
structured_tool_result(output, screenshot)
}
#[derive(Debug, Clone, Default, Deserialize, Serialize, JsonSchema)]
struct GetTreePageParams {
/// First node to return, counted from the start of the cached tree (default 0).
#[serde(default)]
offset: Option<usize>,
/// snapshot_id of the get_app_state or include_tree result being paged.
/// If a later snapshot replaced it, no nodes are returned.
#[serde(default)]
snapshot_id: Option<String>,
}
#[derive(Debug, Clone, Serialize, JsonSchema)]
struct GetTreePageOutput {
/// The paged snapshot; pass it with element_index actions on these nodes.
/// Absent when no tree is cached or the named snapshot was replaced.
#[serde(skip_serializing_if = "Option::is_none")]
snapshot_id: Option<String>,
/// Nodes from `offset`, as many as fit in one result.
nodes: TreeNodes,
offset: usize,
/// Offset of the next page, or null when this page ends the tree.
next_offset: Option<usize>,
/// Nodes in the cached tree.
node_count: usize,
message: String,
}
/// A page of `nodes` from `offset` that fits the inline budget. Always at least
/// one node, so paging advances even past a node larger than the budget.
fn tree_page(nodes: &[AccessibilityNode], detail: NodeDetail, offset: usize) -> GetTreePageOutput {
let node_count = nodes.len();
if node_count == 0 {
return GetTreePageOutput {
snapshot_id: None,
nodes: TreeNodes::new(Vec::new(), detail),
offset,
next_offset: None,
node_count,
message: "No accessibility tree is cached; call get_app_state first.".to_string(),
};
}
let start = offset.min(node_count);
let rest = &nodes[start..];
let fit = crate::tree_spill::inline_prefix_len(rest, detail, tree_budget(TREE_PAGE_RESERVE))
.max(usize::from(!rest.is_empty()));
let end = start + fit;
let next_offset = (end < node_count).then_some(end);
let message = match next_offset {
Some(next) => format!(
"Nodes {start}-{} of {node_count}; continue with offset {next}.",
end - 1
),
None if start < end => format!(
"Nodes {start}-{} of {node_count}; this is the end of the tree.",
end - 1
),
None => format!("Offset {offset} is past the end of the {node_count}-node tree."),
};
GetTreePageOutput {
snapshot_id: None,
nodes: TreeNodes::new(rest[..fit].to_vec(), detail),
offset: start,
next_offset,
node_count,
message,
}
}
/// Bytes kept for a tree page's other fields.
const TREE_PAGE_RESERVE: usize = 500;
/// What one get_app_state call found, in the terms its readiness block makes
/// claims about.
#[derive(Debug, Clone, Copy)]
struct AppStateObservation<'a> {
screenshot_failed: bool,
window: WindowLookup,
/// Window backends that failed in a lookup another backend answered.
failed_window_backends: &'a [&'static str],
tree: TreeRead,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum WindowLookup {
NotRequested,
Resolved,
/// The window list failed, or listed no window matching the target.
Failed,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum TreeRead {
Nodes,
/// The read failed or returned no nodes, as the post-action observation
/// also counts it.
Failed,
/// Not read, because this call found accessibility off, either in a probe
/// it ran or in the live check that agreed with the cached report.
AccessibilityOff,
}
/// How an observation disagrees with the readiness the call would report.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Contradiction {
/// Something failed. The desktop may still be changing, so the next call
/// probes again as well, to see a recovery.
Failure,
/// Something worked that the readiness says cannot. The refreshed report
/// postdates that success, so the next call can reuse it. If the
/// contradiction persists, that call refreshes again.
Success,
}
/// Why a get_app_state call must refresh `readiness` before it reports it, or
/// `None` when the readiness stands. `windowing` comes from the same report.
fn readiness_contradiction(
readiness: &ReadinessReport,
windowing: &WindowingReport,
observed: AppStateObservation,
) -> Option<Contradiction> {
// A backend the report already calls unhealthy fails as predicted. On KDE
// both GNOME backends fail before KWin answers, on every call.
let healthy_backend_failed = observed.failed_window_backends.iter().any(|backend| {
windowing
.backends
.get(*backend)
.is_some_and(|check| check.ok)
});
if observed.screenshot_failed
|| observed.window == WindowLookup::Failed
|| healthy_backend_failed
|| observed.tree == TreeRead::Failed
{
Some(Contradiction::Failure)
} else if observed.window == WindowLookup::Resolved && !readiness.can_query_windows {
Some(Contradiction::Success)
} else {
None
}
}
#[derive(Debug, Clone, Serialize, JsonSchema)]
struct GetAppStateOutput {
app_name_or_bundle_identifier: Option<String>,
window_context: Option<WindowInfo>,
window_error: Option<String>,
window_permissions_hint: Option<String>,
backend: String,
/// Screenshot metadata; the image itself is the first `image` content block.
screenshot: Option<ScreenshotSummary>,
screenshot_error: Option<String>,
/// Names this result's tree. Pass it with element_index or selector
/// actions and get_tree_page: a call against a tree a later snapshot
/// replaced is refused. Absent when the tree read failed.
#[serde(skip_serializing_if = "Option::is_none")]
snapshot_id: Option<String>,
/// Compacted tree, or its leading nodes when the whole tree is in
/// `accessibility_tree_spill`. Nodes use the compact encoding (empty and
/// default fields left out, actions by name) unless verbose=true.
accessibility_tree: TreeNodes,
accessibility_tree_raw_count: usize,
/// Set when the compacted tree was too large to return inline: the whole
/// tree is in this private file, with the same element_index values.
#[serde(skip_serializing_if = "Option::is_none")]
accessibility_tree_spill: Option<crate::tree_spill::TreeSpill>,
/// True when the snapshot was limited to selected app roots rather than the
/// full desktop AT-SPI registry.
tree_scoped: bool,
/// True when max_nodes, max_depth, the child read budget, or the 10-second
/// deadline stopped traversal with unread elements left, so the tree is
/// incomplete. Recover by scoping to a narrower app or window target and
/// raising max_nodes or max_depth (hard caps 10000 and 64). Failed element
/// reads do not count.
accessibility_tree_truncated: bool,
accessibility_error: Option<String>,
/// Compact readiness summary (always present).
readiness: crate::diagnostics::ReadinessReport,
/// Full diagnostics; populated only when verbose=true.
#[serde(skip_serializing_if = "Option::is_none")]
diagnostics: Option<DoctorReport>,
message: String,
}
#[derive(Debug, Clone, Copy, Default, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
enum ClickCoordinateSpace {
#[default]
Capture,
Atspi,
}
#[derive(Debug, Clone, Default, Deserialize, Serialize, JsonSchema)]
struct ClickParams {
/// Element index from the latest get_app_state or include_tree snapshot.
#[serde(default)]
element_index: Option<u32>,
/// snapshot_id of the get_app_state or include_tree result the
/// element_index or selector comes from. If a later snapshot replaced
/// it, the call is refused before any input. Without it, the element
/// resolves in the latest snapshot.
#[serde(default)]
snapshot_id: Option<String>,
/// Case-insensitive substring of the element's AT-SPI role.
#[serde(default)]
role: Option<String>,
/// Case-insensitive substring of the element's accessible name.
#[serde(default)]
name: Option<String>,
/// Case-insensitive substring of the element's text, name or description.
#[serde(default)]
text: Option<String>,
/// AT-SPI states the element must all have, case-insensitive (for
/// example focused, checked).
#[serde(default)]
states: Vec<String>,
/// Point x in raw screenshot capture pixels: the coordinate_width and
/// coordinate_height space. Divide preview pixels by the screenshot scale
/// first. With relative=true, measured from the window screenshot's crop origin.
/// With coordinate_space="atspi", a global AT-SPI screen x instead.
#[serde(default)]
x: Option<i32>,
/// Point y in raw screenshot capture pixels: the coordinate_width and
/// coordinate_height space. Divide preview pixels by the screenshot scale
/// first. With relative=true, measured from the window screenshot's crop origin.
/// With coordinate_space="atspi", a global AT-SPI screen y instead.
#[serde(default)]
y: Option<i32>,
/// Pixel space for explicit x/y: raw screenshot capture pixels (default),
/// or global AT-SPI screen coordinates mapped through the element's KWin window.
#[serde(default)]
coordinate_space: ClickCoordinateSpace,
/// Mouse button: left (default), right or middle. side, extra, forward and
/// back reach only the portal and ydotool routes, which treat an unknown
/// name as left. Native X11 refuses any other name.
#[serde(default)]
button: Option<String>,
/// Number of clicks (default 1), clamped to 1 to 10.
#[serde(default)]
click_count: Option<u32>,
// Optional window target: when set, the window is raised/focused before the
// click so a coordinate click reliably lands on the intended app rather than
// whatever window happens to be stacked on top at that pixel.
/// Window id from a recent list_windows. The first target field set
/// decides, in this order: window_id, pid, app_id, wm_class, window_title.
#[serde(
default,
deserialize_with = "crate::windowing::types::deserialize_window_id"
)]
#[schemars(range(max = crate::windowing::types::MAX_WINDOW_ID))]
window_id: Option<u64>,
/// Process id. It must match exactly one window.
#[serde(default)]
pid: Option<u32>,
/// Application id, ignoring ASCII case: the full desktop id, or its last
/// dot-separated segment. A segment that matches several apps is
/// rejected.
#[serde(default)]
app_id: Option<String>,
/// Window manager class, an exact match ignoring ASCII case. When several
/// windows match, the first wins.
#[serde(default)]
wm_class: Option<String>,
/// Window title substring, ignoring ASCII case. When several windows
/// match, the first wins.
/// This tool names the field window_title; a title field is ignored.
#[serde(default)]
window_title: Option<String>,
/// Interpret `x`/`y` from the clipped window screenshot crop origin, in
/// coordinate pixels before preview resizing. Divide preview pixels by its
/// scale first. This is not a toolkit widget or raw GDK surface origin.
/// Requires a window target; missing targets are rejected.
#[serde(default)]
relative: Option<bool>,
#[serde(flatten)]
capture: PostActionCaptureParams,
}
impl ClickParams {
/// A window target if any window-identifying field was supplied.
fn window_target(&self) -> Option<WindowTarget> {
optional_window_target(
self.window_id,
self.pid,
&self.app_id,
&self.wm_class,
&self.window_title,
)
}
fn selector(&self) -> ElementSelector<'_> {
ElementSelector {
role: self.role.as_deref(),
name: self.name.as_deref(),
text: self.text.as_deref(),
states: &self.states,
}
}
}
/// The window target named by the click, scroll, perform_action and set_value
/// fields, if any of them was supplied.
fn optional_window_target(
window_id: Option<u64>,
pid: Option<u32>,
app_id: &Option<String>,
wm_class: &Option<String>,
window_title: &Option<String>,
) -> Option<WindowTarget> {
if window_id.is_none()
&& pid.is_none()
&& app_id.is_none()
&& wm_class.is_none()
&& window_title.is_none()
{
return None;
}
Some(WindowTarget {
window_id,
pid,
tty: None,
terminal_pid: None,
terminal_command: None,
terminal_cwd: None,
app_id: app_id.clone(),
wm_class: wm_class.clone(),
title: window_title.clone(),
})
}
/// The optional app or window target of perform_action and set_value, the
/// same fields click and scroll take. AT-SPI actions need no focus, so the
/// target is not raised; it names the app whose get_app_state snapshot a
/// cached element_index or selector must come from.
#[derive(Debug, Clone, Default, Deserialize, Serialize, JsonSchema)]
struct ElementActionTarget {
/// Window id from a recent list_windows. The first target field set
/// decides, in this order: window_id, pid, app_id, wm_class, window_title.
#[serde(
default,
deserialize_with = "crate::windowing::types::deserialize_window_id"
)]
#[schemars(range(max = crate::windowing::types::MAX_WINDOW_ID))]
window_id: Option<u64>,
/// Process id of the target app: an element another process owns is
/// refused. With pid alone, no window is needed. When pid picks the window
/// (with app_id, wm_class or window_title, or for
/// screenshot_after="window"), it must match exactly one.
#[serde(default)]
pid: Option<u32>,
/// Application id, ignoring ASCII case: the full desktop id, or its last
/// dot-separated segment. A segment that matches several apps is
/// rejected.
#[serde(default)]
app_id: Option<String>,
/// Window manager class, an exact match ignoring ASCII case. When several
/// windows match, the first wins.
#[serde(default)]
wm_class: Option<String>,
/// Window title substring, ignoring ASCII case. When several windows
/// match, the first wins.
/// This tool names the field window_title; a title field is ignored.
#[serde(default)]
window_title: Option<String>,
}
impl ElementActionTarget {
/// A window target if any window-identifying field was supplied.
fn window_target(&self) -> Option<WindowTarget> {
optional_window_target(
self.window_id,
self.pid,
&self.app_id,
&self.wm_class,
&self.window_title,
)
}
}
#[derive(Debug, Clone, Default, Deserialize, Serialize, JsonSchema)]
struct ActionParams {
/// Element index from the latest get_app_state or include_tree snapshot.
#[serde(default)]
element_index: Option<u32>,
/// snapshot_id of the get_app_state or include_tree result the
/// element_index or selector comes from. If a later snapshot replaced
/// it, the call is refused before any input. Without it, the element
/// resolves in the latest snapshot.
#[serde(default)]
snapshot_id: Option<String>,
/// A node's object_ref from get_app_state or get_tree_page. It names the
/// AT-SPI object itself, not a snapshot index, and wins over element_index
/// and the selectors.
#[serde(default)]
element_identifier: Option<String>,
/// Case-insensitive substring of the element's AT-SPI role.
#[serde(default)]
role: Option<String>,
/// Case-insensitive substring of the element's accessible name.
#[serde(default)]
name: Option<String>,
/// Case-insensitive substring of the element's text, name or description.
#[serde(default)]
text: Option<String>,
/// AT-SPI states the element must all have, case-insensitive (for
/// example focused, checked).
#[serde(default)]
states: Vec<String>,
/// AT-SPI action to invoke: its name or description, ignoring ASCII case,
/// or its 0-based index. Default: the element's first action.
#[serde(default)]
action: Option<String>,
#[serde(flatten)]
target: ElementActionTarget,
#[serde(flatten)]
capture: PostActionCaptureParams,
}
impl ActionParams {
fn selector(&self) -> ElementSelector<'_> {
ElementSelector {
role: self.role.as_deref(),
name: self.name.as_deref(),
text: self.text.as_deref(),
states: &self.states,
}
}
}
#[derive(Debug, Clone, Default, Deserialize, Serialize, JsonSchema)]
struct SetValueParams {
/// Element index from the latest get_app_state or include_tree snapshot.
#[serde(default)]
element_index: Option<u32>,
/// snapshot_id of the get_app_state or include_tree result the
/// element_index or selector comes from. If a later snapshot replaced
/// it, the call is refused before any input. Without it, the element
/// resolves in the latest snapshot.
#[serde(default)]
snapshot_id: Option<String>,
/// A node's object_ref from get_app_state or get_tree_page. It names the
/// AT-SPI object itself, not a snapshot index, and wins over element_index
/// and the selectors.
#[serde(default)]
element_identifier: Option<String>,
/// Case-insensitive substring of the element's AT-SPI role.
#[serde(default)]
role: Option<String>,
/// Case-insensitive substring of the element's accessible name.
#[serde(default)]
name: Option<String>,
/// Case-insensitive substring of the element's text, name or description.
#[serde(default)]
text: Option<String>,
/// AT-SPI states the element must all have, case-insensitive (for
/// example focused, checked).
#[serde(default)]
states: Vec<String>,
/// A value that parses as a floating-point number (such as 5, -2.5, 1e3,
/// inf or NaN, with no surrounding spaces) sets an element with the
/// AT-SPI Value interface (slider, spin button). Otherwise the text
/// replaces an editable text element's contents. Text on a Value-only
/// element is refused.
value: String,
#[serde(flatten)]
target: ElementActionTarget,
#[serde(flatten)]
capture: PostActionCaptureParams,
}
impl SetValueParams {
fn selector(&self) -> ElementSelector<'_> {
ElementSelector {
role: self.role.as_deref(),
name: self.name.as_deref(),
text: self.text.as_deref(),
states: &self.states,
}
}
}
#[derive(Debug, Clone, Default, Deserialize, Serialize, JsonSchema)]
struct ScrollParams {
/// Element index from the latest get_app_state or include_tree snapshot.
#[serde(default)]
element_index: Option<u32>,
/// snapshot_id of the get_app_state or include_tree result the
/// element_index comes from. If a later snapshot replaced it, the call
/// is refused before any input. Without it, the element resolves in the
/// latest snapshot.
#[serde(default)]
snapshot_id: Option<String>,
/// Point x to scroll at in raw screenshot capture pixels: the coordinate_width and
/// coordinate_height space. Divide preview pixels by the screenshot scale
/// first. With relative=true, measured from the window screenshot's crop origin.
/// Without x/y or element_index, the target window's centre, or the pointer.
#[serde(default)]
x: Option<i32>,
/// Point y to scroll at in raw screenshot capture pixels: the coordinate_width and
/// coordinate_height space. Divide preview pixels by the screenshot scale
/// first. With relative=true, measured from the window screenshot's crop origin.
#[serde(default)]
y: Option<i32>,
/// up, down, left or right (case-insensitive).
direction: String,
/// Distance in pages (default 1), 5 wheel notches each. The sign is
/// ignored (direction sets it), and any value sends at least one notch.
#[serde(default)]
#[schemars(range(min = -100, max = 100))]
pages: Option<f64>,
// Optional window target (parity with click): the window is raised/focused
// before scrolling so the wheel events land on the intended app.
/// Window id from a recent list_windows. The first target field set
/// decides, in this order: window_id, pid, app_id, wm_class, window_title.
#[serde(
default,
deserialize_with = "crate::windowing::types::deserialize_window_id"
)]
#[schemars(range(max = crate::windowing::types::MAX_WINDOW_ID))]
window_id: Option<u64>,
/// Process id. It must match exactly one window.
#[serde(default)]
pid: Option<u32>,
/// Application id, ignoring ASCII case: the full desktop id, or its last
/// dot-separated segment. A segment that matches several apps is
/// rejected.
#[serde(default)]
app_id: Option<String>,
/// Window manager class, an exact match ignoring ASCII case. When several
/// windows match, the first wins.
#[serde(default)]
wm_class: Option<String>,
/// Window title substring, ignoring ASCII case. When several windows
/// match, the first wins.
/// This tool names the field window_title; a title field is ignored.
#[serde(default)]
window_title: Option<String>,
/// Interpret `x`/`y` from the clipped window screenshot crop origin, in
/// coordinate pixels before preview resizing. Divide preview pixels by its
/// scale first. This is not a toolkit widget or raw GDK surface origin.
/// Requires a window target; missing targets are rejected.
#[serde(default)]
relative: Option<bool>,
#[serde(flatten)]
capture: PostActionCaptureParams,
}
/// Whether a click resolves its element in the cached tree: an index or
/// selector, unless a full capture-space point wins over it.
fn click_resolves_cached_element(params: &ClickParams) -> bool {
(params.element_index.is_some() || !params.selector().is_empty())
&& (params.x.zip(params.y).is_none()
|| params.coordinate_space == ClickCoordinateSpace::Atspi)
}
/// Whether a scroll takes its point from a cached element: an element_index
/// without a full x/y point. A lone x or y does not make a point.
fn scroll_resolves_cached_element(params: &ScrollParams) -> bool {
params.element_index.is_some() && params.x.zip(params.y).is_none()
}
fn scroll_units(pages: Option<f64>) -> std::result::Result<i32, String> {
let pages = pages.unwrap_or(1.0);
if !pages.is_finite() || pages.abs() > MAX_SCROLL_PAGES {
return Err(format!(
"pages must be finite and have an absolute value no greater than {MAX_SCROLL_PAGES}"
));
}
Ok(((pages.abs().max(0.1) * 5.0).round() as i32).max(1))
}
impl ScrollParams {
/// A window target if any window-identifying field was supplied.
fn window_target(&self) -> Option<WindowTarget> {
optional_window_target(
self.window_id,
self.pid,
&self.app_id,
&self.wm_class,
&self.window_title,
)
}
}
#[derive(Debug, Clone, Default, Deserialize, Serialize, JsonSchema)]
struct DragParams {
/// Start x in raw screenshot capture pixels: the coordinate_width space.
/// Divide preview pixels by the screenshot scale first.
start_x: i32,
/// Start y in raw screenshot capture pixels.
start_y: i32,
/// End x in raw screenshot capture pixels.
end_x: i32,
/// End y in raw screenshot capture pixels.
end_y: i32,
#[serde(flatten)]
capture: PostActionCaptureParams,
}
#[derive(Debug, Clone, Default, Deserialize, Serialize, JsonSchema)]
struct PressKeyParams {
/// Key or chord in the grammar the tool description gives, for example
/// Enter or Ctrl+Shift+T.
key: String,
/// Window id from a recent list_windows. The first target field set
/// decides, in this order: window_id, terminal fields, pid, app_id,
/// wm_class, title. With a terminal field, pid, app_id, wm_class and title
/// only narrow the terminal match, which must name one window.
#[serde(
default,
deserialize_with = "crate::windowing::types::deserialize_window_id"
)]
#[schemars(range(max = crate::windowing::types::MAX_WINDOW_ID))]
window_id: Option<u64>,
/// Process id. It must match exactly one window.
#[serde(default)]
pid: Option<u32>,
/// Terminal tty: /dev/pts/N, pts/N or N.
#[serde(default)]
tty: Option<String>,
/// Pid of the shell or foreground process on a terminal's tty, not the
/// terminal emulator (use pid for that).
#[serde(default)]
terminal_pid: Option<u32>,
/// Substring of the command name or command line of a terminal's shell
/// or foreground process, ignoring ASCII case.
#[serde(default)]
terminal_command: Option<String>,
/// Working directory of a terminal's shell or foreground process: an
/// absolute path, or a trailing path such as project/src.
#[serde(default)]
terminal_cwd: Option<String>,
/// Application id, ignoring ASCII case: the full desktop id, or its last
/// dot-separated segment outside a terminal match. A segment that matches several apps is
/// rejected.
#[serde(default)]
app_id: Option<String>,
/// Window manager class, an exact match ignoring ASCII case. When several
/// windows match outside a terminal match, the first wins.
#[serde(default)]
wm_class: Option<String>,
/// Window title substring, ignoring ASCII case. When several windows
/// match outside a terminal match, the first wins.
#[serde(default)]
title: Option<String>,
#[serde(flatten)]
capture: PostActionCaptureParams,
}
#[derive(Debug, Clone, Default, Deserialize, Serialize, JsonSchema)]
struct TypeTextParams {
/// Literal text to type, sent as characters, not as key names.
text: String,
/// Window id from a recent list_windows. The first target field set
/// decides, in this order: window_id, terminal fields, pid, app_id,
/// wm_class, title. With a terminal field, pid, app_id, wm_class and title
/// only narrow the terminal match, which must name one window.
#[serde(
default,
deserialize_with = "crate::windowing::types::deserialize_window_id"
)]
#[schemars(range(max = crate::windowing::types::MAX_WINDOW_ID))]
window_id: Option<u64>,
/// Process id. It must match exactly one window.
#[serde(default)]
pid: Option<u32>,
/// Terminal tty: /dev/pts/N, pts/N or N.
#[serde(default)]
tty: Option<String>,
/// Pid of the shell or foreground process on a terminal's tty, not the
/// terminal emulator (use pid for that).
#[serde(default)]
terminal_pid: Option<u32>,
/// Substring of the command name or command line of a terminal's shell
/// or foreground process, ignoring ASCII case.
#[serde(default)]
terminal_command: Option<String>,
/// Working directory of a terminal's shell or foreground process: an
/// absolute path, or a trailing path such as project/src.
#[serde(default)]
terminal_cwd: Option<String>,
/// Application id, ignoring ASCII case: the full desktop id, or its last
/// dot-separated segment outside a terminal match. A segment that matches several apps is
/// rejected.
#[serde(default)]
app_id: Option<String>,
/// Window manager class, an exact match ignoring ASCII case. When several
/// windows match outside a terminal match, the first wins.
#[serde(default)]
wm_class: Option<String>,
/// Window title substring, ignoring ASCII case. When several windows
/// match outside a terminal match, the first wins.
#[serde(default)]
title: Option<String>,
#[serde(flatten)]
capture: PostActionCaptureParams,
}
impl PressKeyParams {
fn window_target(&self) -> WindowTarget {
WindowTarget {
window_id: self.window_id,
pid: self.pid,
tty: self.tty.clone(),
terminal_pid: self.terminal_pid,
terminal_command: self.terminal_command.clone(),
terminal_cwd: self.terminal_cwd.clone(),
app_id: self.app_id.clone(),
wm_class: self.wm_class.clone(),
title: self.title.clone(),
}
}
}
impl TypeTextParams {
fn window_target(&self) -> WindowTarget {
WindowTarget {
window_id: self.window_id,
pid: self.pid,
tty: self.tty.clone(),
terminal_pid: self.terminal_pid,
terminal_command: self.terminal_command.clone(),
terminal_cwd: self.terminal_cwd.clone(),
app_id: self.app_id.clone(),
wm_class: self.wm_class.clone(),
title: self.title.clone(),
}
}
}
#[derive(Debug, Clone, Serialize, JsonSchema)]
struct ActionOutput {
ok: bool,
implemented: bool,
action: String,
message: String,
// See ActivateWindowOutput: kept in the response, omitted from the schema
// because `serde_json::Value` produces a non-object schema strict MCP
// clients reject.
#[schemars(skip)]
received: Option<serde_json::Value>,
}
#[derive(Debug, Clone, Serialize, JsonSchema)]
struct ActionToolOutput {
#[serde(flatten)]
action: ActionOutput,
#[serde(flatten)]
observation: PostActionObservation,
}
#[derive(Debug, Clone, Serialize, JsonSchema)]
struct SetValueToolOutput {
#[serde(flatten)]
action: ActionOutput,
/// Exact editable-text readback result; omitted for numeric values and early failures.
#[serde(skip_serializing_if = "Option::is_none")]
verified: Option<bool>,
#[serde(flatten)]
observation: PostActionObservation,
}
/// Start of every type_text success message from the AT-SPI route. Those
/// results were read back, so type_text does not add its unverified note.
const TYPE_TEXT_ATSPI_ROUTE: &str = "Typed through AT-SPI EditableText";
/// Start of an AT-SPI route failure after an edit was attempted.
const TYPE_TEXT_ATSPI_UNVERIFIED: &str = "AT-SPI EditableText insert was not verified";
/// The portal preflight refusal that may take the AT-SPI route: the keymap
/// was read and a layout group cannot type a character. A keymap that could
/// not be read or checked keeps its error.
fn atspi_fallback_trigger(
error: &anyhow::Error,
) -> Option<crate::keyboard_keymap::MissingLayoutKeysym> {
error
.downcast_ref::<crate::keyboard_keymap::MissingLayoutKeysym>()
.copied()
}
/// The first character that portal typing sends as a key press (Enter, Tab,
/// BackSpace, Escape, Delete and the like), not as text.
fn key_press_character(text: &str) -> Option<char> {
text.chars().find(|ch| ch.is_control())
}
fn type_text_control_character_refusal(refusal: &str, key: char) -> String {
format!(
"{refusal} The AT-SPI text fallback was not used: the text contains U+{:04X}, which typing sends as a key press such as Enter or Tab, and inserting it is not the same. Type the text without it, then send the key with press_key.",
key as u32
)
}
/// The type_text result for one AT-SPI insert outcome. `Ok` is a verified
/// success message.
fn type_text_atspi_result(
refusal: &str,
missing: crate::keyboard_keymap::MissingLayoutKeysym,
text: &str,
insert: Result<FocusedTextInsert>,
) -> std::result::Result<String, String> {
const MAY_HAVE_CHANGED: &str = "The field may have changed; re-observe it before retrying.";
let because = format!("because GNOME {}", missing.summary());
let route = format!("{TYPE_TEXT_ATSPI_ROUTE} {because}");
let unverified = format!("{TYPE_TEXT_ATSPI_UNVERIFIED}: the text was sent {because}");
match insert {
Ok(FocusedTextInsert::Verified {
role,
replaced_selection,
caret_after_text,
}) => {
let mut message = format!(
"{route}. The focused {role} reads back the text at the insertion point, with the rest of the field unchanged."
);
if replaced_selection {
message.push_str(" The text replaced the selection.");
}
if !caret_after_text {
message.push_str(" The caret could not be moved after the text.");
}
Ok(message)
}
Ok(FocusedTextInsert::NotEditable(reason)) => {
Err(format!("{refusal} No AT-SPI text fallback: {reason}."))
}
Ok(FocusedTextInsert::PasswordField) => Err(format!(
"{refusal} The focused field is a password field, so the AT-SPI text fallback was not used. set_value can fill it."
)),
Ok(FocusedTextInsert::Mismatch {
expected_count,
read_count,
read_text,
read_truncated,
}) => Err(format!(
"{unverified}, but the field does not read it back: expected {text:?} at the insertion point, the rest unchanged, and {expected_count} characters in all; read {read_text:?}{} at the insertion point and {read_count} characters. {MAY_HAVE_CHANGED}",
if read_truncated { " (first part)" } else { "" }
)),
Ok(FocusedTextInsert::ReadbackFailed) => Err(format!(
"{unverified}, but reading the field back failed. {MAY_HAVE_CHANGED}"
)),
Ok(FocusedTextInsert::ReadbackTimedOut) => Err(format!(
"{unverified}, but reading the field back timed out. {MAY_HAVE_CHANGED}"
)),
Ok(FocusedTextInsert::EditFailed {
reason,
may_have_changed,
}) => Err(format!(
"{TYPE_TEXT_ATSPI_UNVERIFIED}: {reason}. {}",
if may_have_changed {
MAY_HAVE_CHANGED
} else {
"No text was changed."
}
)),
Err(error) => Err(format!("{refusal} No AT-SPI text fallback: {error:#}.")),
}
}
fn editable_text_action_result(
outcome: EditableTextSetOutcome,
received: Option<serde_json::Value>,
) -> (ActionOutput, Option<bool>) {
let (ok, verified, message) = match outcome {
EditableTextSetOutcome::Verified => (
true,
Some(true),
"AT-SPI editable text contents set and verified.",
),
EditableTextSetOutcome::MismatchAfterDeadline => (
false,
Some(false),
"AT-SPI EditableText returned success, but the resulting contents did not match the requested value. The contents may be unchanged, normalized, or partially changed; re-observe the element before retrying.",
),
EditableTextSetOutcome::ReadbackFailed => (
false,
Some(false),
"AT-SPI EditableText returned success, but content verification failed. The value may have changed; re-observe the element before retrying.",
),
EditableTextSetOutcome::ReadbackTimedOut => (
false,
Some(false),
"AT-SPI EditableText returned success, but content verification timed out. The value may have changed; re-observe the element before retrying.",
),
EditableTextSetOutcome::PasswordUnverifiable => (
true,
Some(false),
"AT-SPI editable text contents dispatched to a password field; exact contents cannot be verified.",
),
};
(
ActionOutput {
ok,
implemented: true,
action: "set_value".to_string(),
message: message.to_string(),
received,
},
verified,
)
}
impl ComputerUseLinux {
fn is_wayland_session(&self) -> bool {
crate::diagnostics::hydrate_session_bus_env();
let session_type = env::var("XDG_SESSION_TYPE").ok();
let wayland_display = env::var("WAYLAND_DISPLAY").ok();
session_is_wayland(session_type.as_deref(), wayland_display.as_deref())
}
// The Wayland remote-desktop portal is a fallback for input: when a
// compatible ydotool CLI and working `ydotoold` socket are present we
// prefer ydotool, because it injects input without a permission prompt.
// Without `COMPUTER_USE_LINUX_PERSIST_REMOTE_DESKTOP=1` the portal asks
// again on every new process. That opt-in sends `persist_mode=2` on
// RemoteDesktop SelectDevices and reuses the single-use restore token.
// `COMPUTER_USE_LINUX_FORCE_YDOTOOL_*=1` always uses ydotool;
// `COMPUTER_USE_LINUX_FORCE_PORTAL_*=1` always uses the portal.
async fn should_prefer_portal_pointer_backend(&self) -> bool {
if env_flag_enabled("COMPUTER_USE_LINUX_FORCE_YDOTOOL_POINTER") {
return false;
}
if env_flag_enabled("COMPUTER_USE_LINUX_FORCE_PORTAL_POINTER") {
return self.is_wayland_session();
}
// COSMIC maps uinput absolute axes to the active output, not the
// desktop bounding box. Portal streams address each output directly.
if self.is_cosmic_wayland_session() {
return true;
}
should_prefer_portal_backend_by_default(
self.is_wayland_session(),
ydotool_backend_available().await,
)
}
// Literal text needs a layout-aware route even when raw ydotool input is
// available. KDE uses clipboard paste; compatible compositors use wtype.
fn should_prefer_portal_text_backend(&self) -> bool {
prefer_portal_text_backend(
env_flag_enabled("COMPUTER_USE_LINUX_FORCE_YDOTOOL_KEYBOARD"),
self.should_prefer_xdotool_keyboard(),
self.is_wayland_session(),
self.is_kde_wayland_session(),
env_flag_enabled("COMPUTER_USE_LINUX_FORCE_PORTAL_KEYBOARD"),
self.should_prefer_wtype_keyboard(),
)
}
/// Portal keyboard policy for `press_key` chords. Unlike literal text
/// (where KDE prefers the clipboard paste backend), key chords have no
/// clipboard route, so the portal keyboard session is preferred on ANY
/// Wayland session, including Plasma. An already-active keyboard
/// session (e.g. established by a KDE clipboard paste) is reused even
/// when ydotool is available, so the consent the user already granted
/// keeps covering key chords; otherwise the portal is preferred only
/// when ydotool is absent or the portal is forced.
async fn should_prefer_portal_keyboard_for_chords(&self) -> bool {
if env_flag_enabled("COMPUTER_USE_LINUX_FORCE_YDOTOOL_KEYBOARD") {
return false;
}
if self.should_prefer_xdotool_keyboard() {
return false;
}
if !self.is_wayland_session() {
return false;
}
if self.cached_portal_keyboard_session().is_some()
|| env_flag_enabled("COMPUTER_USE_LINUX_FORCE_PORTAL_KEYBOARD")
{
return true;
}
!ydotool_backend_available().await
}
/// Chord route for `press_key`. The portal stays preferred wherever it
/// exists; the portal probe only runs when wtype could replace it.
async fn chord_route(&self) -> ChordRoute {
let prefer_portal = self.should_prefer_portal_keyboard_for_chords().await;
let portal_pinned = self.cached_portal_keyboard_session().is_some()
|| env_flag_enabled("COMPUTER_USE_LINUX_FORCE_PORTAL_KEYBOARD");
let wtype = self.should_prefer_wtype_keyboard();
let portal_missing = prefer_portal
&& !portal_pinned
&& wtype
&& crate::remote_desktop::remote_desktop_portal_missing().await;
chord_route_from(prefer_portal, portal_pinned, wtype, portal_missing)
}
fn should_prefer_kde_clipboard_text_backend(&self) -> bool {
!env_flag_enabled("COMPUTER_USE_LINUX_FORCE_YDOTOOL_KEYBOARD")
&& !self.should_prefer_xdotool_keyboard()
&& self.is_kde_wayland_session()
}
/// Keyboard policy for X11 sessions: prefer `xdotool` (XTEST).
///
/// ydotool writes raw evdev scancodes to a virtual uinput device. On X11
/// the server then re-interprets them through the active XKB layout, so
/// `press_key "Return"` and chords like `ctrl+a` land as stray characters,
/// and literal text can mangle symbols/digits (issue #58). XTEST resolves
/// keysyms against the live layout instead.
///
/// `COMPUTER_USE_LINUX_FORCE_YDOTOOL_KEYBOARD=1` opts out;
/// `COMPUTER_USE_LINUX_FORCE_XDOTOOL_KEYBOARD=1` forces it on.
fn should_prefer_xdotool_keyboard(&self) -> bool {
prefer_xdotool_keyboard(
env_flag_enabled("COMPUTER_USE_LINUX_FORCE_YDOTOOL_KEYBOARD"),
env_flag_enabled("COMPUTER_USE_LINUX_FORCE_XDOTOOL_KEYBOARD"),
self.is_wayland_session(),
env_var_non_empty("DISPLAY"),
xdotool_available(),
)
}
fn should_prefer_wtype_keyboard(&self) -> bool {
prefer_wtype_keyboard(
env_flag_enabled("COMPUTER_USE_LINUX_FORCE_YDOTOOL_KEYBOARD"),
self.is_wayland_session(),
crate::diagnostics::wtype_compatible_wayland_desktop(
env::var("XDG_CURRENT_DESKTOP").ok().as_deref(),
),
wtype_available(),
)
}
fn wtype_text_mode(&self) -> Option<WtypeTextMode> {
if self.should_prefer_xdotool_keyboard()
|| self.should_prefer_kde_clipboard_text_backend()
|| self.should_prefer_portal_text_backend()
|| !self.should_prefer_wtype_keyboard()
{
return None;
}
Some(wtype_text_mode_for_desktop(
env::var("XDG_CURRENT_DESKTOP").ok().as_deref(),
))
}
fn should_prefer_xdotool_pointer(&self) -> bool {
crate::diagnostics::hydrate_session_bus_env();
prefer_xdotool_pointer(
env_flag_enabled("COMPUTER_USE_LINUX_FORCE_YDOTOOL_POINTER"),
env::var("XDG_SESSION_TYPE").ok().as_deref(),
env_var_non_empty("DISPLAY"),
env::var("WAYLAND_DISPLAY").ok().as_deref(),
)
}
fn is_cosmic_wayland_session(&self) -> bool {
self.is_wayland_session()
&& (env_contains("XDG_CURRENT_DESKTOP", "cosmic")
|| env_contains("XDG_SESSION_DESKTOP", "cosmic")
|| env_contains("DESKTOP_SESSION", "cosmic"))
}
fn is_gnome_wayland_session(&self) -> bool {
self.is_wayland_session()
&& (env_contains("XDG_CURRENT_DESKTOP", "gnome")
|| env_contains("DESKTOP_SESSION", "gnome")
|| env_contains("XDG_SESSION_DESKTOP", "gnome"))
}
fn is_kde_wayland_session(&self) -> bool {
self.is_wayland_session()
&& (env_contains("XDG_CURRENT_DESKTOP", "kde")
|| env_contains("DESKTOP_SESSION", "plasma"))
}
fn cached_portal_pointer_session(&self) -> Option<PortalPointerSession> {
if env_flag_enabled("COMPUTER_USE_LINUX_FORCE_YDOTOOL_POINTER") {
return None;
}
let mut cached = self.portal_pointer_session.lock().ok()?;
if cached.as_ref().is_some_and(|session| !session.is_valid()) {
*cached = None;
}
cached.clone()
}
fn clear_portal_pointer_session(&self, failed: &PortalPointerSession) {
failed.invalidate_and_close();
if let Ok(mut cached) = self.portal_pointer_session.lock() {
if cached
.as_ref()
.is_some_and(|session| session.same_session(failed))
{
*cached = None;
}
}
}
fn cached_portal_keyboard_session(&self) -> Option<PortalKeyboardSession> {
let mut cached = self.portal_keyboard_session.lock().ok()?;
if cached.as_ref().is_some_and(|session| !session.is_valid()) {
*cached = None;
}
cached.clone()
}
fn clear_portal_keyboard_session(&self, failed: &PortalKeyboardSession) {
failed.invalidate_and_close();
if let Ok(mut cached) = self.portal_keyboard_session.lock() {
if cached
.as_ref()
.is_some_and(|session| session.same_session(failed))
{
*cached = None;
}
}
}
async fn ensure_portal_pointer_session(&self) -> Result<Option<PortalPointerSession>> {
if !self.should_prefer_portal_pointer_backend().await {
return Ok(None);
}
if let Some(session) = self.cached_portal_pointer_session() {
return Ok(Some(session));
}
let _guard = self.portal_session_init_lock.lock().await;
if let Some(session) = self.cached_portal_pointer_session() {
return Ok(Some(session));
}
let session = start_portal_pointer_session().await?;
self.capture_space_rect().await;
if let Ok(mut cached) = self.portal_pointer_session.lock() {
*cached = Some(session.clone());
}
Ok(Some(session))
}
async fn portal_keyboard_target(
&self,
target: &WindowTarget,
focus: Option<&WindowFocusResult>,
) -> Result<WindowTarget> {
// Preserve the receiving target across consent. Pin the selected
// window when its backend supports exact focus; otherwise keep app focus.
if let Some(focus) = focus {
return Ok(portal_focus_target(target, &focus.requested_window));
}
if target.has_target() {
anyhow::bail!(
"Portal keyboard input requires verified target-window focus. No input was sent."
);
}
let window = focused_window().await?
.context("Portal keyboard input requires a focused window before consent; pass a window target. No input was sent.")?;
let target = portal_focus_target(target, &window);
if !target.has_target() {
anyhow::bail!(
"Portal keyboard input could not preserve the receiving app. No input was sent."
);
}
Ok(target)
}
async fn ensure_portal_keyboard_session(&self) -> Result<Option<PortalKeyboardSession>> {
if env_flag_enabled("COMPUTER_USE_LINUX_FORCE_YDOTOOL_KEYBOARD")
|| !self.is_wayland_session()
{
return Ok(None);
}
if let Some(session) = self.cached_portal_keyboard_session() {
return Ok(Some(session));
}
let _guard = self.portal_session_init_lock.lock().await;
if let Some(session) = self.cached_portal_keyboard_session() {
return Ok(Some(session));
}
let session = start_portal_keyboard_session().await?;
if let Ok(mut cached) = self.portal_keyboard_session.lock() {
*cached = Some(session.clone());
}
Ok(Some(session))
}
async fn resolve_window_context(
&self,
params: &GetAppStateParams,
) -> (
Option<WindowInfo>,
Option<String>,
Option<String>,
bool,
Vec<&'static str>,
) {
let target = params.window_target();
if !target.has_target() {
return (None, None, None, false, Vec::new());
}
match registry::list_windows_with_failures().await {
Ok(listing) => {
let (window, error) = match resolve_window_target(&listing.windows, &target) {
Ok(window) => (Some(window.clone()), None),
Err(error) => (None, Some(format!("{error:#}"))),
};
(window, error, None, true, listing.failed_backends)
}
Err(error) => {
let error = format!("{error:#}");
let hint = window_permission_hint(&error);
(None, Some(error), hint, false, Vec::new())
}
}
}
async fn resolve_screenshot_window(
&self,
target: &WindowTarget,
raise_window: bool,
) -> Result<WindowInfo> {
let window = if raise_window {
let focus = focus_window_target(target).await?;
if !focus.exact_window_focused {
anyhow::bail!(
"the requested window could not be focused exactly; refusing to capture unrelated desktop pixels"
);
}
sleep(Duration::from_millis(250)).await;
focus
.focused_window
.filter(|window| window.window_id == focus.requested_window.window_id)
.ok_or_else(|| anyhow::anyhow!("focused-window verification returned no window"))?
} else {
let windows = list_windows().await?;
let window = resolve_window_target(&windows, target)?.clone();
if !window.focused {
anyhow::bail!(
"raise_window=false requires the requested window to already be focused"
);
}
window
};
if window.hidden {
anyhow::bail!("the requested window is hidden or minimized");
}
Ok(window)
}
async fn resolve_post_action_window(
&self,
target: Option<&WindowTarget>,
fallback_window: Option<&WindowInfo>,
) -> Result<WindowInfo> {
let window = if let Some(target) = target.filter(|target| target.has_target()) {
match list_windows().await {
Ok(windows) => resolve_window_target(&windows, target)?.clone(),
Err(error) => {
self.diagnostics.invalidate();
if let Some(window) = fallback_window {
window.clone()
} else {
return Err(error);
}
}
}
} else if let Some(window) = fallback_window {
window.clone()
} else {
focused_window().await?.ok_or_else(|| {
anyhow::anyhow!("no focused window is available for screenshot_after=\"window\"")
})?
};
if window.hidden {
anyhow::bail!("the target window is hidden or minimized");
}
Ok(window)
}
/// Saves a prepared screenshot to `path` for screenshot_file_path and
/// returns the path. The caller then returns no image content for it.
async fn save_screenshot_file(
&self,
capture: &ScreenshotCapture,
path: &str,
) -> Result<String> {
let target = crate::screenshot_file::validate_screenshot_file_path(
path,
capture.format,
self.http_transport,
)?;
let bytes = {
use base64::Engine as _;
base64::engine::general_purpose::STANDARD
.decode(data_url_payload(&capture.data_url))
.context("failed to decode the screenshot for screenshot_file_path")?
};
tokio::task::spawn_blocking(move || {
crate::screenshot_file::write_screenshot_file(&target, &bytes)
})
.await
.context("screenshot file writer stopped")??;
Ok(path.to_string())
}
async fn capture_post_action_observation(
&self,
capture: &PostActionCaptureParams,
target: Option<&WindowTarget>,
fallback_window: Option<&WindowInfo>,
raised: TargetRaised,
scope: PostActionScope,
) -> (
PostActionObservation,
Option<ScreenshotCapture>,
Vec<String>,
) {
self.capture_post_action_observation_with(
capture,
target,
fallback_window,
raised,
scope,
false,
)
.await
}
/// `settled` skips the post-action settle delay for a tree read right
/// after a screenshot of the same action, which already waited it.
async fn capture_post_action_observation_with(
&self,
capture: &PostActionCaptureParams,
target: Option<&WindowTarget>,
fallback_window: Option<&WindowInfo>,
raised: TargetRaised,
scope: PostActionScope,
settled: bool,
) -> (
PostActionObservation,
Option<ScreenshotCapture>,
Vec<String>,
) {
let screenshot_scope = capture.screenshot_scope();
let include_tree = capture.include_tree.unwrap_or(false);
if screenshot_scope.is_none() && !include_tree {
return (PostActionObservation::default(), None, Vec::new());
}
let bound = scope.bound;
let mut observation = PostActionObservation::default();
let mut screenshot_capture = None;
let mut notes = Vec::new();
let mut resolved_window: Option<WindowInfo> = None;
// A failed file output is reported like a failed capture, but it says
// nothing about the desktop, so it leaves the diagnostics cached.
let mut screenshot_capture_failed = false;
if let Some(scope) = screenshot_scope {
let options = capture.screenshot_options();
let mut file_output_failed = false;
#[allow(clippy::type_complexity)]
let result: Result<(
ScreenshotCapture,
bool,
Option<String>,
Option<String>,
Option<String>,
)> = async {
// A path that cannot be written is reported without a capture.
if let Some(path) = capture.screenshot_file_path.as_deref() {
if let Err(error) = crate::screenshot_file::validate_screenshot_file_path(
path,
options.format.unwrap_or(ScreenshotOutputFormat::Png),
self.http_transport,
) {
file_output_failed = true;
return Err(error);
}
}
let _pipeline = crate::screenshot_impl::screenshot_pipeline_permit().await?;
let crop_window = match scope {
ScreenshotAfterScope::Screen => None,
ScreenshotAfterScope::Window => {
let window = self
.resolve_post_action_window(target, fallback_window)
.await
.context("targeted post-action screenshot failed")?;
ensure_post_action_crop_shows_target(raised, target, &window)?;
resolved_window = Some(window.clone());
Some(window)
}
};
let window_label = crop_window.as_ref().and_then(|window| window.title.clone());
let raw_capture = {
let hold_started = std::time::Instant::now();
let _hold = self.indicator.hold_for_capture().await?;
if !cfg!(test) {
let remaining =
Duration::from_millis(120).saturating_sub(hold_started.elapsed());
if !remaining.is_zero() {
sleep(remaining).await;
}
}
capture_screenshot_raw()
.await
.context("post-action screenshot failed")?
};
self.cache_desktop_size(raw_capture.width, raw_capture.height);
let off_screen_note = match crop_window
.as_ref()
.and_then(|window| window.bounds.as_ref())
{
Some(bounds) => self.off_screen_note_for_bounds(bounds).await,
None => None,
};
let (raw_for_payload, cropped) = match crop_window.as_ref() {
Some(window) => {
let (x, y, width, height) = self
.window_crop_rect_for_capture(window, &raw_capture)
.await
.context("targeted post-action screenshot crop failed")?;
let (raw_capture, bytes, width, height) =
crate::screenshot_impl::run_image_task(move || {
let (bytes, width, height) =
crop_png(&raw_capture.bytes, x, y, width, height)
.map_err(anyhow::Error::msg)
.context("targeted post-action screenshot crop failed")?;
Ok((raw_capture, bytes, width, height))
})
.await?;
(
RawScreenshotCapture {
mime_type: raw_capture.mime_type,
bytes,
source: raw_capture.source,
width,
height,
},
true,
)
}
None => (raw_capture, false),
};
let prepared = crate::screenshot_impl::run_image_task(move || {
prepare_screenshot_payload(raw_for_payload, options)
})
.await
.context("post-action screenshot resize failed")?;
let file_path = match capture.screenshot_file_path.as_deref() {
Some(path) => match self.save_screenshot_file(&prepared, path).await {
Ok(saved) => Some(saved),
Err(error) => {
file_output_failed = true;
return Err(error);
}
},
None => None,
};
Ok((prepared, cropped, window_label, off_screen_note, file_path))
}
.await;
match result {
Ok((prepared, cropped, window_title, off_screen_note, file_path)) => {
let mut summary = ScreenshotSummary::from(&prepared);
summary.cropped_to_window = Some(cropped);
summary.window_title = window_title;
// A saved image is not returned inline.
if file_path.is_none() {
screenshot_capture = Some(prepared);
}
summary.file_path = file_path;
observation.screenshot = Some(summary);
if let Some(note) = off_screen_note {
notes.push(note);
}
}
Err(error) => {
screenshot_capture_failed = !file_output_failed;
let error = format!("{error:#}");
notes.push(format!("Post-action screenshot failed: {error}"));
observation.screenshot_error = Some(error);
}
}
} else if !settled && !cfg!(test) {
sleep(Duration::from_millis(120)).await;
}
if include_tree {
if resolved_window.is_none() {
resolved_window = if let Some(target) = target.filter(|target| target.has_target())
{
list_windows()
.await
.inspect_err(|_| self.diagnostics.invalidate())
.ok()
.and_then(|windows| {
resolve_window_target(&windows, target)
.inspect_err(|_| {
// A bound app can legitimately have zero or several windows.
if !bound {
self.diagnostics.invalidate();
}
})
.ok()
.cloned()
})
.or_else(|| fallback_window.cloned())
} else if let Some(window) = fallback_window {
Some(window.clone())
} else {
focused_window()
.await
.inspect_err(|_| self.diagnostics.invalidate())
.ok()
.flatten()
};
}
// A tree for a named window must not borrow the previous
// snapshot's app: on i3 without xprop and on COSMIC the window has no pid, and
// that app may not be the one holding it.
let window_targeted = resolved_window.is_some()
&& (target.is_some_and(WindowTarget::has_target) || fallback_window.is_some());
let PostActionScope {
pid: cached_pid,
tree: cached_tree,
..
} = self.post_action_scope_for_capture(scope);
// A bound owner's pid is the element's verified bus owner. Its
// window's listed pid can differ: a Flatpak app's roots belong
// to xdg-dbus-proxy, not to the process the compositor lists.
let target_pid = if bound {
target
.and_then(|target| target.pid)
.or_else(|| resolved_window.as_ref().and_then(|window| window.pid))
} else {
resolved_window
.as_ref()
.and_then(|window| window.pid)
.or_else(|| target.and_then(|target| target.pid))
.or(cached_pid.filter(|_| !window_targeted))
};
// A pid scopes the tree to every root it owns. A window the
// compositor listed is alive, so when its pid owns no root (a
// Flatpak app registers through xdg-dbus-proxy) its title finds
// the root instead. A bound element owner gets no such fallback.
let window_pid = resolved_window.as_ref().and_then(|window| window.pid);
let candidates = accessibility_filter_candidates(resolved_window.as_ref());
let fallback_name = candidates
.first()
.filter(|_| !bound && window_pid.is_some() && window_pid == target_pid)
.cloned();
let app_filter = if target_pid.is_none() {
candidates.into_iter().next()
} else {
None
};
let accessibility_target_requested = target.is_some_and(WindowTarget::has_target)
|| target_pid.is_some()
|| app_filter.is_some();
let (max_nodes, max_depth) = crate::atspi_tree::snapshot_limits(None, None);
let window_title = post_action_window_title(
target,
resolved_window.as_ref(),
target_pid,
cached_tree
.as_ref()
.map(|cached| (cached.scope.clone(), cached.pid)),
);
// Without a window_id or title of its own, a narrowed tree was
// narrowed through the previous snapshot's window scope.
let narrowed_through_cache = cached_tree.filter(|_| {
window_title.is_some()
&& !target
.is_some_and(|target| target.window_id.is_some() || target.title.is_some())
});
match snapshot_accessibility_tree(
app_filter.as_deref(),
target_pid,
window_title.as_deref(),
fallback_name.as_deref(),
max_nodes,
max_depth,
)
.await
{
Ok(snapshot)
if snapshot.nodes.is_empty()
&& !tokio::task::spawn_blocking(
crate::diagnostics_impl::can_build_accessibility_tree_now,
)
.await
.unwrap_or(false) =>
{
self.commit_cleared_snapshot().await;
let error = "GNOME accessibility is disabled; call setup_accessibility first."
.to_string();
notes.push(format!("Post-action accessibility tree failed: {error}"));
observation.accessibility_tree = Some(TreeNodes::default());
observation.accessibility_tree_raw_count = Some(0);
observation.tree_scoped = Some(false);
observation.accessibility_tree_truncated = Some(false);
observation.accessibility_error = Some(error);
}
Ok(snapshot) => {
let raw_count = snapshot.nodes.len();
let (nodes, shell_only_note) = compact_accessibility_tree_with_note(
snapshot.nodes,
snapshot.scoped,
snapshot.truncated,
);
let tree_window = post_action_tree_window(
resolved_window.as_ref().filter(|_| window_targeted),
snapshot.root_pid,
snapshot.window_scope.as_deref(),
narrowed_through_cache.as_ref(),
);
observation.snapshot_id = Some(
self.commit_observed_snapshot(
&nodes,
snapshot.root_pid,
tree_window,
snapshot.window_scope.clone(),
)
.await,
);
if let Some(warning) = unscoped_accessibility_tree_warning(
snapshot.scoped,
true,
accessibility_target_requested,
) {
notes.push(warning.to_string());
}
if let Some(note) = unmatched_accessibility_pid_note(
target_pid,
snapshot.root_pid,
snapshot.app_filter_matched,
) {
notes.push(note);
}
if let Some(note) = truncated_accessibility_tree_note(snapshot.truncated) {
notes.push(note.to_string());
}
if let Some(note) = shell_only_note {
notes.push(note.to_string());
}
let mut nodes = TreeNodes::from(nodes);
let (spill, spill_note) = fit_tree(
&self.tree_spiller,
&mut nodes,
tree_budget(POST_ACTION_RESULT_RESERVE),
);
notes.extend(spill_note);
observation.accessibility_tree_spill = spill;
observation.accessibility_tree = Some(nodes);
observation.accessibility_tree_raw_count = Some(raw_count);
observation.tree_scoped = Some(snapshot.scoped);
observation.accessibility_tree_truncated = Some(snapshot.truncated);
}
Err(error) => {
self.commit_cleared_snapshot().await;
let error = format!("{error:#}");
notes.push(format!("Post-action accessibility tree failed: {error}"));
observation.accessibility_tree = Some(TreeNodes::default());
observation.accessibility_tree_raw_count = Some(0);
observation.tree_scoped = Some(false);
observation.accessibility_tree_truncated = Some(false);
observation.accessibility_error = Some(error);
}
}
}
if screenshot_capture_failed
|| observation.accessibility_error.is_some()
|| observation.accessibility_tree_raw_count == Some(0)
{
self.diagnostics.invalidate();
}
(observation, screenshot_capture, notes)
}
async fn capture_element_observation(
&self,
capture: &PostActionCaptureParams,
screenshot_target: Option<&WindowTarget>,
tree_target: std::result::Result<&ElementObservationTarget, &String>,
) -> (
PostActionObservation,
Option<ScreenshotCapture>,
Vec<String>,
) {
// Screenshots keep the caller's target (or focused-window fallback).
// An element's verified owner scopes only the accessibility tree.
let screenshot_params = PostActionCaptureParams {
include_tree: Some(false),
..capture.clone()
};
let (screenshot_observation, screenshot, mut notes) = self
.capture_post_action_observation(
&screenshot_params,
screenshot_target,
None,
TargetRaised::No,
PostActionScope::default(),
)
.await;
if !capture.include_tree.unwrap_or(false) {
return (screenshot_observation, screenshot, notes);
}
let (mut observation, _, tree_notes) = match tree_target {
Ok(target) => {
// The dispatched element's scope was bound before the action.
// A later snapshot must not narrow its process to another window.
let tree_params = PostActionCaptureParams {
include_tree: Some(true),
..Default::default()
};
// A screenshot of this action already waited the settle delay.
self.capture_post_action_observation_with(
&tree_params,
Some(&target.owner),
None,
TargetRaised::No,
PostActionScope::bound_to(target),
capture.screenshot_scope().is_some(),
)
.await
}
Err(error) => {
// Like every failed post-action read: empty tree fields, and
// the replaced snapshot is not left for index calls.
self.commit_cleared_snapshot().await;
let message = format!("Post-action element observation failed: {error}");
let observation = PostActionObservation {
accessibility_tree: Some(TreeNodes::default()),
accessibility_tree_raw_count: Some(0),
tree_scoped: Some(false),
accessibility_tree_truncated: Some(false),
accessibility_error: Some(message.clone()),
..Default::default()
};
self.diagnostics.invalidate();
(observation, None, vec![message])
}
};
observation.screenshot = screenshot_observation.screenshot;
observation.screenshot_error = screenshot_observation.screenshot_error;
notes.extend(tree_notes);
(observation, screenshot, notes)
}
/// `structured_tool_result` for an action. A failed action can mean the
/// desktop changed under the cached diagnostics, such as a dropped portal
/// session or a window backend that stopped answering, so any failure marks
/// them stale without classifying its reason.
fn action_tool_result<T: Serialize>(
&self,
output: T,
screenshot: Option<ScreenshotCapture>,
) -> Result<CallToolResult, ErrorData> {
let result = structured_tool_result(output, screenshot)?;
if result.is_error == Some(true) {
self.diagnostics.invalidate();
}
Ok(result)
}
async fn finish_action_output(
&self,
mut output: ActionOutput,
capture: &PostActionCaptureParams,
target: Option<&WindowTarget>,
raised: TargetRaised,
scope: PostActionScope,
) -> Result<CallToolResult, ErrorData> {
if !output.ok {
return self.action_tool_result(
ActionToolOutput {
action: output,
observation: PostActionObservation::default(),
},
None,
);
}
let (observation, screenshot, notes) = self
.capture_post_action_observation(capture, target, None, raised, scope)
.await;
output = with_notes(output, notes);
self.action_tool_result(
ActionToolOutput {
action: output,
observation,
},
screenshot,
)
}
async fn finish_set_value_output(
&self,
mut output: ActionOutput,
verified: Option<bool>,
capture: &PostActionCaptureParams,
screenshot_target: Option<&WindowTarget>,
target: std::result::Result<&ElementObservationTarget, &String>,
) -> Result<CallToolResult, ErrorData> {
if !output.ok && verified != Some(false) {
return self.action_tool_result(
SetValueToolOutput {
action: output,
verified,
observation: PostActionObservation::default(),
},
None,
);
}
let (observation, screenshot, notes) = self
.capture_element_observation(capture, screenshot_target, target)
.await;
output = with_notes(output, notes);
self.action_tool_result(
SetValueToolOutput {
action: output,
verified,
observation,
},
screenshot,
)
}
async fn finish_activate_window_output(
&self,
output: ActivateWindowOutput,
capture: &PostActionCaptureParams,
target: &WindowTarget,
) -> Result<CallToolResult, ErrorData> {
if !output.ok {
return self.action_tool_result(
ActivateWindowToolOutput {
output,
message: None,
observation: PostActionObservation::default(),
},
None,
);
}
let fallback_window = output
.focus
.as_ref()
.and_then(|focus| focus.focused_window.as_ref())
.or_else(|| output.focus.as_ref().map(|focus| &focus.requested_window));
let (observation, screenshot, notes) = self
.capture_post_action_observation(
capture,
Some(target),
fallback_window,
TargetRaised::Yes,
PostActionScope::default(),
)
.await;
let message = (!notes.is_empty()).then(|| notes.join(" "));
self.action_tool_result(
ActivateWindowToolOutput {
output,
message,
observation,
},
screenshot,
)
}
async fn finish_window_geometry_output(
&self,
mut output: WindowGeometryOutput,
capture: &PostActionCaptureParams,
target: &WindowTarget,
) -> Result<CallToolResult, ErrorData> {
if !output.ok {
return self.action_tool_result(
WindowGeometryToolOutput {
output,
observation: PostActionObservation::default(),
},
None,
);
}
let fallback_window = output.window.as_ref();
let (observation, screenshot, notes) = self
.capture_post_action_observation(
capture,
Some(target),
fallback_window,
TargetRaised::Yes,
PostActionScope::default(),
)
.await;
for note in notes {
if !output.message.is_empty() {
output.message.push(' ');
}
output.message.push_str(¬e);
}
self.action_tool_result(
WindowGeometryToolOutput {
output,
observation,
},
screenshot,
)
}
async fn window_crop_rect_for_capture(
&self,
window: &WindowInfo,
raw: &RawScreenshotCapture,
) -> Result<(i32, i32, u32, u32)> {
self.window_crop_rect_for_dimensions(window, raw.width, raw.height)
.await
}
async fn window_crop_rect_for_dimensions(
&self,
window: &WindowInfo,
capture_width: u32,
capture_height: u32,
) -> Result<(i32, i32, u32, u32)> {
Ok(self
.window_coordinate_map_for_dimensions(window, capture_width, capture_height)
.await?
.capture_rect)
}
async fn window_coordinate_map_for_dimensions(
&self,
window: &WindowInfo,
capture_width: u32,
capture_height: u32,
) -> Result<WindowCoordinateMap> {
let bounds = window.bounds.as_ref().ok_or_else(|| {
anyhow::anyhow!(
"targeted screenshot requires window bounds; refusing to return the full desktop"
)
})?;
let logical_rect = window_crop_rect(bounds).ok_or_else(|| {
anyhow::anyhow!(
"targeted screenshot has unusable window bounds; refusing to return the full desktop"
)
})?;
let monitors = if window.backend == GNOME_SHELL_EXTENSION_BACKEND {
Some(
crate::windowing::backends::gnome::extension_monitor_layout()
.await
.map_err(|error| {
anyhow::anyhow!(
"GNOME targeted screenshot requires logical monitor geometry: {error:#}"
)
})?
.into_iter()
.map(|monitor| (monitor.x, monitor.y, monitor.width, monitor.height))
.collect(),
)
} else if window.backend == GNOME_SHELL_INTROSPECT_BACKEND {
crate::windowing::backends::gnome::extension_monitor_layout()
.await
.ok()
.map(|monitors| {
monitors
.into_iter()
.map(|monitor| (monitor.x, monitor.y, monitor.width, monitor.height))
.collect()
})
} else if window.backend == crate::windowing::backends::cosmic::COSMIC_WAYLAND_BACKEND {
Some(
crate::cosmic_helper::monitor_layout()
.await?
.into_iter()
.map(|monitor| (monitor.x, monitor.y, monitor.width, monitor.height))
.collect(),
)
} else if window.backend == KWIN_BACKEND {
Some(vec![
crate::windowing::backends::kwin::logical_desktop_rect()
.await
.map_err(|error| {
anyhow::anyhow!(
"KWin targeted screenshot requires logical workspace geometry: {error:#}"
)
})?,
])
} else {
None
};
let (full_capture_rect, portal_rect) = match monitors {
Some(monitors) => (
logical_window_crop_rect(bounds, &monitors, capture_width, capture_height)?,
Some(logical_rect),
),
None => (logical_rect, None),
};
Ok(WindowCoordinateMap {
capture_rect: clip_capture_rect(full_capture_rect, capture_width, capture_height)?,
full_capture_rect,
portal_rect,
})
}
async fn focused_window_coordinate_map(
&self,
focus: &WindowFocusResult,
) -> std::result::Result<WindowCoordinateMap, String> {
let window = focus
.focused_window
.as_ref()
.unwrap_or(&focus.requested_window);
if !matches!(
window.backend.as_str(),
GNOME_SHELL_EXTENSION_BACKEND
| GNOME_SHELL_INTROSPECT_BACKEND
| KWIN_BACKEND
| crate::windowing::backends::cosmic::COSMIC_WAYLAND_BACKEND
) {
let full_capture_rect = window
.bounds
.as_ref()
.and_then(window_crop_rect)
.ok_or_else(|| {
"Window-relative coordinates require usable target-window bounds.".to_string()
})?;
let capture_rect = self
.desktop_size
.lock()
.ok()
.and_then(|guard| *guard)
.map(|(width, height)| {
clip_capture_rect(full_capture_rect, width, height).map_err(|error| {
format!("Could not map target-window coordinates: {error:#}")
})
})
.transpose()?
.unwrap_or(full_capture_rect);
return Ok(WindowCoordinateMap {
capture_rect,
full_capture_rect,
portal_rect: None,
});
}
let (_, _, width, height) = self.capture_space_rect().await.ok_or_else(|| {
"Could not determine screenshot dimensions for window-relative coordinates.".to_string()
})?;
self.window_coordinate_map_for_dimensions(window, width as u32, height as u32)
.await
.map_err(|error| format!("Could not map target-window coordinates: {error:#}"))
}
async fn resolve_accessibility_app_filter(
&self,
params: &GetAppStateParams,
window_context: Option<&WindowInfo>,
window_enumerated: bool,
) -> Option<String> {
self.resolve_accessibility_app_filter_with(
params,
window_context,
window_enumerated,
|| list_accessible_apps(200),
)
.await
}
async fn resolve_accessibility_app_filter_with<F, Fut>(
&self,
params: &GetAppStateParams,
window_context: Option<&WindowInfo>,
window_enumerated: bool,
list_apps: F,
) -> Option<String>
where
F: FnOnce() -> Fut,
Fut: Future<Output = Result<Vec<AccessibleAppSummary>>>,
{
if let Some(app_id) = trimmed_nonempty(params.app_id.as_deref()) {
// A failed lookup against a successfully enumerated desktop is
// authoritative. Only an unavailable backend permits an AT-SPI-only
// fallback. Higher-priority window selectors must not contradict
// the explicit app identity either.
let rejected = window_enumerated && window_context.is_none();
let contradictory = window_context.is_some_and(|window| {
!window.app_id.as_deref().is_some_and(|actual| {
actual.eq_ignore_ascii_case(app_id)
|| (!app_id.contains('.')
&& crate::windows::app_id_leaf(actual).eq_ignore_ascii_case(app_id))
})
});
if rejected || contradictory {
return Some(UNMATCHABLE_ACCESSIBILITY_APP_FILTER.to_string());
}
}
if let Some(explicit) = trimmed_nonempty(params.app_name_or_bundle_identifier.as_deref()) {
return Some(explicit.to_string());
}
if params.window_id.is_some() && window_context.is_none() {
return Some(UNMATCHABLE_ACCESSIBILITY_APP_FILTER.to_string());
}
if window_context
.and_then(|window| window.pid)
.or(params.pid.filter(|_| params.window_id.is_none()))
.is_some()
{
// Let the PID-scoped traversal discover every toolkit root.
return None;
}
let candidates = accessibility_filter_candidates(window_context);
let app_id = trimmed_nonempty(params.app_id.as_deref()).map(|requested| {
window_context
.and_then(|window| trimmed_nonempty(window.app_id.as_deref()))
.unwrap_or(requested)
});
let needs_app_id_evidence = app_id.is_some_and(|app_id| {
!app_id.starts_with("window:") && !crate::windows::app_id_leaf(app_id).is_empty()
});
let apps = if needs_app_id_evidence {
list_apps().await.ok()
} else {
None
};
if let Some(app_id) = app_id {
if !app_id.starts_with("window:") {
return Some(select_accessibility_app_id_filter(apps.as_deref(), app_id));
}
}
if candidates.is_empty()
&& (params.window_id.is_some() || trimmed_nonempty(params.app_id.as_deref()).is_some())
{
return Some(UNMATCHABLE_ACCESSIBILITY_APP_FILTER.to_string());
}
candidates.into_iter().next()
}
/// Hyprland background capture: when opted in with
/// `COMPUTER_USE_LINUX_HYPRLAND_BACKGROUND=1` and the target is an
/// unfocused Hyprland window, capture just that window with `grim -T`
/// instead of raising it or grabbing the desktop through the portal.
/// Returns `None` to fall through to the regular capture path. The note
/// explains the repeated capture used for windows on hidden workspaces.
async fn hyprland_background_capture(
&self,
window: &WindowInfo,
) -> Option<Result<(RawScreenshotCapture, Option<String>)>> {
if !crate::hyprland_input::background_enabled()
|| window.backend != registry::HYPRLAND_BACKEND
|| window.focused
{
return None;
}
Some(
async {
let capture =
crate::windowing::backends::hyprland::capture_window(window).await?;
let note = capture.hidden_workspace.as_ref().map(|hidden| {
let state = if hidden.settled {
"the last two frames matched, so the image is current"
} else {
"frames were still changing (animation, caret or loading), so details may lag"
};
format!(
"Window_id {} is on hidden workspace {}, which Hyprland does not repaint; it was captured {} times so the window drew current frames, and {state}. The accessibility tree stays authoritative.",
window.window_id, hidden.workspace, hidden.captures
)
});
let source = match &capture.hidden_workspace {
Some(hidden) if hidden.settled => {
"hyprland-grim-toplevel (hidden workspace, settled)"
}
Some(_) => "hyprland-grim-toplevel (hidden workspace, still changing)",
None => "hyprland-grim-toplevel",
};
Ok((
RawScreenshotCapture {
mime_type: "image/png".to_string(),
bytes: capture.png,
source: source.to_string(),
width: capture.width,
height: capture.height,
},
note,
))
}
.await,
)
}
/// Hyprland background input: when opted in with
/// `COMPUTER_USE_LINUX_HYPRLAND_BACKGROUND=1` and the call names a
/// target window, keys go to that window through `send_key_state` without
/// focusing it, so the user's focus, cursor and workspace stay put.
/// Returns `None` to fall through to the focus-then-type path.
async fn hyprland_background_input(
&self,
action: &str,
target: &WindowTarget,
shown: &str,
presses: std::result::Result<Vec<crate::hyprland_input::KeyPress>, String>,
received: Option<serde_json::Value>,
) -> Option<ActionOutput> {
if !hyprland_background_applies(target) {
return None;
}
let fail = |message: String| Some(action_result(action, Err(message), received.clone()));
let windows = match list_windows().await {
Ok(windows) => windows,
Err(error) => return fail(format!("Did not send input: {error:#}")),
};
let window = match resolve_window_target(&windows, target) {
Ok(window) => window.clone(),
Err(error) => return fail(format!("Did not send input: {error:#}")),
};
if window.backend != registry::HYPRLAND_BACKEND {
return None;
}
// Background mode never moves focus, so input it cannot map for the
// target is refused rather than typed through the focusing path.
let presses = match presses {
Ok(presses) => presses,
Err(message) => return fail(message),
};
match crate::windowing::backends::hyprland::devices_json().await {
Ok(json) if crate::hyprland_input::keymap_is_us(&json) => {}
Ok(_) => {
return fail(
"Did not send input: Hyprland background input maps keys for a US layout, and the active keymap is not US.".to_string(),
)
}
Err(error) => return fail(format!("Did not send input: {error:#}")),
}
// The target is not focused, so probe its own app. Anything that is
// not a confirmed non-secret field is masked on the indicator.
// Without a pid the probe would read the user's focused app instead.
let probe = match window.pid {
Some(pid) if self.indicator.enabled() => {
timeout(Duration::from_millis(250), probe_focused_element(Some(pid)))
.await
.ok()
.and_then(|probe| probe.ok())
}
_ => None,
};
let secret = probe.as_ref().is_none_or(focus_requires_secret_protection);
if action == "press_key" {
self.indicator.keys(shown, secret).await;
} else {
self.indicator.text(action, shown, secret).await;
}
let result =
crate::windowing::backends::hyprland::send_key_events(window.window_id, &presses).await;
Some(match result {
Ok(()) => successful_action_with_focus(
action,
&format!(
"Action sent through Hyprland send_key_state to window_id {} ({} key presses) without focusing it; focus, cursor and workspace were not changed.",
window.window_id,
presses.len()
),
received,
None,
),
Err(error) => action_result(action, Err(format!("{error:#}")), received),
})
}
async fn focus_target_for_input(
&self,
target: &WindowTarget,
) -> std::result::Result<Option<WindowFocusResult>, String> {
if !target.has_target() {
return Ok(None);
}
let focus = focus_window_target(target).await.map_err(|error| {
let error = format!("{error:#}");
if let Some(hint) = window_permission_hint(&error) {
format!("Did not send input because the target window could not be focused: {error}. {hint}")
} else {
format!("Did not send input because the target window could not be focused: {error}")
}
})?;
if focus_satisfies_target(&focus, target) {
Ok(Some(focus))
} else {
let required = if target.requires_exact_focus() {
"exact target-window focus"
} else {
"app-level focus"
};
Err(format!(
"Did not send input because {required} verification failed after activating the target window. Focus result: requested window_id {}, focused window_id {:?}.",
focus.requested_window.window_id,
focus.focused_window.as_ref().map(|window| window.window_id)
))
}
}
fn cache_desktop_size(&self, width: u32, height: u32) {
if width == 0 || height == 0 {
return;
}
if let Ok(mut guard) = self.desktop_size.lock() {
*guard = Some((width, height));
}
if let Ok(mut layout) = self.capture_layout.lock() {
layout.generation = layout.generation.wrapping_add(1);
layout.monitors = CaptureMonitors::Unread;
}
}
async fn pixelflux_element_point(
&self,
bounds: Option<&crate::atspi_tree::Bounds>,
) -> Option<(i32, i32)> {
let bounds = bounds?;
let capture = self.desktop_size.lock().ok().and_then(|guard| *guard)?;
// kscreen-doctor misreports nested KWin scale (2864x1536 at scale 1); KWin does not.
let layout = match crate::remote_desktop::kwin_desktop_layout().await {
Some(layout) => layout,
None => crate::remote_desktop::logical_desktop_layout().await?,
};
pixelflux_point_for_element(
(bounds.x, bounds.y, bounds.width, bounds.height),
capture,
&layout,
)
}
#[allow(clippy::too_many_arguments)]
async fn try_pixelflux_click(
&self,
x: i32,
y: i32,
element_click: bool,
element_bounds: Option<&crate::atspi_tree::Bounds>,
button: Option<&str>,
count: u32,
received: Option<serde_json::Value>,
input_guard: Arc<tokio::sync::OwnedMutexGuard<()>>,
) -> Option<ActionOutput> {
if crate::pixelflux::PixelfluxConfig::from_env().is_none() || count.clamp(1, 10) > 3 {
return None;
}
let button = pixelflux_pointer_button(button)?;
if count.clamp(1, 10) > 1 && !matches!(button, PointerButton::Left) {
return None;
}
let point = if element_click {
self.pixelflux_element_point(element_bounds).await
} else {
Some((x, y))
};
let Some((px, py)) = point else {
return self.pixelflux_unmapped("click", received).await;
};
self.pixelflux_send(
"click",
crate::pixelflux::PixelfluxAction::Click {
x: px,
y: py,
button,
count: count.clamp(1, 3) as u8,
},
received,
input_guard,
)
.await
}
/// An element pixelflux cannot map. Falls through only when pixelflux is absent or down.
async fn pixelflux_unmapped(
&self,
action: &str,
received: Option<serde_json::Value>,
) -> Option<ActionOutput> {
let config = crate::pixelflux::PixelfluxConfig::from_env()?;
match crate::pixelflux::probe(&config).await {
Ok(()) => Some(ActionOutput {
ok: false,
implemented: true,
action: action.to_string(),
message: "pixelflux could not map the element onto the capture".to_string(),
received,
}),
Err(error) if crate::pixelflux::is_unreachable(&error) => None,
Err(error) => Some(pixelflux_action_output(action, Err(error), received)),
}
}
async fn pixelflux_send<G: Send + 'static>(
&self,
action: &str,
request: crate::pixelflux::PixelfluxAction,
received: Option<serde_json::Value>,
input_guard: G,
) -> Option<ActionOutput> {
let config = crate::pixelflux::PixelfluxConfig::from_env()?;
if let Err(error) = crate::pixelflux::probe(&config).await {
if crate::pixelflux::is_unreachable(&error) {
return None;
}
return Some(pixelflux_action_output(action, Err(error), received));
}
let (_retained, dispatched) = run_cancellation_safe_input(input_guard, async move {
Ok::<_, String>(crate::pixelflux::send(&config, &request).await.map(|_| ()))
})
.await;
let result = match dispatched {
Ok(result) => result,
Err(error) => return Some(ActionOutput {
ok: false, implemented: true, action: action.to_string(), received,
message: format!("pixelflux input task stopped ({error}); it may have run, so check the screen before retrying"),
}),
};
if let Err(error) = &result {
if crate::pixelflux::outcome_unknown(error) {
return Some(ActionOutput {
ok: false,
implemented: true,
action: action.to_string(),
message: format!(
"pixelflux took {action} but did not answer ({error:#}); it may have run, so check the screen before retrying"
),
received,
});
}
}
Some(pixelflux_action_output(action, result, received))
}
fn logical_portal_point(
&self,
session: &PortalPointerSession,
x: i32,
y: i32,
) -> Option<(i32, i32)> {
let capture_size = self.desktop_size.lock().ok().and_then(|guard| *guard);
session.logical_point_from_capture(x, y, capture_size)
}
/// Capture size for overlay points, which are in screenshot pixels like
/// the click tools. Only a cached size is used: capturing the screen just
/// for the overlay could prompt the user.
fn indicator_space(&self) -> Option<(u32, u32)> {
*self.desktop_size.lock().ok()?
}
/// Cosmetic secrecy probe: timeout only masks the indicator, never gates input.
async fn typing_into_secret(&self, focus: Option<&WindowFocusResult>) -> bool {
if !self.indicator.enabled() {
return true;
}
timeout(Duration::from_millis(250), self.input_focus_probe(focus))
.await
.ok()
.flatten()
.as_ref()
.is_none_or(focus_requires_secret_protection)
}
/// Probe only the receiving app; the caller chooses its own time budget.
async fn input_focus_probe(&self, focus: Option<&WindowFocusResult>) -> Option<FocusProbe> {
let pid = match focus {
Some(focus) => focus
.focused_window
.as_ref()
.and_then(|window| window.pid)
.or(focus.requested_window.pid),
None => focused_window().await.ok().flatten()?.pid,
}?;
probe_focused_element(Some(pid)).await.ok()
}
/// COORDINATE SPACES: window bounds (list_windows / extension frame rects)
/// and the extension monitor layout are in LOGICAL pixels, while click/
/// scroll coordinates and screenshot captures are in PHYSICAL capture
/// pixels. On fractionally-scaled displays the two differ, so each check
/// below only ever compares values from the same space.
///
/// Logical monitor rectangles from the GNOME Shell extension, for checks
/// against logical window bounds. None when the extension is unavailable.
async fn logical_monitor_rects(&self) -> Option<Vec<(i32, i32, i32, i32)>> {
let monitors = crate::windowing::backends::gnome::extension_monitor_layout()
.await
.ok()?;
(!monitors.is_empty()).then(|| {
monitors
.iter()
.map(|m| (m.x, m.y, m.width, m.height))
.collect()
})
}
/// Physical capture-space desktop rectangle (union of monitors as captured
/// by the screenshot pipeline), for checks against click coordinates.
/// Best-effort; None disables the check.
async fn capture_space_rect(&self) -> Option<(i32, i32, i32, i32)> {
let cached = self.desktop_size.lock().ok().and_then(|guard| *guard);
if let Some((w, h)) = cached {
return Some((0, 0, w as i32, h as i32));
}
// One-time prime: a full-frame capture reveals the desktop size when
// no prior capture is available.
let raw = capture_screenshot_raw().await.ok()?;
self.cache_desktop_size(raw.width, raw.height);
(raw.width > 0 && raw.height > 0).then_some((0, 0, raw.width as i32, raw.height as i32))
}
/// Warn when a targeted window pokes outside every monitor: clicks and
/// screenshots silently truncate to visible pixels there, which reads as
/// "success" while landing nowhere.
async fn off_screen_note_for_bounds(
&self,
bounds: &crate::windowing::WindowBounds,
) -> Option<String> {
let (x, y) = bounds.x.zip(bounds.y)?;
if bounds.width == 0 || bounds.height == 0 {
return None;
}
// Window bounds are logical pixels: prefer the extension's logical
// monitor layout (same space). The physical capture rect is a safe
// fallback: on scaled displays it is at least as large as the logical
// union, so it can only under-warn, never false-positive.
let rects = match self.logical_monitor_rects().await {
Some(rects) => rects,
None => vec![self.capture_space_rect().await?],
};
let (w, h) = (bounds.width as i64, bounds.height as i64);
let window_area = w * h;
let mut visible_area = 0_i64;
for (mx, my, mw, mh) in &rects {
let ix = (x as i64).max(*mx as i64);
let iy = (y as i64).max(*my as i64);
let ix2 = (x as i64 + w).min(*mx as i64 + *mw as i64);
let iy2 = (y as i64 + h).min(*my as i64 + *mh as i64);
if ix2 > ix && iy2 > iy {
// Overlapping monitors are rare; treating them as additive keeps
// this a cheap best-effort heuristic.
visible_area += (ix2 - ix) * (iy2 - iy);
}
}
let visible_pct = (visible_area.min(window_area) * 100) / window_area.max(1);
if visible_pct >= 100 {
return None;
}
Some(format!(
"WARNING: the target window (bounds {x},{y} {w}x{h}) is only ~{visible_pct}% on-screen; off-screen regions are missing from screenshots and unreachable by coordinate input. Use move_window/resize_window (GNOME Shell extension or X11/EWMH only) to bring it fully on-screen."
))
}
/// The off-screen note for a click or scroll coordinate.
async fn off_screen_note_for_point(
&self,
x: i32,
y: i32,
route: PointerRoute,
) -> Option<String> {
self.named_off_screen_note("coordinate", x, y, route).await
}
/// Warn when a point is outside the captured desktop or on none of its
/// monitors. `name` calls the point out, such as "coordinate" or "start
/// coordinate". Points are capture pixels, so the logical monitor layout
/// is scaled onto the capture first; without a layout that fits the
/// capture, only the capture rectangle is checked. The text says what
/// `route` did with such a point. This is a best-effort warning: never
/// request a new capture (and possibly consent) merely to decide whether
/// to append a note to an input result.
async fn named_off_screen_note(
&self,
name: &str,
x: i32,
y: i32,
route: PointerRoute,
) -> Option<String> {
let (mw, mh) = self.desktop_size.lock().ok().and_then(|guard| *guard)?;
let in_capture = x >= 0 && y >= 0 && (x as u32) < mw && (y as u32) < mh;
if !in_capture {
return match route {
PointerRoute::X11 => {
// The X server clamps each axis to the screen first.
let clamped = (
x.clamp(0, i32::try_from(mw).unwrap_or(i32::MAX) - 1),
y.clamp(0, i32::try_from(mh).unwrap_or(i32::MAX) - 1),
);
let in_gap =
self.capture_monitor_rects((mw, mh))
.await
.is_some_and(|monitors| {
!point_on_capture_monitor(&monitors, clamped.0, clamped.1)
});
Some(if in_gap {
format!(
"WARNING: {name} {x},{y} is outside the captured desktop ({mw}x{mh}); the X server keeps the pointer on the screen, but the nearest point there, {},{}, is on no monitor, so the input landed at a monitor edge or on no visible pixel.",
clamped.0, clamped.1
)
} else {
format!(
"WARNING: {name} {x},{y} is outside the captured desktop ({mw}x{mh}); the X server keeps the pointer on the screen, so the input landed at an edge instead."
)
})
}
// The device clamped the point, and the clamp note says where.
PointerRoute::Uinput => None,
// The portal sent nothing; its error says so.
PointerRoute::Portal => Some(format!(
"WARNING: {name} {x},{y} is outside the captured desktop ({mw}x{mh})."
)),
};
}
let monitors = self.capture_monitor_rects((mw, mh)).await?;
if point_on_capture_monitor(&monitors, x, y) {
return None;
}
Some(match route {
PointerRoute::Portal => format!(
"WARNING: {name} {x},{y} is on no monitor of the captured desktop ({mw}x{mh})."
),
PointerRoute::X11 | PointerRoute::Uinput => format!(
"WARNING: {name} {x},{y} is on no monitor of the captured desktop ({mw}x{mh}); the input landed at a monitor edge or on no visible pixel."
),
})
}
/// The off-screen note for each drag endpoint, naming the start and the
/// end. Only the endpoints are checked: the portal moves intermediate
/// motion onto the nearest shared monitor, and on X11 and uinput the drop
/// goes to the end point.
async fn drag_off_screen_notes(
&self,
start: (i32, i32),
end: (i32, i32),
route: PointerRoute,
) -> Vec<String> {
let mut notes = Vec::new();
for (name, (x, y)) in [("start coordinate", start), ("end coordinate", end)] {
notes.extend(self.named_off_screen_note(name, x, y, route).await);
}
notes
}
/// The notes for a uinput drag: per endpoint, the clamp note, then the
/// monitor check of the point the device emitted.
async fn uinput_drag_notes(
&self,
result: &std::result::Result<crate::abs_pointer::DragLanding, String>,
) -> Vec<String> {
let Ok(landing) = result else {
return Vec::new();
};
let mut notes = Vec::new();
for (name, landing) in [
("start coordinate", landing.start),
("end coordinate", landing.end),
] {
notes.extend(abs_pointer_clamp_note(name, landing));
let (x, y) = landing.emitted;
notes.extend(
self.named_off_screen_note(name, x, y, PointerRoute::Uinput)
.await,
);
}
notes
}
/// The off-screen note for a portal click or scroll at a capture point.
/// The caller holds a cached portal session, so a layout read takes the
/// session's layout and needs no query when the session has one.
async fn portal_off_screen_note(&self, point: Option<(i32, i32)>) -> Option<String> {
let (x, y) = point?;
self.off_screen_note_for_point(x, y, PointerRoute::Portal)
.await
}
/// The off-screen note for the point the uinput pointer emitted, which
/// is inside the capture after its clamp.
async fn uinput_off_screen_note(
&self,
result: &std::result::Result<crate::abs_pointer::PointerLanding, String>,
) -> Option<String> {
let (x, y) = result.as_ref().ok()?.emitted;
self.off_screen_note_for_point(x, y, PointerRoute::Uinput)
.await
}
/// The cached capture's monitors in capture pixels, or None when the
/// layout is unknown or a Wayland layout does not scale evenly onto the
/// capture. The layout is read at most once per capture: from the portal
/// session when one is cached, as the portal checks points against it,
/// otherwise from the compositor, or from RandR on X11.
async fn capture_monitor_rects(
&self,
capture: (u32, u32),
) -> Option<Vec<(i64, i64, i64, i64)>> {
let (generation, monitors) = {
let layout = self.capture_layout.lock().ok()?;
(layout.generation, layout.monitors.clone())
};
let monitors = match monitors {
CaptureMonitors::Unread => {
let read = self.read_monitor_layout().await;
if let Ok(mut layout) = self.capture_layout.lock() {
if layout.generation == generation {
layout.monitors = read.clone();
}
}
read
}
read => read,
};
match monitors {
CaptureMonitors::Logical(rects) => monitor_rects_in_capture(&rects, capture),
CaptureMonitors::Root(rects) => root_monitor_rects(&rects),
CaptureMonitors::Unread | CaptureMonitors::Unknown => None,
}
}
async fn read_monitor_layout(&self) -> CaptureMonitors {
let wayland = self.is_wayland_session();
let layout = match self
.cached_portal_pointer_session()
.and_then(|session| session.monitor_layout().map(<[_]>::to_vec))
{
Some(layout) => Some(layout),
None => match crate::remote_desktop::kwin_desktop_layout().await {
Some(layout) => Some(layout),
// An X client could make a Wayland compositor start Xwayland.
None if wayland => crate::remote_desktop::compositor_desktop_layout().await,
None => crate::remote_desktop::logical_desktop_layout().await,
},
};
let Some(layout) = layout else {
return CaptureMonitors::Unknown;
};
let rects = layout
.iter()
.map(|monitor| (monitor.x, monitor.y, monitor.width, monitor.height))
.collect();
if wayland {
CaptureMonitors::Logical(rects)
} else {
CaptureMonitors::Root(rects)
}
}
/// Post-input feedback: which AT-SPI element holds keyboard focus in the
/// target app, and whether it is editable. Guards against the blind-typing
/// trap where verified *window* focus still sends keystrokes nowhere.
async fn focused_element_feedback(
&self,
focus: Option<&WindowFocusResult>,
expects_editable: bool,
) -> Option<String> {
let focus = focus?;
let pid = focus
.focused_window
.as_ref()
.and_then(|window| window.pid)
.or(focus.requested_window.pid);
match timeout(Duration::from_millis(1500), probe_focused_element(pid)).await {
Ok(Ok(probe)) => Some(focus_probe_feedback(
&probe,
expects_editable,
focus.exact_window_focused,
)),
Ok(Err(error)) => Some(format!(
"Focused-element feedback unavailable ({}).",
first_line(&format!("{error:#}"))
)),
Err(_) => {
Some("Focused-element feedback unavailable (AT-SPI probe timed out).".to_string())
}
}
}
/// Shared move/resize plumbing: resolve the window target, run the GNOME
/// Shell extension operation, then re-query bounds to report the result.
async fn window_geometry_op<F, Fut>(
&self,
received: Option<serde_json::Value>,
target: &WindowTarget,
op: F,
) -> Json<WindowGeometryOutput>
where
F: FnOnce(crate::windowing::WindowInfo) -> Fut,
Fut: Future<Output = Result<String>>,
{
let windows = match list_windows().await {
Ok(windows) => windows,
Err(error) => {
let error = format!("{error:#}");
return Json(WindowGeometryOutput {
ok: false,
implemented: true,
backend: "unknown".to_string(),
window: None,
message: format!("Window listing failed: {error}"),
permissions_hint: window_permission_hint(&error),
received,
});
}
};
let window = match resolve_window_target(&windows, target) {
Ok(window) => window.clone(),
Err(error) => {
return Json(WindowGeometryOutput {
ok: false,
implemented: true,
backend: "unknown".to_string(),
window: None,
message: format!("{error:#}"),
permissions_hint: None,
received,
});
}
};
let backend = window.backend.clone();
let window_id = window.window_id;
match op(window).await {
Ok(message) => {
// Re-query so the caller sees the compositor-final geometry
// (tiling constraints, minimum sizes, etc. may adjust it).
let window = list_windows()
.await
.inspect_err(|_| self.diagnostics.invalidate())
.ok()
.and_then(|windows| {
windows
.into_iter()
.find(|window| window.window_id == window_id)
});
let mut message = message;
if let Some(bounds) = window.as_ref().and_then(|window| window.bounds.as_ref()) {
if let Some(note) = self.off_screen_note_for_bounds(bounds).await {
message = format!("{message} {note}");
}
}
Json(WindowGeometryOutput {
ok: true,
implemented: true,
backend,
window,
message,
permissions_hint: None,
received,
})
}
Err(error) => {
let error = format!("{error:#}");
Json(WindowGeometryOutput {
ok: false,
implemented: true,
backend,
window: None,
permissions_hint: window_permission_hint(&error),
message: error,
received,
})
}
}
}
/// Notes appended after targeted keyboard input: off-screen window warning
/// plus focused-element feedback.
async fn input_landing_notes(
&self,
focus: Option<&WindowFocusResult>,
expects_editable: bool,
) -> Vec<String> {
let mut notes = Vec::new();
if let Some(focus) = focus {
let bounds = focus
.focused_window
.as_ref()
.and_then(|window| window.bounds.as_ref())
.or(focus.requested_window.bounds.as_ref());
if let Some(bounds) = bounds {
if let Some(note) = self.off_screen_note_for_bounds(bounds).await {
notes.push(note);
}
}
}
if let Some(note) = self.focused_element_feedback(focus, expects_editable).await {
notes.push(note);
}
notes
}
#[cfg(test)]
fn cache_nodes(&self, nodes: &[AccessibilityNode]) {
self.cache_snapshot(nodes, None);
}
/// Readers may perform slow AT-SPI owner checks before resolving a click
/// or scroll again. Keep the snapshot stable for the whole input operation;
/// otherwise a checked index could resolve to a different app's node.
#[cfg(test)]
async fn commit_snapshot(&self, nodes: &[AccessibilityNode], target_pid: Option<u32>) {
let _input_guard = Arc::clone(&self.input_operation_lock).lock_owned().await;
self.cache_snapshot(nodes, target_pid);
}
/// Drop the cached tree, under the input lock like every store. Over HTTP
/// the tree stays: other clients still hold its snapshot_id.
async fn commit_cleared_snapshot(&self) {
if self.require_snapshot_ids {
return;
}
let _input_guard = Arc::clone(&self.input_operation_lock).lock_owned().await;
self.clear_snapshot();
}
/// The cached tree's pid and window scope, for a post-action tree with
/// no target of its own: always on stdio, and over HTTP only when the
/// action named that snapshot, since another client may have stored it.
fn post_action_scope(
&self,
cached: &CachedAccessibilitySnapshot,
named_snapshot: Option<&str>,
) -> PostActionScope {
if !self.snapshot_readable_by_call(cached, named_snapshot) {
return PostActionScope::default();
}
let tree = cached.window_scope.clone().map(|scope| CachedTreeWindow {
scope,
pid: cached.pid,
window: cached.window_id.zip(cached.window_backend.clone()),
});
PostActionScope {
pid: cached.pid,
tree,
bound: false,
}
}
/// Whether a call may read the cached snapshot: always on stdio, and over
/// HTTP only when the call named it, since another client may have stored it.
fn snapshot_readable_by_call(
&self,
cached: &CachedAccessibilitySnapshot,
named_snapshot: Option<&str>,
) -> bool {
!self.require_snapshot_ids
|| trimmed_nonempty(named_snapshot).is_some_and(|named| named == cached.id)
}
/// On stdio the post-action tree reads the scope cached at capture time.
fn cached_post_action_context(&self) -> PostActionScope {
self.last_snapshot
.lock()
.map(|snapshot| self.post_action_scope(&snapshot, None))
.unwrap_or_default()
}
/// Over HTTP the scope was taken under the action's input lock; stdio
/// reads the cache. A scope bound to an element's owner holds on both.
fn post_action_scope_for_capture(&self, carried: PostActionScope) -> PostActionScope {
if self.require_snapshot_ids || carried.bound {
carried
} else {
self.cached_post_action_context()
}
}
/// Drop the cached tree and its spill file. The empty snapshot has no id,
/// so no result names it and every snapshot_id is refused until the next
/// tree is stored.
fn clear_snapshot(&self) {
self.tree_spiller.remove_current();
if let Ok(mut cached) = self.last_snapshot.lock() {
*cached = CachedAccessibilitySnapshot::default();
}
}
/// The snapshot_id of the cached tree, empty when none is cached.
#[cfg(test)]
fn cached_snapshot_id(&self) -> String {
self.last_snapshot
.lock()
.map(|snapshot| snapshot.id.clone())
.unwrap_or_default()
}
/// Refuse an element call whose `snapshot_id` names a replaced snapshot,
/// and return the scope its post-action tree may reuse. Callers hold the
/// input lock, which every store takes, so the snapshot checked here is
/// the one the call then resolves its element in; one lock section reads
/// the id and the scope.
fn check_element_snapshot(
&self,
action: &str,
snapshot_id: Option<&str>,
names_cached_element: bool,
) -> std::result::Result<PostActionScope, String> {
let (cached_id, scope) = self
.last_snapshot
.lock()
.map(|snapshot| {
(
snapshot.id.clone(),
self.post_action_scope(&snapshot, snapshot_id),
)
})
.unwrap_or_default();
if !names_cached_element {
return Ok(scope);
}
self.require_snapshot_id(action, snapshot_id)?;
check_snapshot_id(snapshot_id, &cached_id, &self.snapshot_ids.prefix)
.map_err(|message| format!("{action} was not sent: {message}"))?;
Ok(scope)
}
/// Over HTTP, refuse a cached-element call that names no snapshot.
fn require_snapshot_id(
&self,
action: &str,
snapshot_id: Option<&str>,
) -> std::result::Result<(), String> {
if self.require_snapshot_ids && trimmed_nonempty(snapshot_id).is_none() {
return Err(format!(
"{action} was not sent: snapshot_id is required over HTTP for element_index and selector calls, because another client's get_app_state can replace the cached tree. Pass the snapshot_id from your get_app_state result."
));
}
Ok(())
}
/// Commit a post-action tree with the window it was taken for and its
/// window scope, under the same guard as the nodes.
async fn commit_observed_snapshot(
&self,
nodes: &[AccessibilityNode],
target_pid: Option<u32>,
window: Option<(u64, String)>,
window_scope: Option<String>,
) -> String {
let _input_guard = Arc::clone(&self.input_operation_lock).lock_owned().await;
self.store_snapshot_for(
nodes,
target_pid,
None,
window,
TreeShape {
detail: NodeDetail::Compact,
window_scope,
},
)
}
async fn commit_app_state_snapshot(
&self,
nodes: &[AccessibilityNode],
target_pid: Option<u32>,
window: Option<&WindowInfo>,
params: &GetAppStateParams,
scoped: bool,
shape: TreeShape,
) -> String {
let _input_guard = Arc::clone(&self.input_operation_lock).lock_owned().await;
let recovery_scope = scoped.then(|| AccessibilityRecoveryScope {
params: params.clone(),
target_pid,
});
self.store_snapshot(nodes, target_pid, recovery_scope, window, shape)
}
#[cfg(test)]
fn cache_snapshot(&self, nodes: &[AccessibilityNode], target_pid: Option<u32>) -> String {
self.cache_snapshot_with_scope(nodes, target_pid, None)
}
#[cfg(test)]
fn cache_snapshot_with_scope(
&self,
nodes: &[AccessibilityNode],
target_pid: Option<u32>,
recovery_scope: Option<AccessibilityRecoveryScope>,
) -> String {
self.cache_snapshot_with_window(nodes, target_pid, recovery_scope, None)
}
#[cfg(test)]
fn cache_snapshot_with_window(
&self,
nodes: &[AccessibilityNode],
target_pid: Option<u32>,
recovery_scope: Option<AccessibilityRecoveryScope>,
window: Option<&WindowInfo>,
) -> String {
self.store_snapshot(
nodes,
target_pid,
recovery_scope,
window,
TreeShape::default(),
)
}
/// Replace the cached snapshot in one write, so its nodes, ownership,
/// encoding and window scope always come from the same tree.
fn store_snapshot(
&self,
nodes: &[AccessibilityNode],
target_pid: Option<u32>,
recovery_scope: Option<AccessibilityRecoveryScope>,
window: Option<&WindowInfo>,
shape: TreeShape,
) -> String {
self.store_snapshot_for(
nodes,
target_pid,
recovery_scope,
window.map(|window| (window.window_id, window.backend.clone())),
shape,
)
}
/// store_snapshot for a window given as its id and backend. Returns the
/// new snapshot's id.
fn store_snapshot_for(
&self,
nodes: &[AccessibilityNode],
target_pid: Option<u32>,
recovery_scope: Option<AccessibilityRecoveryScope>,
window: Option<(u64, String)>,
shape: TreeShape,
) -> String {
// A spilled file matches only the snapshot it was written from.
self.tree_spiller.remove_current();
let id = self.snapshot_ids.next();
if let Ok(mut cached) = self.last_snapshot.lock() {
*cached = CachedAccessibilitySnapshot {
id: id.clone(),
nodes: nodes.to_vec(),
pid: target_pid,
window_id: window.as_ref().map(|(window_id, _)| *window_id),
window_backend: window.map(|(_, backend)| backend),
recovery_scope,
detail: shape.detail,
window_scope: shape.window_scope,
};
}
id
}
#[cfg(test)]
fn clear_cached_nodes(&self) {
self.clear_snapshot();
}
/// The window the cached snapshot was taken for, with its backend.
fn snapshot_window(&self) -> Option<(u64, String)> {
self.last_snapshot
.lock()
.ok()
.and_then(|snapshot| snapshot.window_id.zip(snapshot.window_backend.clone()))
}
/// Reject an element-targeted action whose cached node belongs to a
/// different app or window than the action's target. With a target pid,
/// a snapshot taken for a pid is judged by that pid; an untargeted
/// snapshot mixes apps, so its node is judged by the pid that owns it on
/// the accessibility bus. When neither is known the action proceeds, since
/// a mismatch cannot be shown. When the pids match and the snapshot's tree
/// was narrowed to one window of the app, a window target must name that
/// window, since an app's windows share its pid. A window target with no known pid (i3
/// without xprop, COSMIC) is judged by the snapshot's window and
/// fails closed when the snapshot recorded no comparable window.
async fn check_node_target(
&self,
node: &AccessibilityNode,
target: &ElementOwnerTarget,
) -> std::result::Result<(), String> {
if let Some(target_pid) = target.pid {
let snapshot_pid = self
.last_snapshot
.lock()
.ok()
.and_then(|snapshot| snapshot.pid);
let owner_pid = match snapshot_pid {
Some(pid) => Some(pid),
None => object_ref_owner_pid(&node.object_ref).await.ok().flatten(),
};
if let Some(owner_pid) = owner_pid.filter(|owner_pid| *owner_pid != target_pid) {
return Err(node_target_mismatch_message(
node.index, owner_pid, target_pid,
));
}
// Windows of one app share its pid. A tree narrowed to another
// window of the same app is still refused when the target names
// one window exactly; an app-wide tree holds every window, and an
// inexact target picked one arbitrarily, so both pass on the pid.
// A narrowed tree that recorded no comparable window cannot show
// which window it holds, so an exact target fails closed.
let Some((target_window, target_backend)) =
target.window.as_ref().filter(|_| target.exact_window)
else {
return Ok(());
};
let narrowed = self
.last_snapshot
.lock()
.ok()
.is_some_and(|snapshot| snapshot.window_scope.is_some());
if !narrowed {
return Ok(());
}
return self.check_snapshot_window(
node.index,
*target_window,
target_backend,
node_window_unrecorded_message,
);
}
let Some((target_window, target_backend)) = target.window.as_ref() else {
return Ok(());
};
self.check_snapshot_window(
node.index,
*target_window,
target_backend,
node_window_unverified_message,
)
}
/// Compare the snapshot's recorded window with the target window. A
/// snapshot with no window, or one listed by another window backend,
/// cannot be compared and is refused with `uncomparable`'s message.
fn check_snapshot_window(
&self,
element_index: u32,
target_window: u64,
target_backend: &str,
uncomparable: fn(u32, u64) -> String,
) -> std::result::Result<(), String> {
match self.snapshot_window() {
Some((window_id, backend)) if backend == target_backend => {
if window_id == target_window {
Ok(())
} else {
Err(node_window_mismatch_message(
element_index,
window_id,
target_window,
))
}
}
_ => Err(uncomparable(element_index, target_window)),
}
}
/// Reject an element action whose cached element_index or selector
/// resolves to a node from another app's or window's snapshot. Shared by
/// click, scroll, perform_action and set_value. A call that names no
/// cached element or no target passes; resolution errors are left to the
/// tool.
async fn check_cached_element_target(
&self,
element_index: Option<u32>,
selector: &ElementSelector<'_>,
purpose: ElementResolvePurpose,
target: &ElementOwnerTarget,
) -> std::result::Result<(), String> {
match self.cached_node_for(element_index, selector, purpose) {
Some(node) => self.check_node_target(&node, target).await,
None => Ok(()),
}
}
/// The app or window an AT-SPI element action targets, without raising
/// the window: the action needs no focus. A bare pid is used as given;
/// window_id, app_id, wm_class and title resolve through the window
/// matcher click uses before it focuses. An unresolvable target fails
/// closed.
async fn element_action_owner(
&self,
target: Option<&WindowTarget>,
) -> std::result::Result<ElementOwnerTarget, String> {
let Some(target) = target.filter(|target| target.has_target()) else {
return Ok(ElementOwnerTarget::default());
};
let pid_only = target.window_id.is_none()
&& trimmed_nonempty(target.app_id.as_deref()).is_none()
&& trimmed_nonempty(target.wm_class.as_deref()).is_none()
&& trimmed_nonempty(target.title.as_deref()).is_none();
if pid_only {
return Ok(ElementOwnerTarget::pid(target.pid));
}
element_action_owner_from(list_windows().await, target)
}
/// The cached node an element-targeted action would act on, when the
/// call names one. Resolution errors are left to the tool's own path.
fn cached_node_for(
&self,
element_index: Option<u32>,
selector: &ElementSelector<'_>,
purpose: ElementResolvePurpose,
) -> Option<AccessibilityNode> {
if element_index.is_none() && selector.is_empty() {
return None;
}
self.resolve_cached_node(element_index, selector, purpose)
.ok()
}
fn resolve_optional_target_point(
&self,
x: Option<i32>,
y: Option<i32>,
element_index: Option<u32>,
) -> std::result::Result<Option<(i32, i32)>, String> {
match (x.zip(y), element_index) {
(Some(point), _) => Ok(Some(point)),
(None, Some(index)) => self
.center_for_cached_node(index)
.map(Some)
.ok_or_else(|| {
format!(
"No clickable bounds cached for element_index {index}. Call get_app_state first and choose a node with positive width and height."
)
}),
(None, None) => Ok(None),
}
}
fn should_map_kwin_element_click(&self, focus: Option<&WindowFocusResult>) -> bool {
let cached_backend = self
.last_snapshot
.lock()
.ok()
.and_then(|snapshot| snapshot.window_backend.clone());
kwin_element_adapter_required(
self.is_wayland_session(),
self.is_kde_wayland_session(),
cached_backend.as_deref(),
focus.map(|focus| focus.requested_window.backend.as_str()),
)
}
async fn prepare_kwin_element_click(
&self,
node: &AccessibilityNode,
center: (i32, i32),
global_point: bool,
focus: Option<&WindowFocusResult>,
) -> std::result::Result<PreparedKwinElementClick, String> {
let mut target = self.cached_kwin_element_target(node, center, global_point, focus)?;
if target.pid.is_none() {
target.pid = Some(
object_ref_owner_pid(&node.object_ref)
.await
.map_err(|error| format!("could not identify the cached element's root PID: {error:#}"))?
.ok_or_else(|| {
"cached element's AT-SPI root PID is unavailable, so its KWin window identity cannot be verified; retry get_app_state with window_id".to_string()
})?,
);
}
let snapshot = crate::windowing::backends::kwin::element_click_snapshot()
.await
.map_err(|error| format!("could not obtain a coherent KWin snapshot: {error:#}"))?;
let raw = capture_screenshot_raw().await.map_err(|error| {
format!("could not obtain a current raw desktop capture: {error:#}")
})?;
if raw.width == 0
|| raw.height == 0
|| raw.width > i32::MAX as u32
|| raw.height > i32::MAX as u32
{
return Err(format!(
"current raw desktop capture has unusable dimensions {}x{}",
raw.width, raw.height
));
}
let point = snapshot
.map_element_click(&target, raw.width, raw.height)
.map_err(|error| format!("{error:#}"))?;
self.cache_desktop_size(raw.width, raw.height);
Ok(PreparedKwinElementClick {
element_index: node.index,
point,
capture_size: (raw.width, raw.height),
})
}
fn cached_kwin_element_target(
&self,
node: &AccessibilityNode,
center: (i32, i32),
global_point: bool,
focus: Option<&WindowFocusResult>,
) -> std::result::Result<KwinElementClickTarget, String> {
let cached = self
.last_snapshot
.lock()
.map_err(|_| "cached accessibility snapshot is unavailable".to_string())?;
let window_id = if let Some(focus) = focus {
let requested_id = focus.requested_window.window_id;
let exact = focus.exact_window_focused
&& focus
.focused_window
.as_ref()
.is_some_and(|window| window.window_id == requested_id);
if !exact {
return Err(
"explicit window targeting requires exact KWin window focus, not app-only focus"
.to_string(),
);
}
if let Some(cached_window_id) = cached.window_id {
if cached_window_id != requested_id {
return Err(format!(
"cached element belongs to window_id {cached_window_id}, not the explicitly targeted window_id {requested_id}"
));
}
}
Some(requested_id)
} else {
cached.window_id
};
let frame = depth_one_kwin_element_frame(&cached.nodes, node, center, global_point)?;
Ok(KwinElementClickTarget {
window_id,
pid: cached.pid,
frame,
center,
global_point,
require_exact_focus: focus.is_some(),
})
}
fn kwin_element_click_failure(
&self,
element_index: u32,
detail: &str,
received: Option<serde_json::Value>,
) -> Json<ActionOutput> {
let cached_window_id = self
.last_snapshot
.lock()
.ok()
.and_then(|snapshot| snapshot.window_id);
let window = cached_window_id
.map(|window_id| format!("window_id {window_id}"))
.unwrap_or_else(|| "window_id unavailable".to_string());
Json(ActionOutput {
ok: false,
implemented: true,
action: "click".to_string(),
message: format!(
"click was not sent: KWin element coordinate mapping failed for element_index {element_index} ({window}): {detail}. No pointer event was emitted. Call get_app_state with the intended window_id and a full-desktop screenshot, then retry with the refreshed element_index, or pass explicit screenshot x/y coordinates."
),
received,
})
}
fn resolve_click_target(
&self,
params: &ClickParams,
) -> std::result::Result<ClickTarget, String> {
if params.coordinate_space == ClickCoordinateSpace::Capture {
if let Some((x, y)) = params.x.zip(params.y) {
return Ok(ClickTarget::Coordinates(x, y));
}
}
let selector = params.selector();
let node = self.resolve_cached_node(
params.element_index,
&selector,
ElementResolvePurpose::Click,
)?;
if params.coordinate_space == ClickCoordinateSpace::Atspi {
let point = params
.x
.zip(params.y)
.ok_or_else(|| "AT-SPI coordinate clicks require both x and y.".to_string())?;
return Ok(ClickTarget::ElementPoint {
node: Box::new(node),
point,
});
}
// Toolkit extents are not necessarily in the pointer backend's space
// (GTK3 X11 HiDPI, GTK4 zero-origin bounds). Ordinary activation does
// not need a guessed scale when the target exposes its native action.
if is_plain_left_click(params.button.as_deref(), params.click_count) {
if let Some(action) =
primary_action(&node.actions).filter(|action| click_equivalent_action(action))
{
return Ok(ClickTarget::PrimaryAction {
object_ref: node.object_ref.clone(),
action_name: Some(action.name.clone()),
action_index: action.index,
});
}
}
if let Some(center) = node.bounds.as_ref().and_then(bounds_center) {
return Ok(ClickTarget::ElementBounds {
node: Box::new(node),
center,
});
}
if !is_plain_left_click(params.button.as_deref(), params.click_count) {
return Err(format!(
"No clickable bounds cached for element_index {}. Call get_app_state first and choose a node with positive width and height.",
node.index
));
}
Err(format!(
"No clickable bounds or click-equivalent AT-SPI action for element_index {}. Use perform_action explicitly for other actions.",
node.index
))
}
fn center_for_cached_node(&self, element_index: u32) -> Option<(i32, i32)> {
bounds_center(&self.cached_node_bounds(element_index)?)
}
fn cached_node_bounds(&self, element_index: u32) -> Option<crate::atspi_tree::Bounds> {
let cached = self.last_snapshot.lock().ok()?;
cached
.nodes
.iter()
.find(|node| node.index == element_index)?
.bounds
.clone()
}
/// The target check perform_action and set_value run before resolving
/// their element. An element_identifier names the object itself, not a
/// cached snapshot node, so it is checked only against a known target pid,
/// by the pid that owns it on the accessibility bus.
async fn check_element_action_target(
&self,
target: &ElementActionTarget,
element_index: Option<u32>,
element_identifier: Option<&str>,
selector: &ElementSelector<'_>,
purpose: ElementResolvePurpose,
) -> std::result::Result<(), String> {
let owner = self
.element_action_owner(target.window_target().as_ref())
.await?;
if let Some(element_identifier) = trimmed_nonempty(element_identifier) {
let Some(target_pid) = owner.pid else {
return Ok(());
};
let owner_pid = object_ref_owner_pid(element_identifier)
.await
.ok()
.flatten();
return check_identifier_owner(owner_pid, Some(target_pid));
}
self.check_cached_element_target(element_index, selector, purpose, &owner)
.await
}
fn resolve_object_ref(
&self,
element_index: Option<u32>,
element_identifier: Option<&str>,
selector: &ElementSelector<'_>,
purpose: ElementResolvePurpose,
) -> std::result::Result<String, String> {
if let Some(element_identifier) = element_identifier
.map(str::trim)
.filter(|value| !value.is_empty())
{
return Ok(element_identifier.to_string());
}
self.resolve_cached_node(element_index, selector, purpose)
.map(|node| node.object_ref)
}
fn resolve_cached_node(
&self,
element_index: Option<u32>,
selector: &ElementSelector<'_>,
purpose: ElementResolvePurpose,
) -> std::result::Result<AccessibilityNode, String> {
let cached = self.last_snapshot.lock().map_err(|_| {
"Could not read cached accessibility nodes. Call get_app_state and retry.".to_string()
})?;
if let Some(element_index) = element_index {
return cached.nodes
.iter()
.find(|node| node.index == element_index)
.cloned()
.ok_or_else(|| {
format!(
"No cached accessibility node for element_index {element_index}. Call get_app_state first."
)
});
}
if selector.is_empty() {
return Err(
"Pass element_index, element_identifier, or a semantic selector such as role/name/text/states from the latest get_app_state result."
.to_string(),
);
}
resolve_semantic_node(cached.nodes.as_slice(), selector, purpose)
}
async fn refresh_action_snapshot(
&self,
scope: &AccessibilityRecoveryScope,
) -> Result<RefreshedAccessibilitySnapshot> {
let params = &scope.params;
let (window_context, _, _, window_enumerated, _) =
self.resolve_window_context(params).await;
let app_filter = self
.resolve_accessibility_app_filter(params, window_context.as_ref(), window_enumerated)
.await;
let target_pid = action_recovery_target_pid(
scope,
window_context.as_ref().and_then(|window| window.pid),
);
let (max_nodes, max_depth) =
crate::atspi_tree::snapshot_limits(params.max_nodes, params.max_depth);
let window_title = accessibility_window_title(params, window_context.as_ref());
let snapshot = snapshot_accessibility_tree(
app_filter.as_deref(),
accessibility_snapshot_target_pid(app_filter.as_deref(), target_pid),
window_title.as_deref(),
// Recovery re-finds an element by its verified owner only.
None,
max_nodes,
max_depth,
)
.await?;
Ok(RefreshedAccessibilitySnapshot {
snapshot,
target_pid,
window: window_context,
})
}
async fn perform_element_action(
&self,
params: &ActionParams,
requested_action: Option<&str>,
) -> (
Json<ActionOutput>,
std::result::Result<ElementObservationTarget, String>,
) {
let target = Mutex::new(None);
let capture_requested = params.capture.include_tree.unwrap_or(false);
// The post-action tree follows the element's bound owner, not the
// snapshot scope the call checked.
let (output, _) = self
.perform_element_action_with(
params,
requested_action,
|object_ref, requested_action| {
let target = ⌖
async move {
if capture_requested {
let mut bound = self
.element_observation_target(
¶ms.target,
&object_ref,
params.snapshot_id.as_deref(),
)
.await;
let mut stored = target.lock().unwrap();
if let (Ok(bound), Some(Ok(previous))) = (&mut bound, stored.as_ref()) {
bound.inherit_tree_window(previous);
}
*stored = Some(bound);
}
invoke_accessibility_action(&object_ref, requested_action.as_deref()).await
}
},
|scope| async move { self.refresh_action_snapshot(&scope).await },
)
.await;
(
output,
target
.into_inner()
.unwrap()
.unwrap_or_else(|| Ok(ElementObservationTarget::default())),
)
}
/// The dispatched element's verified owner and the window range its
/// tree keeps. Over HTTP the range comes only from a snapshot the call
/// named, as for every other post-action tree.
async fn element_observation_target(
&self,
requested: &ElementActionTarget,
object_ref: &str,
named_snapshot: Option<&str>,
) -> std::result::Result<ElementObservationTarget, String> {
let selector_owner = if requested.window_id.is_none() && requested.pid.is_none() {
Some(
self.element_action_owner(requested.window_target().as_ref())
.await?,
)
} else {
None
};
let mut target = self
.element_observation_target_with(requested, object_ref, named_snapshot, || async {
object_ref_application_pid(object_ref).await
})
.await?;
if let Some(owner) = selector_owner.filter(|owner| owner.exact_window) {
check_identifier_owner(target.pid, owner.pid)?;
// The preceding target check verifies pidless windows through the
// observed snapshot. A raw, uncached identifier has no such proof.
let observed_pidless_window = owner.pid.is_none()
&& self.last_snapshot.lock().ok().is_some_and(|snapshot| {
self.snapshot_readable_by_call(&snapshot, named_snapshot)
&& snapshot
.nodes
.iter()
.any(|node| node.object_ref == object_ref)
&& owner.window.as_ref().is_some_and(|(window_id, backend)| {
snapshot.window_id == Some(*window_id)
&& snapshot.window_backend.as_ref() == Some(backend)
})
});
// Adding the verified PID must not override a title/app/class
// selector that already identified one of this process's windows.
if owner.pid == target.pid || observed_pidless_window {
target.window_id = owner.window.map(|(window_id, _)| window_id);
}
}
let tree_window = self.last_snapshot.lock().ok().and_then(|snapshot| {
if snapshot.pid.is_none()
|| snapshot.pid != target.pid
|| !snapshot
.nodes
.iter()
.any(|node| node.object_ref == object_ref)
{
return None;
}
self.post_action_scope(&snapshot, named_snapshot).tree
});
Ok(ElementObservationTarget {
owner: target,
tree_window,
})
}
async fn element_observation_target_with<F, Fut>(
&self,
requested: &ElementActionTarget,
object_ref: &str,
named_snapshot: Option<&str>,
owner_pid: F,
) -> std::result::Result<WindowTarget, String>
where
F: FnOnce() -> Fut,
Fut: Future<Output = Result<Option<u32>>>,
{
let cached_target = self.last_snapshot.lock().ok().and_then(|snapshot| {
if !self.snapshot_readable_by_call(&snapshot, named_snapshot) {
return None;
}
let node = snapshot
.nodes
.iter()
.find(|node| node.object_ref == object_ref)?;
let window_id = snapshot.window_scope.as_deref().and_then(|scope| {
// A narrowed tree also keeps dialogs and other toolkit roots.
// Only this node's nearest window ancestor proves its window.
let mut ancestor = node;
for _ in 0..snapshot.nodes.len() {
if crate::atspi_tree::is_window_role(&ancestor.role) {
return ancestor
.name
.as_deref()
.filter(|name| *name == scope)
.and(snapshot.window_id);
}
let parent_index = ancestor.parent_index?;
ancestor = snapshot
.nodes
.iter()
.find(|node| node.index == parent_index)?;
}
None
});
Some(WindowTarget {
pid: snapshot.pid,
window_id,
..Default::default()
})
.filter(WindowTarget::has_target)
});
let observed_pid = cached_target.as_ref().and_then(|target| target.pid);
let mut target = requested
.window_target()
.or(cached_target)
.unwrap_or_default();
// window_id takes precedence over a caller's pid. Keep the dispatched
// object's observed owner for a window that the action may close.
if observed_pid.is_some() {
target.pid = observed_pid;
} else if requested.window_id.is_some() {
target.pid = None;
}
if target.pid.is_some() {
return Ok(target);
}
// Resolve the dispatched object, never the app that happens to be in
// front or an unrelated snapshot replaced by another MCP client. Even
// a pidless compositor window needs this owner if the window closes.
let pid = owner_pid()
.await
.map_err(|error| format!("could not read the element owner's PID: {error:#}"))?
.ok_or_else(|| {
"the dispatched element has no verified app or window owner".to_string()
})?;
target.pid = Some(pid);
Ok(target)
}
async fn perform_element_action_with<I, R, IFut, RFut>(
&self,
params: &ActionParams,
requested_action: Option<&str>,
invoke: I,
refresh: R,
) -> (Json<ActionOutput>, PostActionScope)
where
I: FnMut(String, Option<String>) -> IFut,
IFut: Future<Output = Result<crate::atspi_tree_impl::ActionInvocation>>,
R: FnOnce(AccessibilityRecoveryScope) -> RFut,
RFut: Future<Output = Result<RefreshedAccessibilitySnapshot>>,
{
let _input_guard = Arc::clone(&self.input_operation_lock).lock_owned().await;
let received = Some(serde_json::json!(params.clone()));
// An element_identifier names the object itself, not a snapshot node.
match self.check_element_snapshot(
"perform_action",
params.snapshot_id.as_deref(),
trimmed_nonempty(params.element_identifier.as_deref()).is_none()
&& (params.element_index.is_some() || !params.selector().is_empty()),
) {
Ok(scope) => (
self.perform_checked_element_action(
params,
requested_action,
invoke,
refresh,
received,
)
.await,
scope,
),
Err(message) => (
action_error_output(message, &received),
PostActionScope::default(),
),
}
}
/// The rest of perform_action; the caller holds the input lock.
async fn perform_checked_element_action<I, R, IFut, RFut>(
&self,
params: &ActionParams,
requested_action: Option<&str>,
mut invoke: I,
refresh: R,
received: Option<serde_json::Value>,
) -> Json<ActionOutput>
where
I: FnMut(String, Option<String>) -> IFut,
IFut: Future<Output = Result<crate::atspi_tree_impl::ActionInvocation>>,
R: FnOnce(AccessibilityRecoveryScope) -> RFut,
RFut: Future<Output = Result<RefreshedAccessibilitySnapshot>>,
{
if let Err(message) = self
.check_element_action_target(
¶ms.target,
params.element_index,
params.element_identifier.as_deref(),
¶ms.selector(),
ElementResolvePurpose::Action,
)
.await
{
return action_error_output(message, &received);
}
let object_ref = match self.resolve_object_ref(
params.element_index,
params.element_identifier.as_deref(),
¶ms.selector(),
ElementResolvePurpose::Action,
) {
Ok(object_ref) => object_ref,
Err(message) => return action_error_output(message, &received),
};
self.indicator.action("perform_action").await;
let error = match invoke(object_ref.clone(), requested_action.map(str::to_string)).await {
Ok(invocation) => return action_invocation_output(invocation, &received),
Err(error) => error,
};
let semantic_recovery = params.element_index.is_none()
&& trimmed_nonempty(params.element_identifier.as_deref()).is_none()
&& !params.selector().is_empty();
if !semantic_recovery || !is_recoverable_action_query_error(&error) {
return action_error_output(error.to_string(), &received);
}
let original_error = format!("{error:#}");
let (recovery_scope, original_window) = self
.last_snapshot
.lock()
.ok()
.map(|snapshot| {
(
snapshot.recovery_scope.clone(),
snapshot.window_id.zip(snapshot.window_backend.clone()),
)
})
.unwrap_or((None, None));
let Some(recovery_scope) = recovery_scope else {
return action_error_output(
format!(
"{original_error}. The cached snapshot has no stable scoped recovery target. Call get_app_state for the target app, then retry perform_action."
),
&received,
);
};
let original_pid = action_recovery_target_pid(&recovery_scope, None);
let observed_window_scope =
recovery_scope.params.window_id.is_some() && original_window.is_some();
if original_pid.is_none() && !observed_window_scope {
return action_error_output(
format!(
"{original_error}. Automatic AT-SPI target recovery requires an observed PID or exact window-scoped snapshot. An app name, app_id, title, or an unobserved window_id cannot establish the original app identity. Call get_app_state with pid or window_id before retrying."
),
&received,
);
}
let refreshed = match refresh(recovery_scope.clone()).await {
Ok(refreshed) => refreshed,
Err(refresh_error) => {
return action_error_output(
format!(
"{original_error}. Automatic AT-SPI target recovery could not refresh the original scoped snapshot: {refresh_error:#}. Call get_app_state for the target app, then retry perform_action."
),
&received,
);
}
};
let snapshot = refreshed.snapshot;
// Fresh window discovery must not replace the identity originally
// observed by get_app_state. A process change needs a new observation.
// A window match alone cannot identify its accessibility root.
let pid_scope_matches = original_pid
.or(refreshed.target_pid)
.is_some_and(|pid| snapshot.root_pid == Some(pid));
let window_scope_matches = if recovery_scope.params.window_id.is_some() {
// The resolver accepts JSON-rounded IDs. Compare the actual
// observed identity, not the raw requested number.
original_window
.as_ref()
.zip(refreshed.window.as_ref())
.is_some_and(|((window_id, backend), window)| {
window.window_id == *window_id && window.backend == *backend
})
} else {
true
};
if !snapshot.scoped
|| !snapshot.app_filter_matched
|| !pid_scope_matches
|| !window_scope_matches
{
return action_error_output(
format!(
"{original_error}. Automatic AT-SPI target recovery could not prove that the refreshed tree has the original app scope and any explicit window_id target. Call get_app_state for the target app, then retry perform_action."
),
&received,
);
}
let (nodes, _) = compact_accessibility_tree_with_note(
snapshot.nodes,
snapshot.scoped,
snapshot.truncated,
);
// A recovery tree has not been shown to the caller. Its compacted
// indices must never silently replace indices from get_app_state.
self.clear_snapshot();
let recovered = match resolve_semantic_node(
&nodes,
¶ms.selector(),
ElementResolvePurpose::Action,
) {
Ok(node) => node,
Err(message) => {
return action_error_output(
format!(
"{original_error}. AT-SPI target recovery refreshed the scoped snapshot but could not re-resolve the semantic selector: {message}. Call get_app_state for the target app and choose the element again."
),
&received,
);
}
};
if recovered.object_ref == object_ref {
return action_error_output(
format!(
"{original_error}. AT-SPI target recovery resolved the same dead object identity. Call get_app_state for the target app, then retry perform_action."
),
&received,
);
}
let Json(output) =
match invoke(recovered.object_ref, requested_action.map(str::to_string)).await {
Ok(invocation) => action_invocation_output(invocation, &received),
Err(error) => action_error_output(error.to_string(), &received),
};
Json(with_notes(output, [
"Recovery invalidated the cached element indices. Call get_app_state before any further element-indexed action.".to_string(),
]))
}
}
fn action_recovery_target_pid(
scope: &AccessibilityRecoveryScope,
resolved_window_pid: Option<u32>,
) -> Option<u32> {
scope
.params
.pid
.filter(|_| scope.params.window_id.is_none())
.or(scope.target_pid)
.or(resolved_window_pid)
}
fn is_recoverable_action_query_error(error: &anyhow::Error) -> bool {
if error.chain().any(|cause| {
cause
.to_string()
.starts_with("failed to invoke AT-SPI action ")
}) {
return false;
}
error.chain().any(|cause| {
if let Some(zbus::Error::MethodError(name, _, _)) = cause.downcast_ref::<zbus::Error>() {
if matches!(
name.as_str(),
"org.freedesktop.DBus.Error.ServiceUnknown"
| "org.freedesktop.DBus.Error.NameHasNoOwner"
) {
return true;
}
}
if matches!(
cause.downcast_ref::<zbus::fdo::Error>(),
Some(zbus::fdo::Error::ServiceUnknown(_) | zbus::fdo::Error::NameHasNoOwner(_))
) {
return true;
}
let message = cause.to_string();
message.contains("ServiceUnknown") || message.contains("NameHasNoOwner")
})
}
fn action_error_output(
message: String,
received: &Option<serde_json::Value>,
) -> Json<ActionOutput> {
Json(ActionOutput {
ok: false,
implemented: true,
action: "perform_action".to_string(),
message,
received: received.clone(),
})
}
fn action_invocation_output(
invocation: crate::atspi_tree_impl::ActionInvocation,
received: &Option<serde_json::Value>,
) -> Json<ActionOutput> {
Json(ActionOutput {
ok: invocation.ok,
implemented: true,
action: "perform_action".to_string(),
message: if invocation.ok {
format!(
"AT-SPI action {} ({}) invoked.",
invocation.action_index,
invocation
.action_name
.as_deref()
.filter(|name| !name.is_empty())
.unwrap_or("unnamed")
)
} else {
format!(
"AT-SPI action {} ({}) returned false.",
invocation.action_index,
invocation
.action_name
.as_deref()
.filter(|name| !name.is_empty())
.unwrap_or("unnamed")
)
},
received: received.clone(),
})
}
#[derive(Debug, Clone, Copy)]
struct PreparedKwinElementClick {
element_index: u32,
point: KwinElementClickPoint,
capture_size: (u32, u32),
}
impl PreparedKwinElementClick {
fn uinput_point(self) -> (i32, i32) {
self.point.capture
}
fn portal_point(self) -> (i32, i32) {
self.point.logical
}
}
fn kwin_element_adapter_required(
is_wayland: bool,
is_kde_wayland: bool,
cached_backend: Option<&str>,
explicit_backend: Option<&str>,
) -> bool {
if !is_wayland {
return false;
}
if let Some(backend) = explicit_backend {
return backend == KWIN_BACKEND;
}
cached_backend.map_or(is_kde_wayland, |backend| backend == KWIN_BACKEND)
}
fn depth_one_kwin_element_frame(
nodes: &[AccessibilityNode],
node: &AccessibilityNode,
center: (i32, i32),
global_point: bool,
) -> std::result::Result<KwinElementFrame, String> {
let mut parent_index = node.parent_index;
let mut visited = Vec::new();
while let Some(index) = parent_index {
if visited.contains(&index) {
return Err("cached element ancestry contains a cycle".to_string());
}
visited.push(index);
let parent = nodes
.iter()
.find(|candidate| candidate.index == index)
.ok_or_else(|| {
"cached element is missing its depth-1 frame/window ancestry".to_string()
})?;
if parent.depth == 1 {
if !matches!(
parent.role.trim().to_ascii_lowercase().as_str(),
"frame" | "window" | "dialog" | "alert"
) {
return Err(
"cached element's depth-1 ancestor is not a frame/window/dialog/alert"
.to_string(),
);
}
let bounds = parent
.bounds
.as_ref()
.ok_or_else(|| "cached element's depth-1 frame/window has no bounds".to_string())?;
bounds_center(bounds).ok_or_else(|| {
"cached element's depth-1 frame/window bounds are unusable".to_string()
})?;
let right = i64::from(bounds.x) + i64::from(bounds.width);
let bottom = i64::from(bounds.y) + i64::from(bounds.height);
if !global_point
&& (i64::from(center.0) < i64::from(bounds.x)
|| i64::from(center.0) >= right
|| i64::from(center.1) < i64::from(bounds.y)
|| i64::from(center.1) >= bottom)
{
return Err("element center is outside its depth-1 AT-SPI frame/window".to_string());
}
return Ok(KwinElementFrame {
x: bounds.x,
y: bounds.y,
width: bounds.width,
height: bounds.height,
});
}
if parent.depth == 0 {
break;
}
parent_index = parent.parent_index;
}
Err("cached element has no depth-1 frame/window ancestor".to_string())
}
pub(crate) fn pixelflux_point_for_element(
bounds: (i32, i32, i32, i32),
capture: (u32, u32),
layout: &[crate::remote_desktop::LogicalMonitor],
) -> Option<(i32, i32)> {
if layout.is_empty() || capture.0 == 0 || capture.1 == 0 {
return None;
}
let logical_w = layout
.iter()
.map(|monitor| monitor.x.saturating_add(monitor.width))
.max()?;
let logical_h = layout
.iter()
.map(|monitor| monitor.y.saturating_add(monitor.height))
.max()?;
if logical_w <= 0 || logical_h <= 0 {
return None;
}
let scale_x = f64::from(capture.0) / f64::from(logical_w);
let scale_y = f64::from(capture.1) / f64::from(logical_h);
if (scale_x - scale_y).abs() > 0.02 {
return None;
}
let (x, y, width, height) = bounds;
let centre_x = f64::from(x) + f64::from(width) / 2.0;
let centre_y = f64::from(y) + f64::from(height) / 2.0;
Some((
(centre_x * scale_x).round() as i32,
(centre_y * scale_y).round() as i32,
))
}
fn pixelflux_pointer_button(button: Option<&str>) -> Option<crate::remote_desktop::PointerButton> {
match button.unwrap_or("left").to_ascii_lowercase().as_str() {
"left" => Some(crate::remote_desktop::PointerButton::Left),
"right" => Some(crate::remote_desktop::PointerButton::Right),
"middle" => Some(crate::remote_desktop::PointerButton::Middle),
_ => None,
}
}
fn pixelflux_action_output(
action: &str,
result: anyhow::Result<()>,
received: Option<serde_json::Value>,
) -> ActionOutput {
match result {
Ok(()) => ActionOutput {
ok: true,
implemented: true,
action: action.to_string(),
message: "Action sent through pixelflux.".to_string(),
received,
},
Err(error) => ActionOutput {
ok: false,
implemented: true,
action: action.to_string(),
message: format!("pixelflux did not accept {action}: {error:#}"),
received,
},
}
}
#[derive(Debug)]
enum ClickTarget {
Coordinates(i32, i32),
ElementBounds {
node: Box<AccessibilityNode>,
center: (i32, i32),
},
ElementPoint {
node: Box<AccessibilityNode>,
point: (i32, i32),
},
PrimaryAction {
object_ref: String,
action_name: Option<String>,
action_index: i32,
},
}
#[derive(Debug, Clone, Copy)]
enum ElementResolvePurpose {
Click,
Action,
SetValue,
}
#[derive(Debug, Clone, Copy, Default)]
struct ElementSelector<'a> {
role: Option<&'a str>,
name: Option<&'a str>,
text: Option<&'a str>,
states: &'a [String],
}
impl ElementSelector<'_> {
fn is_empty(&self) -> bool {
[self.role, self.name, self.text]
.into_iter()
.all(|value| value.map(str::trim).is_none_or(str::is_empty))
&& self.states.iter().all(|value| value.trim().is_empty())
}
}
fn resolve_semantic_node(
nodes: &[AccessibilityNode],
selector: &ElementSelector<'_>,
purpose: ElementResolvePurpose,
) -> std::result::Result<AccessibilityNode, String> {
let mut matches = nodes
.iter()
.filter(|node| node_matches_selector(node, selector))
.collect::<Vec<_>>();
if matches.is_empty() {
return Err(format!(
"No cached accessibility node matched semantic selector {}. Call get_app_state first or pass element_index.",
describe_selector(selector)
));
}
if let Some(node) =
unique_preferred_node(&matches, |node| node_matches_resolve_purpose(node, purpose))
{
return Ok(node.clone());
}
let useful_matches = matches
.iter()
.copied()
.filter(|node| node_matches_resolve_purpose(node, purpose))
.collect::<Vec<_>>();
if !useful_matches.is_empty() {
matches = useful_matches;
}
if let Some(node) = unique_preferred_node(&matches, node_is_showing) {
return Ok(node.clone());
}
let visible_matches = matches
.iter()
.copied()
.filter(|node| node_is_showing(node))
.collect::<Vec<_>>();
if !visible_matches.is_empty() {
matches = visible_matches;
}
if matches.len() == 1 {
return Ok(matches[0].clone());
}
Err(format!(
"Semantic selector {} matched multiple cached nodes: {}. Pass element_index or add more selector fields.",
describe_selector(selector),
describe_matching_nodes(&matches),
))
}
fn unique_preferred_node<'a>(
nodes: &[&'a AccessibilityNode],
predicate: impl Fn(&AccessibilityNode) -> bool,
) -> Option<&'a AccessibilityNode> {
let mut matches = nodes.iter().copied().filter(|node| predicate(node));
let first = matches.next()?;
matches.next().is_none().then_some(first)
}
fn node_matches_selector(node: &AccessibilityNode, selector: &ElementSelector<'_>) -> bool {
selector
.role
.is_none_or(|role| normalized_contains(Some(node.role.as_str()), role))
&& selector
.name
.is_none_or(|name| normalized_contains(node.name.as_deref(), name))
&& selector.text.is_none_or(|text| {
normalized_contains(
node.text
.as_ref()
.and_then(|value| value.content.as_deref()),
text,
) || normalized_contains(node.name.as_deref(), text)
|| normalized_contains(node.description.as_deref(), text)
})
&& selector
.states
.iter()
.filter(|state| !state.trim().is_empty())
.all(|state| {
node.states
.iter()
.any(|node_state| normalized_equals(node_state, state))
})
}
fn node_matches_resolve_purpose(node: &AccessibilityNode, purpose: ElementResolvePurpose) -> bool {
match purpose {
ElementResolvePurpose::Click => {
node.bounds.as_ref().and_then(bounds_center).is_some()
|| primary_action_name(&node.actions).is_some()
}
ElementResolvePurpose::Action => !node.actions.is_empty(),
ElementResolvePurpose::SetValue => node.supports_editable_text || node.value.is_some(),
}
}
fn node_is_showing(node: &AccessibilityNode) -> bool {
node.states
.iter()
.any(|state| normalized_equals(state, "showing"))
&& node
.states
.iter()
.any(|state| normalized_equals(state, "visible"))
}
fn normalized_equals(actual: &str, expected: &str) -> bool {
normalize_text(actual) == normalize_text(expected)
}
fn normalized_contains(actual: Option<&str>, expected: &str) -> bool {
let expected = normalize_text(expected);
!expected.is_empty()
&& actual
.map(normalize_text)
.is_some_and(|actual| actual.contains(&expected))
}
fn normalize_text(value: &str) -> String {
value
.split_whitespace()
.collect::<Vec<_>>()
.join(" ")
.to_lowercase()
}
fn describe_selector(selector: &ElementSelector<'_>) -> String {
let mut parts = Vec::new();
if let Some(role) = selector.role.map(str::trim).filter(|role| !role.is_empty()) {
parts.push(format!("role={role:?}"));
}
if let Some(name) = selector.name.map(str::trim).filter(|name| !name.is_empty()) {
parts.push(format!("name={name:?}"));
}
if let Some(text) = selector.text.map(str::trim).filter(|text| !text.is_empty()) {
parts.push(format!("text={text:?}"));
}
let states = selector
.states
.iter()
.map(|state| state.trim())
.filter(|state| !state.is_empty())
.collect::<Vec<_>>();
if !states.is_empty() {
parts.push(format!("states={states:?}"));
}
if parts.is_empty() {
"<empty>".to_string()
} else {
parts.join(", ")
}
}
fn describe_matching_nodes(nodes: &[&AccessibilityNode]) -> String {
nodes
.iter()
.take(8)
.map(|node| {
format!(
"element_index {} role={:?} name={:?}",
node.index, node.role, node.name
)
})
.collect::<Vec<_>>()
.join("; ")
}
fn is_plain_left_click(button: Option<&str>, click_count: Option<u32>) -> bool {
let button = button.unwrap_or("left");
let click_count = click_count.unwrap_or(1);
matches!(button.to_ascii_lowercase().as_str(), "left" | "primary") && click_count == 1
}
fn click_equivalent_action(action: &AccessibilityAction) -> bool {
matches!(
action.name.to_ascii_lowercase().as_str(),
"click" | "press" | "toggle"
)
}
fn requested_or_primary_action(action: Option<&str>) -> &str {
match action.map(str::trim).filter(|value| !value.is_empty()) {
Some(action) => action,
None => "0",
}
}
fn primary_action(actions: &[AccessibilityAction]) -> Option<&AccessibilityAction> {
actions.first()
}
fn primary_action_name(actions: &[AccessibilityAction]) -> Option<String> {
primary_action(actions).map(|action| action.name.clone())
}
fn bounds_center(bounds: &Bounds) -> Option<(i32, i32)> {
if bounds.width <= 0 || bounds.height <= 0 {
return None;
}
if bounds.x <= i32::MIN / 2 || bounds.y <= i32::MIN / 2 {
return None;
}
Some((
bounds.x.checked_add(bounds.width / 2)?,
bounds.y.checked_add(bounds.height / 2)?,
))
}
/// Context-cost warning for an AT-SPI snapshot that no app target narrowed.
/// Silent when the tree failed (the failure message already explains) or when
/// any filter or pid match scoped the roots. When the caller did pass a target
/// and it still matched no AT-SPI root, telling them to pass a target would
/// only replay the same desktop-wide snapshot, so that case points at the
/// app's accessibility support instead.
fn unscoped_accessibility_tree_warning(
tree_scoped: bool,
tree_ok: bool,
target_requested: bool,
) -> Option<&'static str> {
if !tree_ok || tree_scoped {
return None;
}
Some(if target_requested {
"WARNING: the requested target matched no AT-SPI application root, so the accessibility tree covers the whole desktop and can flood context. The target app may expose no accessibility tree (Electron apps need --force-renderer-accessibility); check list_apps for its AT-SPI name and pass that as app_name_or_bundle_identifier, or lower max_nodes to bound the cost."
} else {
"WARNING: no app target scoped the accessibility tree, so it covers the whole desktop and can flood context. Pass app_name_or_bundle_identifier or a window target (window_id, pid, app_id, wm_class, title) to limit it."
})
}
/// Note appended when raw traversal hit max_nodes; the tree is incomplete.
fn truncated_accessibility_tree_note(truncated: bool) -> Option<&'static str> {
truncated.then_some(
"The node, depth, read, or time budget stopped traversal with unread elements left. To recover missing elements, scope to a narrower app or window target and raise max_nodes or max_depth (hard caps 10000 and 64). Embedded browser content can exhaust an app-scoped budget before its editor; temporarily hide that pane if a larger request still truncates. Lowering max_nodes would drop more.",
)
}
fn unmatched_accessibility_pid_note(
target_pid: Option<u32>,
root_pid: Option<u32>,
app_filter_matched: bool,
) -> Option<String> {
if root_pid.is_some() || app_filter_matched {
return None;
}
let pid = target_pid?;
Some(format!(
"No AT-SPI application root matched pid {pid}; the tree is empty. The app may have closed, may not expose accessibility, or may connect through a proxy. Run list_apps, then call get_app_state with app_name_or_bundle_identifier set to its accessibility name or root reference."
))
}
fn shell_only_accessibility_tree_note(
nodes: &[AccessibilityNode],
scoped: bool,
truncated: bool,
) -> Option<&'static str> {
if !scoped || truncated || nodes.is_empty() {
return None;
}
let mut menu_nodes = std::collections::HashSet::new();
nodes.iter().all(|node| {
let in_menu = node.parent_index.is_some_and(|parent| menu_nodes.contains(&parent));
let Some(menu) = accessibility_chrome_context(&node.role, in_menu) else {
return false;
};
if menu {
menu_nodes.insert(node.index);
}
let has_content_text = !menu && node.text.as_ref().and_then(|text| text.content.as_deref())
.is_some_and(|text| text.chars().any(|ch| ch != '\u{fffc}' && !ch.is_whitespace()));
!node.supports_editable_text && node.value.is_none() && !has_content_text
})
.then_some(
"AT-SPI exposes only window/menu chrome; no content controls were observed. If this is an Electron/Chromium app, relaunch it with --force-renderer-accessibility to expose renderer content. Screenshots can still be used for observation; this tree does not establish that the UI is empty.",
)
}
/// Bytes kept for an action result's other fields when it carries a tree.
const POST_ACTION_RESULT_RESERVE: usize = 4_000;
/// Bytes kept for the spill field and its note.
const SPILL_NOTE_RESERVE: usize = 1_000;
/// Inline tree budget once `rest` bytes of the result are spoken for.
fn tree_budget(rest: usize) -> usize {
crate::tree_spill::INLINE_RESULT_BYTES.saturating_sub(rest + SPILL_NOTE_RESERVE)
}
/// Keep `nodes` within the inline budget, spilling the whole tree to a private
/// file when it does not fit. Returns the spill and a note for the message.
fn fit_tree(
spiller: &crate::tree_spill::TreeSpiller,
tree: &mut TreeNodes,
budget: usize,
) -> (Option<crate::tree_spill::TreeSpill>, Option<String>) {
let total = tree.len();
let detail = tree.detail;
match spiller.fit(&mut tree.nodes, detail, budget) {
Ok(None) => (None, None),
Ok(Some(spill)) => {
let note = format!(
"The compacted tree has {total} nodes ({} KB), too large to return inline: the first {} are here and the whole tree is in {} (a JSON array with the same element_index values), or page through it with get_tree_page. Search that file for a role, name, or text instead of requesting the tree again; it lasts until the next snapshot or until this server exits.",
spill.bytes.div_ceil(1024),
spill.inline_node_count,
spill.path,
);
(Some(spill), Some(note))
}
Err((inline, error)) => (
None,
Some(format!(
"The compacted tree has {total} nodes, too large to return inline, and writing it to a file failed ({error}); only the first {inline} are here. Scope to a narrower target to see the rest."
)),
),
}
}
fn compact_accessibility_tree_with_note(
nodes: Vec<AccessibilityNode>,
scoped: bool,
truncated: bool,
) -> (Vec<AccessibilityNode>, Option<&'static str>) {
// Menu ancestry is observation metadata; compaction can remove its nodes.
let note = shell_only_accessibility_tree_note(&nodes, scoped, truncated);
(compact_accessibility_tree(nodes), note)
}
fn compact_accessibility_tree(nodes: Vec<AccessibilityNode>) -> Vec<AccessibilityNode> {
if nodes.is_empty() {
return nodes;
}
let keep = nodes
.iter()
.map(should_keep_accessibility_node)
.collect::<Vec<_>>();
let mut old_to_new = vec![None; nodes.len()];
let mut compacted = Vec::new();
for (old_index, node) in nodes.iter().enumerate() {
if !keep[old_index] {
continue;
}
let mut compacted_node = node.clone();
compacted_node.index = compacted.len() as u32;
compacted_node.parent_index = nearest_kept_parent(&keep, &nodes, old_index);
old_to_new[old_index] = Some(compacted_node.index);
compacted.push(compacted_node);
}
for node in &mut compacted {
node.parent_index = node
.parent_index
.and_then(|old_parent| old_to_new.get(old_parent as usize).copied().flatten());
}
let child_counts = compacted.iter().filter_map(|node| node.parent_index).fold(
vec![0_i32; compacted.len()],
|mut counts, parent_index| {
counts[parent_index as usize] += 1;
counts
},
);
for (index, node) in compacted.iter_mut().enumerate() {
node.child_count = child_counts[index];
}
compacted
}
fn nearest_kept_parent(
keep: &[bool],
nodes: &[AccessibilityNode],
old_index: usize,
) -> Option<u32> {
let mut parent = nodes[old_index].parent_index;
while let Some(parent_index) = parent {
let parent_usize = parent_index as usize;
if keep.get(parent_usize).copied().unwrap_or(false) {
return Some(parent_index);
}
parent = nodes.get(parent_usize).and_then(|node| node.parent_index);
}
None
}
/// The compacted tree cut to what an agent acts on or reads results from:
/// controls and nodes with a specific action, editable text or a value,
/// headings, status and live regions with their text, and app roots, windows
/// and dialogs. Removed containers are
/// collapsed: a kept node's parent_index names its nearest kept ancestor and
/// child_count counts kept children. Indices are not renumbered, so an
/// element_index means the same node as in the full compacted tree.
fn interactive_accessibility_tree(nodes: Vec<AccessibilityNode>) -> Vec<AccessibilityNode> {
let position: std::collections::HashMap<u32, usize> = nodes
.iter()
.enumerate()
.map(|(position, node)| (node.index, position))
.collect();
// Nodes come in traversal order, so a parent is classified before its children.
let mut in_live_region = vec![false; nodes.len()];
let mut keep = vec![false; nodes.len()];
for (index, node) in nodes.iter().enumerate() {
let below_live_region = node
.parent_index
.and_then(|parent| position.get(&parent))
.is_some_and(|&parent| parent < index && in_live_region[parent]);
in_live_region[index] = below_live_region || is_live_region_role(&node.role);
keep[index] = is_interactive_tree_node(node, below_live_region);
}
let kept_ancestor = |node: &AccessibilityNode| {
let mut parent = node.parent_index;
while let Some(index) = parent {
let position = *position.get(&index)?;
if keep[position] {
return Some(index);
}
parent = nodes[position].parent_index;
}
None
};
let parents: Vec<Option<u32>> = nodes.iter().map(kept_ancestor).collect();
let mut child_counts: std::collections::HashMap<u32, i32> = std::collections::HashMap::new();
for (parent, keep) in parents.iter().zip(&keep) {
if let (true, Some(parent)) = (*keep, parent) {
*child_counts.entry(*parent).or_default() += 1;
}
}
nodes
.into_iter()
.zip(parents)
.zip(keep)
.filter_map(|((mut node, parent), keep)| {
keep.then(|| {
node.parent_index = parent;
node.child_count = child_counts.get(&node.index).copied().unwrap_or_default();
node
})
})
.collect()
}
/// Roles that read out results: headings and status or live regions.
fn is_result_role(role: &str) -> bool {
matches!(
role,
"heading"
| "alert"
| "status bar"
| "statusbar"
| "notification"
| "log"
| "marquee"
| "timer"
| "info bar"
)
}
fn is_live_region_role(role: &str) -> bool {
role != "heading" && is_result_role(role)
}
/// Roles of controls an agent operates, whatever actions the toolkit lists.
fn is_control_role(role: &str) -> bool {
matches!(
role,
"push button"
| "button"
| "toggle button"
| "check box"
| "radio button"
| "link"
| "entry"
| "password text"
| "combo box"
| "menu item"
| "check menu item"
| "radio menu item"
| "page tab"
| "slider"
| "spin button"
| "switch"
)
}
/// Whether interactive_only keeps a node. `in_live_region` is true below a
/// status or live region, whose text is the result an agent reads.
fn is_interactive_tree_node(node: &AccessibilityNode, in_live_region: bool) -> bool {
let role = node.role.as_str();
// App roots and their windows (depth 0 and 1) frame everything else, and
// KWin element clicks map through the depth-1 window.
node.depth <= 1
|| is_actionable_accessibility_node(node)
|| is_control_role(role)
|| crate::atspi_tree::is_window_role(role)
|| ((is_result_role(role) || in_live_region) && has_meaningful_node_copy(node))
// A live region whose text sits in its children frames that text.
|| (is_live_region_role(role) && node.child_count > 0)
}
/// Window title that narrows an app root holding several windows to one, when
/// the caller targeted a window by title or id. The resolved window's full
/// title is preferred over the caller's substring.
fn accessibility_window_title(
params: &GetAppStateParams,
window: Option<&WindowInfo>,
) -> Option<String> {
targeted_window_title(params.window_id, params.title.as_deref(), window)
}
/// The window a post-action tree is narrowed to. An action that names a
/// window by window_id or title uses it. One that names none keeps the window
/// the last snapshot of the same app was narrowed to, so
/// get_app_state(title=...) followed by perform_action(include_tree=true)
/// does not widen to every window of the app again.
fn post_action_window_title(
target: Option<&WindowTarget>,
resolved_window: Option<&WindowInfo>,
target_pid: Option<u32>,
cached_window_scope: Option<(String, Option<u32>)>,
) -> Option<String> {
let targeted = target.and_then(|target| {
targeted_window_title(target.window_id, target.title.as_deref(), resolved_window)
});
if targeted.is_some()
|| target.is_some_and(|target| target.window_id.is_some() || target.title.is_some())
{
return targeted;
}
cached_window_scope
.filter(|(_, cached_pid)| cached_pid.is_some() && *cached_pid == target_pid)
.map(|(scope, _)| scope)
}
/// accessibility_window_title for a target given as its window_id and title.
fn targeted_window_title(
window_id: Option<u64>,
title: Option<&str>,
window: Option<&WindowInfo>,
) -> Option<String> {
let requested_title = title.map(str::trim).filter(|title| !title.is_empty());
if window_id.is_none() && requested_title.is_none() {
return None;
}
window
.and_then(|window| window.title.as_deref())
.map(str::trim)
.filter(|title| !title.is_empty())
.or(requested_title)
.map(str::to_string)
}
fn should_keep_accessibility_node(node: &AccessibilityNode) -> bool {
if node.depth <= 1 {
return true;
}
if is_actionable_accessibility_node(node) || has_meaningful_node_copy(node) {
return true;
}
// interactive_only runs on this tree, so keep what it keeps: controls
// whatever their actions, and a live region whose text sits in its children.
if is_control_role(&node.role) || (is_live_region_role(&node.role) && node.child_count > 0) {
return true;
}
matches!(
node.role.as_str(),
"page tab" | "menu item" | "menu" | "list item" | "tree item"
) && !is_sentinel_or_missing_bounds(node.bounds.as_ref())
}
/// Actions Chromium lists on every element, whether or not it does anything:
/// a generic default action, the context menu, and a click forwarded to the
/// nearest clickable ancestor.
const GENERIC_ACTIONS: &[&str] = &[
"doDefault",
"showContextMenu",
"clickAncestor",
"click-ancestor",
];
fn is_actionable_accessibility_node(node: &AccessibilityNode) -> bool {
node.actions
.iter()
.any(|action| !GENERIC_ACTIONS.contains(&action.name.as_str()))
|| node.supports_editable_text
|| node.value.is_some()
}
fn has_meaningful_node_copy(node: &AccessibilityNode) -> bool {
has_non_empty_text(node.name.as_deref())
|| has_non_empty_text(node.description.as_deref())
|| has_non_empty_text(node.text.as_ref().and_then(|text| text.content.as_deref()))
}
/// Readable copy: not empty, whitespace, or only the U+FFFC characters a
/// container's text holds for its children.
fn has_non_empty_text(value: Option<&str>) -> bool {
value.is_some_and(|value| !is_placeholder_text(value))
}
fn is_sentinel_or_missing_bounds(bounds: Option<&Bounds>) -> bool {
bounds.is_none()
}
fn select_accessibility_app_id_filter(
apps: Option<&[AccessibleAppSummary]>,
app_id: &str,
) -> String {
let leaf = crate::windows::app_id_leaf(app_id);
let Some(apps) = apps.filter(|_| !leaf.is_empty()) else {
return UNMATCHABLE_ACCESSIBILITY_APP_FILTER.to_string();
};
let mut matches = apps.iter().filter(|app| {
app.name
.as_deref()
.is_some_and(|name| crate::windows::app_id_leaf(name).eq_ignore_ascii_case(leaf))
});
let Some(first) = matches.next() else {
return UNMATCHABLE_ACCESSIBILITY_APP_FILTER.to_string();
};
if matches.next().is_some() {
return UNMATCHABLE_ACCESSIBILITY_APP_FILTER.to_string();
}
first.object_ref.clone()
}
fn accessibility_snapshot_target_pid(
app_filter: Option<&str>,
target_pid: Option<u32>,
) -> Option<u32> {
// select_roots normally falls back to PID-only matches. A rejected app ID
// must not regain that window's tree through this fallback.
if app_filter == Some(UNMATCHABLE_ACCESSIBILITY_APP_FILTER) {
None
} else {
target_pid
}
}
fn accessibility_filter_candidates(window_context: Option<&WindowInfo>) -> Vec<String> {
let Some(window) = window_context else {
return Vec::new();
};
let mut candidates = Vec::new();
push_candidate(&mut candidates, window.title.as_deref());
push_candidate(&mut candidates, window.wm_class.as_deref());
if let Some(app_id) = trimmed_nonempty(window.app_id.as_deref()) {
if !app_id.starts_with("window:") {
push_candidate(&mut candidates, Some(app_id));
if let Some(stripped) = app_id.strip_suffix(".desktop") {
push_candidate(&mut candidates, Some(stripped));
let normalized = stripped.replace(['-', '_', '.'], " ");
push_candidate(&mut candidates, Some(normalized.as_str()));
} else {
let normalized = app_id.replace(['-', '_', '.'], " ");
push_candidate(&mut candidates, Some(normalized.as_str()));
}
push_candidate(&mut candidates, Some(crate::windows::app_id_leaf(app_id)));
}
}
candidates
}
fn push_candidate(candidates: &mut Vec<String>, value: Option<&str>) {
let Some(value) = trimmed_nonempty(value) else {
return;
};
if !candidates.iter().any(|candidate| candidate == value) {
candidates.push(value.to_string());
}
}
fn trimmed_nonempty(value: Option<&str>) -> Option<&str> {
value.map(str::trim).filter(|value| !value.is_empty())
}
fn env_contains(key: &str, needle: &str) -> bool {
env::var(key)
.ok()
.is_some_and(|value| value.to_ascii_lowercase().contains(needle))
}
/// True when an environment variable is set to `"1"` (an explicit on switch).
fn env_flag_enabled(key: &str) -> bool {
env::var(key).ok().as_deref() == Some("1")
}
fn env_var_non_empty(key: &str) -> bool {
env::var(key)
.map(|value| !value.trim().is_empty())
.unwrap_or(false)
}
/// Return the base64 payload of a `data:` URL (or the original string if bare).
fn data_url_payload(data_url: &str) -> String {
data_url
.split_once(',')
.map(|(_, payload)| payload)
.unwrap_or(data_url)
.to_string()
}
fn session_is_wayland(session_type: Option<&str>, wayland_display: Option<&str>) -> bool {
match session_type
.map(str::trim)
.filter(|value| !value.is_empty())
{
Some(value) => value.eq_ignore_ascii_case("wayland"),
None => wayland_display.is_some_and(|value| !value.trim().is_empty()),
}
}
fn native_x11_xdotool_pointer_session(
session_type: Option<&str>,
wayland_display: Option<&str>,
) -> bool {
session_type.is_some_and(|value| value.trim().eq_ignore_ascii_case("x11"))
&& wayland_display.is_none_or(|value| value.trim().is_empty())
}
fn prefer_xdotool_pointer(
force_ydotool: bool,
session_type: Option<&str>,
display_available: bool,
wayland_display: Option<&str>,
) -> bool {
!force_ydotool
&& native_x11_xdotool_pointer_session(session_type, wayland_display)
&& display_available
}
fn prefer_xdotool_keyboard(
force_ydotool: bool,
force_xdotool: bool,
is_wayland: bool,
display_available: bool,
xdotool_available: bool,
) -> bool {
!force_ydotool && display_available && xdotool_available && (force_xdotool || !is_wayland)
}
fn prepare_app_state_screenshot(
mut raw: RawScreenshotCapture,
crop: Option<(i32, i32, u32, u32)>,
target_requested: bool,
options: ScreenshotPayloadOptions,
) -> Result<ScreenshotCapture> {
if target_requested && crop.is_none() {
anyhow::bail!(
"targeted screenshot requires a resolved window; refusing to return the full desktop"
);
}
if let Some(rect) = crop {
let (x, y, width, height) = clip_capture_rect(rect, raw.width, raw.height)?;
let (bytes, width, height) = crop_png(&raw.bytes, x, y, width, height)
.map_err(|error| anyhow::anyhow!("targeted screenshot crop failed: {error}"))?;
raw = RawScreenshotCapture {
mime_type: raw.mime_type,
bytes,
source: raw.source,
width,
height,
};
}
prepare_screenshot_payload(raw, options)
}
/// Whether targeted keyboard input goes through Hyprland background mode,
/// which never raises or focuses the target.
fn hyprland_background_applies(target: &WindowTarget) -> bool {
crate::hyprland_input::background_enabled()
&& target.has_target()
&& env::var("XDG_CURRENT_DESKTOP").is_ok_and(|desktop| {
desktop
.split(':')
.any(|part| part.eq_ignore_ascii_case("hyprland"))
})
}
/// What a targeted keyboard action did to its window, for the post-action
/// window crop.
fn keyboard_target_raised(target: &WindowTarget) -> TargetRaised {
if hyprland_background_applies(target) {
TargetRaised::No
} else {
TargetRaised::Yes
}
}
/// The window a post-action tree can be recorded as taken for, so a
/// follow-up action on that window passes the ownership check. A window with
/// a pid needs the tree to come from that pid. A window without one (i3
/// without xprop, COSMIC) needs the tree narrowed to the frame its title names; an
/// app-wide tree may hold another app's nodes, and recording it would let
/// them pass. A focused-window guess is never passed in.
///
/// A target with no window_id or title narrows through the previous
/// snapshot's window scope (`cached`). That tree is the cached window's only
/// when it came from the cached pid and matched the very frame the cache
/// named; then it inherits the cached window. Otherwise it is recorded as no
/// window, never as the window the target resolved to, which can be another
/// window of the app.
fn post_action_tree_window(
window: Option<&WindowInfo>,
root_pid: Option<u32>,
window_scope: Option<&str>,
cached: Option<&CachedTreeWindow>,
) -> Option<(u64, String)> {
if let Some(cached) = cached {
let same_tree = root_pid.is_some()
&& root_pid == cached.pid
&& window_scope == Some(cached.scope.as_str());
return cached.window.clone().filter(|_| same_tree);
}
window
.filter(|window| match window.pid {
Some(pid) => root_pid == Some(pid),
None => window_scope.is_some(),
})
.map(|window| (window.window_id, window.backend.clone()))
}
/// A crop of a targeted window the action did not raise could show whatever
/// covers it, so it needs the window to be focused already.
fn ensure_post_action_crop_shows_target(
raised: TargetRaised,
target: Option<&WindowTarget>,
window: &WindowInfo,
) -> Result<()> {
if raised == TargetRaised::No && target.is_some_and(WindowTarget::has_target) && !window.focused
{
anyhow::bail!(
"targeted post-action screenshot needs the target window to be focused, and this action does not raise it; use screenshot_after=\"screen\" or the screenshot tool"
);
}
Ok(())
}
fn ensure_readonly_screenshot_target_is_visible(window: &WindowInfo) -> Result<()> {
if window.hidden {
anyhow::bail!("targeted get_app_state screenshot requires a visible, unminimized window");
}
if !window.focused {
anyhow::bail!(
"targeted get_app_state screenshot requires the window to already be focused; use the screenshot tool to raise it before capture"
);
}
Ok(())
}
#[derive(Debug, Clone, Copy)]
struct WindowCoordinateMap {
capture_rect: (i32, i32, u32, u32),
full_capture_rect: (i32, i32, u32, u32),
portal_rect: Option<(i32, i32, u32, u32)>,
}
impl WindowCoordinateMap {
fn portal_point(&self, capture_x: i32, capture_y: i32) -> Option<(i32, i32)> {
let (portal_x, portal_y, portal_width, portal_height) = self.portal_rect?;
let (full_x, full_y, full_width, full_height) = self.full_capture_rect;
Some((
map_coordinate_between_rects(capture_x, full_x, full_width, portal_x, portal_width),
map_coordinate_between_rects(capture_y, full_y, full_height, portal_y, portal_height),
))
}
}
fn map_coordinate_between_rects(
value: i32,
source_origin: i32,
source_size: u32,
target_origin: i32,
target_size: u32,
) -> i32 {
let offset = i64::from(value) - i64::from(source_origin);
let scaled = offset.saturating_mul(i64::from(target_size)) / i64::from(source_size.max(1));
(i64::from(target_origin) + scaled).clamp(i64::from(i32::MIN), i64::from(i32::MAX)) as i32
}
fn logical_window_crop_rect(
bounds: &crate::windowing::WindowBounds,
monitors: &[(i32, i32, i32, i32)],
capture_width: u32,
capture_height: u32,
) -> Result<(i32, i32, u32, u32)> {
let mut monitors = monitors
.iter()
.filter(|(_, _, width, height)| *width > 0 && *height > 0);
let first = monitors
.next()
.ok_or_else(|| anyhow::anyhow!("desktop returned no usable monitor geometry"))?;
let (mut min_x, mut min_y) = (i64::from(first.0), i64::from(first.1));
let (mut max_x, mut max_y) = (
i64::from(first.0) + i64::from(first.2),
i64::from(first.1) + i64::from(first.3),
);
for (x, y, width, height) in monitors {
min_x = min_x.min(i64::from(*x));
min_y = min_y.min(i64::from(*y));
max_x = max_x.max(i64::from(*x) + i64::from(*width));
max_y = max_y.max(i64::from(*y) + i64::from(*height));
}
let logical_width = max_x - min_x;
let logical_height = max_y - min_y;
if logical_width <= 0 || logical_height <= 0 || capture_width == 0 || capture_height == 0 {
anyhow::bail!("screenshot or monitor geometry is empty");
}
let scale_x = f64::from(capture_width) / logical_width as f64;
let scale_y = f64::from(capture_height) / logical_height as f64;
if !scale_x.is_finite() || !scale_y.is_finite() || (scale_x - scale_y).abs() > 0.01 {
anyhow::bail!(
"captured desktop {}x{} does not have a uniform scale relative to the logical monitor layout {}x{}",
capture_width,
capture_height,
logical_width,
logical_height
);
}
let x = i64::from(
bounds
.x
.ok_or_else(|| anyhow::anyhow!("window x is unavailable"))?,
);
let y = i64::from(
bounds
.y
.ok_or_else(|| anyhow::anyhow!("window y is unavailable"))?,
);
if bounds.width == 0 || bounds.height == 0 {
anyhow::bail!("window bounds are empty");
}
let left = (((x - min_x) as f64) * scale_x).floor() as i64;
let top = (((y - min_y) as f64) * scale_y).floor() as i64;
let right = (((x + i64::from(bounds.width) - min_x) as f64) * scale_x).ceil() as i64;
let bottom = (((y + i64::from(bounds.height) - min_y) as f64) * scale_y).ceil() as i64;
let width = u32::try_from(right - left)
.map_err(|_| anyhow::anyhow!("scaled window width is invalid"))?;
let height = u32::try_from(bottom - top)
.map_err(|_| anyhow::anyhow!("scaled window height is invalid"))?;
let left = i32::try_from(left).map_err(|_| anyhow::anyhow!("scaled window x is invalid"))?;
let top = i32::try_from(top).map_err(|_| anyhow::anyhow!("scaled window y is invalid"))?;
Ok((left, top, width, height))
}
/// Scale logical monitor rectangles onto a capture of their bounding box, as
/// (left, top, right, bottom) capture pixels. Edges round outward, so a pixel
/// a monitor partly covers counts as on it. None for an empty or invalid
/// layout, or a capture that is not the bounding box at one scale: the
/// layout then describes another space.
fn monitor_rects_in_capture(
monitors: &[(i32, i32, i32, i32)],
(capture_width, capture_height): (u32, u32),
) -> Option<Vec<(i64, i64, i64, i64)>> {
if capture_width == 0
|| capture_height == 0
|| monitors
.iter()
.any(|(_, _, width, height)| *width <= 0 || *height <= 0)
{
return None;
}
let min_x = monitors.iter().map(|m| i64::from(m.0)).min()?;
let min_y = monitors.iter().map(|m| i64::from(m.1)).min()?;
let max_x = monitors
.iter()
.map(|m| i64::from(m.0) + i64::from(m.2))
.max()?;
let max_y = monitors
.iter()
.map(|m| i64::from(m.1) + i64::from(m.3))
.max()?;
let scale_x = f64::from(capture_width) / (max_x - min_x) as f64;
let scale_y = f64::from(capture_height) / (max_y - min_y) as f64;
if !scale_x.is_finite() || !scale_y.is_finite() || (scale_x - scale_y).abs() > 0.01 {
return None;
}
Some(
monitors
.iter()
.map(|&(x, y, width, height)| {
let (x, y) = (i64::from(x) - min_x, i64::from(y) - min_y);
(
(x as f64 * scale_x).floor() as i64,
(y as f64 * scale_y).floor() as i64,
((x + i64::from(width)) as f64 * scale_x).ceil() as i64,
((y + i64::from(height)) as f64 * scale_y).ceil() as i64,
)
})
.collect(),
)
}
/// X11 monitor rectangles as (left, top, right, bottom) root pixels, the
/// space of an X11 capture. They are not rebased or scaled: the root can be
/// larger than the monitors' bounding box. None for an empty or invalid
/// layout.
fn root_monitor_rects(monitors: &[(i32, i32, i32, i32)]) -> Option<Vec<(i64, i64, i64, i64)>> {
if monitors.is_empty()
|| monitors
.iter()
.any(|(_, _, width, height)| *width <= 0 || *height <= 0)
{
return None;
}
Some(
monitors
.iter()
.map(|&(x, y, width, height)| {
let (x, y) = (i64::from(x), i64::from(y));
(x, y, x + i64::from(width), y + i64::from(height))
})
.collect(),
)
}
fn point_on_capture_monitor(monitors: &[(i64, i64, i64, i64)], x: i32, y: i32) -> bool {
let (x, y) = (i64::from(x), i64::from(y));
monitors
.iter()
.any(|&(left, top, right, bottom)| x >= left && x < right && y >= top && y < bottom)
}
fn clip_capture_rect(
(x, y, width, height): (i32, i32, u32, u32),
capture_width: u32,
capture_height: u32,
) -> Result<(i32, i32, u32, u32)> {
let left = i64::from(x).max(0);
let top = i64::from(y).max(0);
let right = (i64::from(x) + i64::from(width)).min(i64::from(capture_width));
let bottom = (i64::from(y) + i64::from(height)).min(i64::from(capture_height));
if right <= left || bottom <= top {
anyhow::bail!(
"targeted screenshot window is outside the captured desktop; refusing to return the full desktop"
);
}
Ok((
i32::try_from(left).map_err(|_| anyhow::anyhow!("capture crop x is invalid"))?,
i32::try_from(top).map_err(|_| anyhow::anyhow!("capture crop y is invalid"))?,
u32::try_from(right - left)
.map_err(|_| anyhow::anyhow!("capture crop width is invalid"))?,
u32::try_from(bottom - top)
.map_err(|_| anyhow::anyhow!("capture crop height is invalid"))?,
))
}
/// Convert a window's bounds into a crop rectangle, if it has a usable origin
/// and non-zero size.
fn window_crop_rect(bounds: &crate::windowing::WindowBounds) -> Option<(i32, i32, u32, u32)> {
let x = bounds.x?;
let y = bounds.y?;
if bounds.width == 0 || bounds.height == 0 {
return None;
}
Some((x, y, bounds.width, bounds.height))
}
fn validate_click_coordinate_space(params: &ClickParams) -> std::result::Result<(), String> {
if params.coordinate_space == ClickCoordinateSpace::Capture {
return Ok(());
}
if params.relative == Some(true) {
return Err(
"AT-SPI coordinate clicks cannot use relative screenshot coordinates.".to_string(),
);
}
if params.x.zip(params.y).is_none() {
return Err("AT-SPI coordinate clicks require both x and y.".to_string());
}
if params.element_index.is_none() && params.selector().is_empty() {
return Err(
"AT-SPI coordinate clicks require element_index or a semantic selector to identify the window geometry."
.to_string(),
);
}
Ok(())
}
fn apply_window_relative_click_coordinates(
params: &mut ClickParams,
capture_rect: (i32, i32, u32, u32),
) -> std::result::Result<(), String> {
let (relative_x, relative_y) = params
.x
.zip(params.y)
.ok_or_else(|| "Relative coordinate clicks require both x and y.".to_string())?;
let (origin_x, origin_y, width, height) = capture_rect;
if width == 0 || height == 0 {
return Err(
"Relative coordinate clicks require non-empty target-window bounds.".to_string(),
);
}
if relative_x < 0 || relative_y < 0 {
return Err("Relative click coordinates must be inside target-window bounds.".to_string());
}
if relative_x as u32 >= width || relative_y as u32 >= height {
return Err("Relative click coordinates must be inside target-window bounds.".to_string());
}
let x = origin_x
.checked_add(relative_x)
.ok_or_else(|| "Relative click x coordinate overflowed.".to_string())?;
let y = origin_y
.checked_add(relative_y)
.ok_or_else(|| "Relative click y coordinate overflowed.".to_string())?;
params.x = Some(x);
params.y = Some(y);
Ok(())
}
/// Point a window-targeted scroll at the centre of the resolved window when
/// the caller supplied no coordinates. Without this the wheel events land on
/// whatever is under the current pointer position.
fn apply_window_center_scroll_point(
params: &mut ScrollParams,
capture_rect: (i32, i32, u32, u32),
) -> std::result::Result<(), String> {
let (origin_x, origin_y, width, height) = capture_rect;
if width == 0 || height == 0 {
return Err(
"Window-targeted scroll requires non-empty target-window bounds; pass x/y explicitly."
.to_string(),
);
}
params.x = Some(origin_x.saturating_add((width / 2) as i32));
params.y = Some(origin_y.saturating_add((height / 2) as i32));
Ok(())
}
fn apply_window_relative_scroll_coordinates(
params: &mut ScrollParams,
capture_rect: (i32, i32, u32, u32),
) -> std::result::Result<(), String> {
let (relative_x, relative_y) = params
.x
.zip(params.y)
.ok_or_else(|| "Relative scroll coordinates require both x and y.".to_string())?;
let (origin_x, origin_y, width, height) = capture_rect;
if width == 0 || height == 0 {
return Err(
"Relative scroll coordinates require non-empty target-window bounds.".to_string(),
);
}
if relative_x < 0 || relative_y < 0 || relative_x as u32 >= width || relative_y as u32 >= height
{
return Err("Relative scroll coordinates must be inside target-window bounds.".to_string());
}
params.x = Some(origin_x.saturating_add(relative_x));
params.y = Some(origin_y.saturating_add(relative_y));
Ok(())
}
/// Crop a PNG image to `(x, y, w, h)` (clamped to the image), returning the
/// re-encoded PNG and the actual cropped dimensions.
fn crop_png(
raw: &[u8],
x: i32,
y: i32,
w: u32,
h: u32,
) -> std::result::Result<(Vec<u8>, u32, u32), String> {
use std::io::Cursor;
let img = image::load_from_memory_with_format(raw, image::ImageFormat::Png)
.map_err(|e| format!("decode png: {e}"))?;
let (iw, ih) = (img.width(), img.height());
let x = x.max(0) as u32;
let y = y.max(0) as u32;
if x >= iw || y >= ih {
return Err("crop origin outside image".into());
}
let w = w.min(iw - x);
let h = h.min(ih - y);
let sub = img.crop_imm(x, y, w, h);
let mut out = Vec::new();
sub.write_to(&mut Cursor::new(&mut out), image::ImageFormat::Png)
.map_err(|e| format!("encode png: {e}"))?;
Ok((out, w, h))
}
fn action_result(
action: &str,
result: std::result::Result<Vec<Output>, String>,
received: Option<serde_json::Value>,
) -> ActionOutput {
match result {
Ok(_) => ActionOutput {
ok: true,
implemented: true,
action: action.to_string(),
message: if action == "type_text" {
"Action sent through ydotool using US physical key positions. The active keyboard layout can change the resulting text; verify the field contents."
} else {
"Action sent through ydotool."
}
.to_string(),
received,
},
Err(message) => ActionOutput {
ok: false,
implemented: true,
action: action.to_string(),
message,
received,
},
}
}
fn unsafe_pointer_fallback(action: &str, received: Option<serde_json::Value>) -> ActionOutput {
ActionOutput {
ok: false,
implemented: true,
action: action.to_string(),
message: "Coordinate input was not sent: ydotool cannot position the pointer accurately on Wayland. Enable the uinput absolute pointer with working screenshot capture, or use COMPUTER_USE_LINUX_FORCE_PORTAL_POINTER=1 with an approved desktop portal session. Remove COMPUTER_USE_LINUX_FORCE_YDOTOOL_POINTER if set.".to_string(),
received,
}
}
fn portal_start_error(
action: &str,
error: anyhow::Error,
received: Option<serde_json::Value>,
) -> ActionOutput {
ActionOutput {
ok: false,
implemented: true,
action: action.to_string(),
message: format!("Input was not sent because the remote desktop portal could not start; input was not replayed through ydotool: {error:#}"),
received,
}
}
fn abs_pointer_error(
action: &str,
error: String,
received: Option<serde_json::Value>,
) -> ActionOutput {
ActionOutput {
ok: false,
implemented: true,
action: action.to_string(),
message: format!("Absolute pointer {action} may have started before it failed; input was not replayed through another backend: {error}"),
received,
}
}
/// Why the uinput pointer was not used, so the action was not sent.
#[derive(Debug, PartialEq)]
struct AbsPointerFailure {
detail: String,
/// The stale device was dropped, so the next call starts without one.
device_dropped: bool,
}
impl AbsPointerFailure {
fn lock_poisoned() -> Self {
Self {
detail: "absolute pointer lock was poisoned".to_string(),
device_dropped: false,
}
}
/// What the next call does once the stale device is gone. `retry` names
/// that call; `route` is [`ComputerUseLinux::pointer_retry_route`].
fn retry_note(&self, retry: &str, route: &str) -> String {
if !self.device_dropped {
return String::new();
}
format!(
" The stale device was removed. {retry} sizes a new device to a fresh screenshot; if that fails, {route}."
)
}
}
fn abs_pointer_not_sent(
action: &str,
failure: &AbsPointerFailure,
retry_route: &str,
received: Option<serde_json::Value>,
) -> ActionOutput {
ActionOutput {
ok: false,
implemented: true,
action: action.to_string(),
message: format!(
"Absolute pointer {action} was not sent: {}; input was not replayed through another backend.{}",
failure.detail,
failure.retry_note("A retry", retry_route)
),
received,
}
}
/// A KWin element click whose mapping worked but whose uinput pointer could
/// not be used.
fn kwin_abs_pointer_not_sent(
element_index: u32,
failure: &AbsPointerFailure,
retry_route: &str,
received: Option<serde_json::Value>,
) -> ActionOutput {
ActionOutput {
ok: false,
implemented: true,
action: "click".to_string(),
message: format!(
"click was not sent: KWin mapped element_index {element_index}, but the uinput absolute pointer could not be used: {}. No pointer event was emitted.{}",
failure.detail,
failure.retry_note(
&format!("A retry of element_index {element_index}"),
retry_route
)
),
received,
}
}
fn abs_pointer_result(
action: &str,
result: std::result::Result<crate::abs_pointer::PointerLanding, String>,
received: Option<serde_json::Value>,
) -> ActionOutput {
match result {
Ok(landing) => with_notes(
ActionOutput {
ok: true,
implemented: true,
action: action.to_string(),
message: if action == "scroll" {
"Pointer positioned and wheel sent through the uinput absolute pointer."
.to_string()
} else {
"Action sent through the uinput absolute pointer.".to_string()
},
received,
},
abs_pointer_clamp_note("coordinate", landing),
),
Err(error) => abs_pointer_error(action, error, received),
}
}
/// Append the KWin mapping to a successful element click. The message
/// already names the backend and holds its clamp or off-screen notes.
fn kwin_element_click_dispatch_output(
mut output: ActionOutput,
kwin_click: Option<PreparedKwinElementClick>,
) -> ActionOutput {
if output.ok {
if let Some(kwin_click) = kwin_click {
output.message = format!(
"{} KWin element coordinate mapping: capture {},{}; portal logical {},{}. Dispatch only: UI activation was not verified.",
output.message,
kwin_click.point.capture.0,
kwin_click.point.capture.1,
kwin_click.point.logical.0,
kwin_click.point.logical.1
);
}
}
output
}
fn portal_action_error(
action: &str,
error: anyhow::Error,
received: Option<serde_json::Value>,
) -> ActionOutput {
ActionOutput {
ok: false,
implemented: true,
action: action.to_string(),
message: format!(
"Remote desktop portal {action} may have started before it failed; input was not replayed through another backend: {error:#}"
),
received,
}
}
fn portal_coordinate_error(action: &str, received: Option<serde_json::Value>) -> ActionOutput {
ActionOutput {
ok: false,
implemented: true,
action: action.to_string(),
message: format!(
"Remote desktop portal {action} was not sent because the coordinate could not be mapped safely to the complete shared desktop; input was not replayed through another backend."
),
received,
}
}
fn action_result_with_focus(
action: &str,
result: std::result::Result<Vec<Output>, String>,
received: Option<serde_json::Value>,
focus: Option<WindowFocusResult>,
) -> ActionOutput {
with_focus_context(action_result(action, result, received), focus)
}
fn successful_action_with_focus(
action: &str,
message: &str,
received: Option<serde_json::Value>,
focus: Option<WindowFocusResult>,
) -> ActionOutput {
with_focus_context(
ActionOutput {
ok: true,
implemented: true,
action: action.to_string(),
message: message.to_string(),
received,
},
focus,
)
}
fn with_focus_context(mut output: ActionOutput, focus: Option<WindowFocusResult>) -> ActionOutput {
if output.ok {
if let Some(focus) = focus {
let verification = if focus.exact_window_focused {
"exact window-focus"
} else {
"app-level focus"
};
output.message = format!(
"{} Target window_id {} was focused with {verification} verification before input.",
output.message, focus.requested_window.window_id,
);
}
}
output
}
/// Exact window focus alone cannot rule out lost input. Only a complete,
/// scoped shell-only tree explains the missing focus as unavailable content.
fn focus_probe_feedback(
probe: &FocusProbe,
expects_editable: bool,
exact_window_focused: bool,
) -> String {
match probe {
FocusProbe::Found(element) => describe_focused_element(element, expects_editable),
FocusProbe::ShellOnly if exact_window_focused => "Focused-element feedback unavailable: window focus was verified before input, but AT-SPI exposes only window/menu chrome in the target app. Field contents were not verified. Electron/Chromium apps may need --force-renderer-accessibility.".to_string(),
FocusProbe::ShellOnly => "WARNING: AT-SPI exposes only window/menu chrome and no focused element in the target app; the input may have landed nowhere.".to_string(),
FocusProbe::NoneFocused => "WARNING: AT-SPI reports no focused element in the target app; the input may have landed nowhere.".to_string(),
FocusProbe::Incomplete => "Focused-element feedback unavailable: the focused element was not reached within the AT-SPI search limits, so the input could not be verified. Its absence here does not mean the input was lost.".to_string(),
FocusProbe::NoAccessibleApp => "Focused-element feedback unavailable: the target app exposes no AT-SPI tree, so the input could not be verified. Electron apps need --force-renderer-accessibility.".to_string(),
}
}
fn node_target_mismatch_message(element_index: u32, owner_pid: u32, target_pid: u32) -> String {
format!(
"element_index {element_index} comes from the latest get_app_state snapshot of pid {owner_pid}, not the target pid {target_pid}. Call get_app_state for the target app and use an index from that result."
)
}
fn node_window_mismatch_message(
element_index: u32,
snapshot_window: u64,
target_window: u64,
) -> String {
format!(
"element_index {element_index} comes from the latest get_app_state snapshot of window_id {snapshot_window}, not the target window_id {target_window}. Call get_app_state for the target window and use an index from that result."
)
}
fn node_window_unverified_message(element_index: u32, target_window: u64) -> String {
format!(
"Did not act because element_index {element_index} could not be tied to target window_id {target_window}: the window backend reports no pid for it and the latest get_app_state snapshot was not taken for that window. Pass pid, or call get_app_state for that window and use an index from that result."
)
}
fn node_window_unrecorded_message(element_index: u32, target_window: u64) -> String {
format!(
"Did not act because element_index {element_index} could not be tied to target window_id {target_window}: the latest get_app_state snapshot was narrowed to one window of this app, and the window it recorded, if any, cannot be compared with the target (none was recorded, or another window backend listed it). Call get_app_state for the target window and use an index from that result."
)
}
fn identifier_target_mismatch_message(owner_pid: u32, target_pid: u32) -> String {
format!(
"element_identifier belongs to pid {owner_pid}, not the target pid {target_pid}. Use an element_identifier from the target app, or drop the target."
)
}
/// Refuse an element_identifier whose bus owner is known and is not the
/// target pid. An unknown owner or target proceeds: the identifier names the
/// object itself.
fn check_identifier_owner(
owner_pid: Option<u32>,
target_pid: Option<u32>,
) -> std::result::Result<(), String> {
match (owner_pid, target_pid) {
(Some(owner_pid), Some(target_pid)) if owner_pid != target_pid => {
Err(identifier_target_mismatch_message(owner_pid, target_pid))
}
_ => Ok(()),
}
}
/// Resolve a window-style element action target against a window listing.
///
/// When the window listing itself is unavailable (GNOME without the Shell
/// extension) and the call also passed a pid, the pid still proves
/// ownership, so the check falls back to it. Without a pid it fails closed.
fn element_action_owner_from(
windows: Result<Vec<WindowInfo>>,
target: &WindowTarget,
) -> std::result::Result<ElementOwnerTarget, String> {
if windows.is_err() && target.pid.is_some() {
return Ok(ElementOwnerTarget::pid(target.pid));
}
windows
.and_then(|windows| {
let exact = names_one_window(&windows, target);
resolve_window_target(&windows, target)
.map(|window| ElementOwnerTarget::window(window, target.pid, exact))
})
.map_err(|error| {
let error = format!("{error:#}");
match window_permission_hint(&error) {
Some(hint) => format!(
"Did not act because the target window could not be resolved: {error}. {hint}"
),
None => {
format!("Did not act because the target window could not be resolved: {error}")
}
}
})
}
/// Whether `target` names exactly one window of the listing: a `window_id`,
/// or a selector that matches a single window on its own.
fn names_one_window(windows: &[WindowInfo], target: &WindowTarget) -> bool {
target.window_id.is_some()
|| windows
.iter()
.filter(|window| resolve_window_target(std::slice::from_ref(*window), target).is_ok())
.take(2)
.count()
== 1
}
/// The ownership target of a click or scroll: the focused window's pid, else
/// the requested window's, else the pid the caller passed, plus the
/// requested window. Only a `window_id` counts as naming that window exactly.
fn focus_owner_target(
focus: Option<&WindowFocusResult>,
pid: Option<u32>,
window_id: Option<u64>,
) -> ElementOwnerTarget {
match focus {
Some(focus) => ElementOwnerTarget {
pid: focus_target_pid(Some(focus)).or(pid),
window: Some((
focus.requested_window.window_id,
focus.requested_window.backend.clone(),
)),
exact_window: window_id.is_some(),
},
None => ElementOwnerTarget::pid(pid),
}
}
fn focus_target_pid(focus: Option<&WindowFocusResult>) -> Option<u32> {
let focus = focus?;
focus
.focused_window
.as_ref()
.and_then(|window| window.pid)
.or(focus.requested_window.pid)
}
fn describe_focused_element(element: &FocusedElementSummary, expects_editable: bool) -> String {
let name = element
.name
.as_deref()
.filter(|name| !name.is_empty())
.map(|name| format!(" \"{name}\""))
.unwrap_or_default();
if focused_element_is_editable(element) {
format!("Focused element: {}{name} (editable).", element.role)
} else if element.is_terminal {
format!(
"Focused element: {}{name} (terminal keyboard input).",
element.role
)
} else if expects_editable {
format!(
"WARNING: focused element is {}{name}, which is not editable, so the text may not have reached a field. Click the intended input or use set_value on an editable one, and re-observe before retrying.",
element.role
)
} else {
format!("Focused element: {}{name} (not editable).", element.role)
}
}
fn first_line(text: &str) -> &str {
text.lines().next().unwrap_or(text)
}
/// Append supplemental notes (off-screen or focused-element feedback) to an
/// action result message without changing ok/implemented semantics.
fn with_notes(mut output: ActionOutput, notes: impl IntoIterator<Item = String>) -> ActionOutput {
for note in notes {
output.message = format!("{} {note}", output.message);
}
output
}
/// Where the device moved a point it clamped. `name` calls out the point,
/// such as "coordinate" or "start coordinate".
fn abs_pointer_clamp_note(
name: &str,
landing: crate::abs_pointer::PointerLanding,
) -> Option<String> {
(landing.requested != landing.emitted).then(|| {
format!(
"Requested {name} {},{} was clamped to {},{} by the uinput absolute pointer.",
landing.requested.0, landing.requested.1, landing.emitted.0, landing.emitted.1
)
})
}
fn portal_focus_target(target: &WindowTarget, window: &WindowInfo) -> WindowTarget {
if registry::backend_can_exact_focus(&window.backend) {
WindowTarget {
window_id: Some(window.window_id),
..Default::default()
}
} else if target.has_target() {
// GNOME Introspect can activate an app, but cannot activate a specific
// window. Preserve the caller's app-level focus contract there.
target.clone()
} else {
WindowTarget {
app_id: window.app_id.clone(),
..Default::default()
}
}
}
fn focus_satisfies_target(focus: &WindowFocusResult, target: &WindowTarget) -> bool {
if target.requires_exact_focus() {
focus.exact_window_focused
} else {
focus.exact_window_focused || focus.app_focused
}
}
async fn window_list_output() -> ListWindowsOutput {
match list_windows().await {
Ok(windows) => {
let backend = window_backend(windows.iter());
let note = registry::list_note(&backend);
ListWindowsOutput {
backend,
windows,
error: None,
permissions_hint: None,
note: note.to_string(),
}
}
Err(error) => {
let error = format!("{error:#}");
ListWindowsOutput {
backend: UNAVAILABLE_BACKEND.to_string(),
windows: Vec::new(),
permissions_hint: window_permission_hint(&error),
error: Some(error),
note: "Window listing failed, so targeted keyboard input cannot safely focus or verify a target window."
.to_string(),
}
}
}
}
fn window_backend<'a>(windows: impl Iterator<Item = &'a WindowInfo>) -> String {
windows
.map(|window| window.backend.clone())
.next()
.unwrap_or_else(|| UNAVAILABLE_BACKEND.to_string())
}
fn absolute_mousemove_args(x: i32, y: i32) -> Vec<String> {
vec![
"mousemove".to_string(),
"--absolute".to_string(),
"--".to_string(),
x.to_string(),
y.to_string(),
]
}
fn xdotool_pointer_click_args(
x: i32,
y: i32,
count: u32,
button: Option<&str>,
) -> Option<Vec<String>> {
let button = xdotool_pointer_button_code(button)?;
Some(vec![
"mousemove".to_string(),
"--".to_string(),
x.to_string(),
y.to_string(),
"click".to_string(),
"--repeat".to_string(),
count.to_string(),
button.to_string(),
])
}
fn xdotool_pointer_button_code(button: Option<&str>) -> Option<&'static str> {
match button.unwrap_or("left").to_ascii_lowercase().as_str() {
"left" => Some("1"),
"middle" => Some("2"),
"right" => Some("3"),
_ => None,
}
}
/// Gap between XTEST wheel-button clicks, in milliseconds.
///
/// xdotool's own default is 100ms. One page is five clicks and valid requests
/// can ask for up to 100 pages, so 12ms keeps the bounded operation responsive
/// while matching the pacing used by `xdotool type`.
const XDOTOOL_SCROLL_CLICK_DELAY_MS: u64 = 12;
/// X11 wheel buttons: 4 up, 5 down, 6 left, 7 right. `repeat` is the notch
/// count, the same number ydotool would pass to `mousemove --wheel`.
fn xdotool_pointer_scroll_args(
point: Option<(i32, i32)>,
repeat: u32,
direction: ScrollDirection,
) -> Vec<String> {
let button = match direction {
ScrollDirection::Up => "4",
ScrollDirection::Down => "5",
ScrollDirection::Left => "6",
ScrollDirection::Right => "7",
};
let mut args = Vec::new();
if let Some((x, y)) = point {
args.extend([
"mousemove".to_string(),
"--".to_string(),
x.to_string(),
y.to_string(),
]);
}
args.extend([
"click".to_string(),
"--delay".to_string(),
XDOTOOL_SCROLL_CLICK_DELAY_MS.to_string(),
"--repeat".to_string(),
repeat.max(1).to_string(),
button.to_string(),
]);
args
}
/// `xdotool click --repeat` sleeps `--delay` after every click, including
/// the last, so the command timeout has to cover that gap.
fn xdotool_scroll_timeout(repeat: u32, delay_ms: u64) -> Duration {
INPUT_COMMAND_TIMEOUT.saturating_add(Duration::from_millis(
u64::from(repeat.max(1)).saturating_mul(delay_ms),
))
}
#[derive(Debug)]
struct PointerCommandResult {
outputs: Vec<Output>,
backend: KeyboardCommandBackend,
}
async fn run_xdotool_pointer_or_fallback<F, Fut>(
program: &Path,
args: &[String],
fallback: F,
) -> std::result::Result<PointerCommandResult, String>
where
F: FnOnce() -> Fut,
Fut: Future<Output = std::result::Result<Vec<Output>, String>>,
{
match run_xdotool(program, args).await {
XdotoolAttempt::Unavailable => fallback().await.map(|outputs| PointerCommandResult {
outputs,
backend: KeyboardCommandBackend::Ydotool,
}),
XdotoolAttempt::Finished(result) => result.map(|output| PointerCommandResult {
outputs: vec![output],
backend: KeyboardCommandBackend::Xdotool,
}),
}
}
async fn run_xdotool_pointer_or_fallback_with_timeout<F, Fut>(
program: &Path,
args: &[String],
command_timeout: Duration,
fallback: F,
) -> std::result::Result<PointerCommandResult, String>
where
F: FnOnce() -> Fut,
Fut: Future<Output = std::result::Result<Vec<Output>, String>>,
{
match run_xdotool_with_timeout(program, args, command_timeout).await {
XdotoolAttempt::Unavailable => fallback().await.map(|outputs| PointerCommandResult {
outputs,
backend: KeyboardCommandBackend::Ydotool,
}),
XdotoolAttempt::Finished(result) => result.map(|output| PointerCommandResult {
outputs: vec![output],
backend: KeyboardCommandBackend::Xdotool,
}),
}
}
/// Wheel notches for a scroll in the kernel's signs: `REL_WHEEL` is positive
/// up and `REL_HWHEEL` is positive right. libinput negates only the vertical
/// axis, before per-device settings such as natural scrolling. Both wheel
/// routes send them
/// unchanged: the uinput pointer writes them, and `ydotool mousemove --wheel`
/// writes its x to `REL_HWHEEL` and its y to `REL_WHEEL`.
fn wheel_deltas(direction: ScrollDirection, units: i32) -> (i32, i32) {
match direction {
ScrollDirection::Up => (0, units),
ScrollDirection::Down => (0, -units),
ScrollDirection::Left => (-units, 0),
ScrollDirection::Right => (units, 0),
}
}
fn wheel_mousemove_args(dx: i32, dy: i32) -> Vec<String> {
vec![
"mousemove".to_string(),
"--wheel".to_string(),
"--".to_string(),
dx.to_string(),
dy.to_string(),
]
}
async fn run_ydotool_sequence(
commands: &[Vec<String>],
) -> std::result::Result<Vec<Output>, String> {
let mut outputs = Vec::new();
for (index, args) in commands.iter().enumerate() {
outputs.push(run_ydotool(args).await?);
if index + 1 < commands.len() {
sleep(Duration::from_millis(35)).await;
}
}
Ok(outputs)
}
async fn xdotool_pointer_output(
program: &Path,
args: &[String],
) -> std::result::Result<Output, String> {
match run_xdotool(program, args).await {
XdotoolAttempt::Finished(result) => result,
XdotoolAttempt::Unavailable => Err(
"Native X11 pointer input requires xdotool; no system-wide input fallback was used."
.to_string(),
),
}
}
async fn run_xdotool_drag(
program: &Path,
start: (i32, i32),
end: (i32, i32),
) -> std::result::Result<Vec<Output>, String> {
let mut outputs = vec![
xdotool_pointer_output(
program,
&[
"mousemove".into(),
"--".into(),
start.0.to_string(),
start.1.to_string(),
],
)
.await?,
];
let mut first_error =
match xdotool_pointer_output(program, &["mousedown".into(), "1".into()]).await {
Ok(output) => {
outputs.push(output);
None
}
Err(error) => Some(error),
};
if first_error.is_none() {
for (x, y) in crate::pointer_path::drag_points(start, end) {
sleep(crate::pointer_path::DRAG_STEP_DELAY).await;
match xdotool_pointer_output(
program,
&[
"mousemove".into(),
"--".into(),
x.to_string(),
y.to_string(),
],
)
.await
{
Ok(output) => outputs.push(output),
Err(error) => {
first_error = Some(error);
break;
}
}
}
}
// A failed command may have sent the press before failing. Always release,
// and do not replay this partially dispatched gesture through another route.
let release = xdotool_pointer_output(program, &["mouseup".into(), "1".into()]).await;
match (first_error, release) {
(None, Ok(output)) => {
outputs.push(output);
Ok(outputs)
}
(Some(error), Ok(_)) => Err(error),
(None, Err(error)) => Err(error),
(Some(error), Err(release)) => Err(format!(
"{error}; X11 button release also failed: {release}"
)),
}
}
async fn run_ydotool_drag(
start_x: i32,
start_y: i32,
end_x: i32,
end_y: i32,
) -> std::result::Result<Vec<Output>, String> {
let mut outputs = vec![run_ydotool(&absolute_mousemove_args(start_x, start_y)).await?];
sleep(Duration::from_millis(35)).await;
let mut first_error = match run_ydotool(&["click".to_string(), "0x40".to_string()]).await {
Ok(output) => {
outputs.push(output);
None
}
Err(error) => Some(error),
};
sleep(Duration::from_millis(35)).await;
if first_error.is_none() {
match run_ydotool(&absolute_mousemove_args(end_x, end_y)).await {
Ok(output) => outputs.push(output),
Err(error) => first_error = Some(error),
}
sleep(Duration::from_millis(35)).await;
}
let release = run_ydotool(&["click".to_string(), "0x80".to_string()]).await;
match (first_error, release) {
(None, Ok(output)) => {
outputs.push(output);
Ok(outputs)
}
(Some(error), Ok(_)) => Err(error),
(None, Err(error)) => Err(error),
(Some(error), Err(release_error)) => Err(format!(
"{error}; ydotool button release also failed: {release_error}"
)),
}
}
async fn run_abs_pointer<T, F>(
pointer: Arc<Mutex<Option<crate::abs_pointer::AbsPointer>>>,
operation: F,
) -> std::result::Result<T, String>
where
T: Send + 'static,
F: FnOnce(&mut crate::abs_pointer::AbsPointer) -> Result<T> + Send + 'static,
{
tokio::task::spawn_blocking(move || {
let mut guard = pointer
.lock()
.map_err(|_| "absolute pointer lock was poisoned".to_string())?;
let pointer = guard
.as_mut()
.ok_or_else(|| "absolute pointer is unavailable".to_string())?;
let result = operation(pointer).map_err(|error| format!("{error:#}"));
if result.is_err() {
// A failed write may have left a button down or an event half
// sent. Drop the device, which the kernel releases, so the next
// action builds a fresh one.
guard.take();
}
result
})
.await
.map_err(|error| format!("absolute pointer task failed: {error}"))?
}
async fn run_cancellation_safe_guarded<G, T, F>(
guard: G,
operation: F,
) -> (Option<G>, std::result::Result<T, String>)
where
G: Send + 'static,
T: Send + 'static,
F: Future<Output = T> + Send + 'static,
{
// Dropping a JoinHandle detaches its task, retaining the guard until the
// stateful input operation has completed even if the caller is cancelled.
match tokio::spawn(async move { (guard, operation.await) }).await {
Ok((guard, result)) => (Some(guard), Ok(result)),
Err(error) => (None, Err(format!("stateful input task failed: {error}"))),
}
}
async fn run_cancellation_safe_input<G, T, F>(
input_guard: G,
operation: F,
) -> (Option<G>, std::result::Result<T, String>)
where
G: Send + 'static,
T: Send + 'static,
F: Future<Output = std::result::Result<T, String>> + Send + 'static,
{
let (input_guard, result) = run_cancellation_safe_guarded(input_guard, operation).await;
(input_guard, result.and_then(|result| result))
}
async fn run_ydotool(args: &[String]) -> std::result::Result<Output, String> {
let support = ydotool::ensure_supported_async().await?;
let socket = ydotool::socket_path_for_command()?;
let mut command = TokioCommand::new(&support.executable);
command.args(args);
command.env("YDOTOOL_SOCKET", socket);
let output =
crate::command_runner::output_with_timeout(command, "run ydotool", INPUT_COMMAND_TIMEOUT)
.await
.map_err(|error| format!("{error:#}"))?;
if output.status.success() {
if let Some(error) = ydotool::cli_error(&output.stderr) {
Err(error)
} else {
Ok(output)
}
} else {
Err(ydotool_output_error(output))
}
}
async fn run_ydotool_type_text(text: &str) -> std::result::Result<Output, String> {
validate_ydotool_text(text)?;
let support = ydotool::ensure_supported_async().await?;
let socket = ydotool::socket_path_for_command()?;
let mut command = TokioCommand::new(&support.executable);
command.args(["type", "--file", "-"]);
command.env("YDOTOOL_SOCKET", socket);
let output = crate::command_runner::output_with_stdin(
command,
"run ydotool type",
ydotool_type_timeout(text),
text.as_bytes().to_vec(),
)
.await
.map_err(|error| format!("{error:#}"))?;
if output.status.success() {
if let Some(error) = ydotool::cli_error(&output.stderr) {
Err(error)
} else {
Ok(output)
}
} else {
Err(ydotool_output_error(output))
}
}
fn validate_ydotool_text(text: &str) -> std::result::Result<(), String> {
if let Some(ch) = text
.chars()
.find(|ch| !matches!(ch, '\t' | '\n' | ' '..='~'))
{
return Err(format!(
"ydotool cannot type U+{:04X}; no text was sent. Use a layout-aware text backend or set_value on an editable field.",
ch as u32
));
}
Ok(())
}
fn ydotool_type_timeout(text: &str) -> Duration {
let text_seconds = (text.chars().count() as u64).div_ceil(YDOTOOL_TYPE_CHARS_PER_SECOND);
Duration::from_secs(INPUT_COMMAND_TIMEOUT.as_secs().saturating_add(text_seconds))
}
const EVDEV_KEY_LEFTCTRL: i32 = 29;
const EVDEV_KEY_LEFTSHIFT: i32 = 42;
const EVDEV_KEY_V: i32 = 47;
const EVDEV_KEY_INSERT: i32 = 110;
const KDE_CLIPBOARD_RESTORE_MIN_DELAY_MS: u64 = 1_500;
const KDE_CLIPBOARD_RESTORE_MAX_DELAY_MS: u64 = 5_000;
const KDE_CLIPBOARD_RESTORE_CHARS_PER_SECOND: u64 = 250;
fn kde_clipboard_restore_delay(text: &str) -> Duration {
let text_delay_ms = (text.chars().count() as u64)
.saturating_mul(1_000)
.div_ceil(KDE_CLIPBOARD_RESTORE_CHARS_PER_SECOND);
Duration::from_millis(text_delay_ms.clamp(
KDE_CLIPBOARD_RESTORE_MIN_DELAY_MS,
KDE_CLIPBOARD_RESTORE_MAX_DELAY_MS,
))
}
#[derive(Debug)]
struct KdeClipboardPasteError {
message: String,
can_fallback_to_ydotool: bool,
clear_portal_keyboard_session: bool,
}
impl KdeClipboardPasteError {
fn before_text_input(message: String) -> Self {
Self {
message,
can_fallback_to_ydotool: true,
clear_portal_keyboard_session: false,
}
}
fn after_portal_input(message: String) -> Self {
Self {
message,
can_fallback_to_ydotool: false,
clear_portal_keyboard_session: true,
}
}
/// The clipboard write failed, but it may have taken effect (a timed out
/// call can still land). No paste was sent, yet the outcome is unknown,
/// so the text is not retyped through another backend.
fn ambiguous_clipboard_set(message: String) -> Self {
Self {
message,
can_fallback_to_ydotool: false,
clear_portal_keyboard_session: false,
}
}
}
/// Put `text` on the clipboard. A failed write may still have landed, so
/// put `previous` back and refuse to retype the text another way.
async fn replace_kde_clipboard<F, Fut>(
text: &str,
previous: &str,
set: F,
) -> std::result::Result<(), KdeClipboardPasteError>
where
F: Fn(String) -> Fut,
Fut: Future<Output = std::result::Result<(), String>>,
{
let Err(set_error) = set(text.to_string()).await else {
return Ok(());
};
let message = match set(previous.to_string()).await {
Ok(()) => format!(
"KDE clipboard replacement failed ambiguously and the previous contents were restored; text was not replayed through another backend: {set_error}"
),
Err(restore_error) => format!(
"KDE clipboard replacement failed ambiguously; restoring the previous contents also failed, and text was not replayed through another backend: {set_error}; restore failed: {restore_error}"
),
};
Err(KdeClipboardPasteError::ambiguous_clipboard_set(message))
}
async fn run_kde_clipboard_paste_text(
session: &PortalKeyboardSession,
text: &str,
paste_shortcut: KdeClipboardPasteShortcut,
operation_guard: InputOperationGuard,
) -> std::result::Result<String, KdeClipboardPasteError> {
let previous = kde_clipboard_contents()
.await
.map_err(KdeClipboardPasteError::before_text_input)?;
replace_kde_clipboard(text, &previous, |value| async move {
kde_set_clipboard_contents(&value).await
})
.await?;
let (modifiers, keycode) = kde_clipboard_paste_chord(paste_shortcut);
let paste_result = press_keycode_chord(session, modifiers, keycode, operation_guard)
.await
.map_err(|error| format!("{error:#}"));
sleep(kde_clipboard_restore_delay(text)).await;
let restore_result = kde_set_clipboard_contents(&previous).await;
match (paste_result, restore_result) {
(Ok(_), Ok(_)) => Ok("Action pasted through KDE clipboard integration.".to_string()),
(Err(error), Ok(_)) => Err(KdeClipboardPasteError::after_portal_input(error)),
(Ok(_), Err(restore_error)) => Ok(format!(
"Action pasted through KDE clipboard integration. Warning: previous KDE clipboard contents could not be restored: {restore_error}"
)),
(Err(error), Err(restore_error)) => Err(KdeClipboardPasteError::after_portal_input(
format!("{error}; previous KDE clipboard contents could not be restored: {restore_error}"),
)),
}
}
/// Paste chord for KDE clipboard text input.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum KdeClipboardPasteShortcut {
/// Ctrl+V, for ordinary widgets.
Standard,
/// Ctrl+Shift+V, for most terminal emulators.
CtrlShiftV,
/// Shift+Insert, for the xterm and rxvt families.
ShiftInsert,
}
async fn kde_clipboard_paste_shortcut(
target: &WindowTarget,
focus: Option<&WindowFocusResult>,
) -> KdeClipboardPasteShortcut {
let current = match focus {
Some(_) => None,
None => focused_window().await.ok().flatten(),
};
let window = focus
.map(|focus| {
focus
.focused_window
.as_ref()
.unwrap_or(&focus.requested_window)
})
.or(current.as_ref());
let terminal = kde_clipboard_terminal_shortcut(target, window);
if terminal.is_none() {
return KdeClipboardPasteShortcut::Standard;
}
// A terminal window can hold focus on a non-terminal widget (a search
// field, a tab rename box) that pastes with plain Ctrl+V. Only an answer
// from the terminal's own AT-SPI app may override the chord: without a pid,
// or when the terminal exposes no AT-SPI root, keep the terminal chord.
let Some(pid) = window.and_then(|window| window.pid) else {
return kde_clipboard_shortcut_for_focus(terminal, None);
};
let focused_element = timeout(
Duration::from_millis(1_500),
focused_element_summary_in_app(pid),
)
.await
.ok()
.and_then(Result::ok)
.flatten();
kde_clipboard_shortcut_for_focus(terminal, focused_element.as_ref())
}
/// The terminal chord for the resolved window, or for an explicit terminal
/// selector when no window resolved. A resolved window always wins over a
/// possibly stale selector.
fn kde_clipboard_terminal_shortcut(
target: &WindowTarget,
window: Option<&WindowInfo>,
) -> Option<TerminalPasteShortcut> {
match window {
Some(window) => terminal_paste_shortcut(window),
None => target
.has_terminal_target()
.then_some(TerminalPasteShortcut::CtrlShiftV),
}
}
fn kde_clipboard_shortcut_for_focus(
terminal: Option<TerminalPasteShortcut>,
focused_element: Option<&FocusedElementSummary>,
) -> KdeClipboardPasteShortcut {
// No AT-SPI answer (xterm exposes none) keeps the terminal chord; only a
// focused non-terminal text field switches back to Ctrl+V. A remaining
// container focus (filler, panel) keeps the terminal chord too.
if focused_element.is_some_and(|element| !element.is_terminal && element.accepts_text()) {
return KdeClipboardPasteShortcut::Standard;
}
match terminal {
Some(TerminalPasteShortcut::CtrlShiftV) => KdeClipboardPasteShortcut::CtrlShiftV,
Some(TerminalPasteShortcut::ShiftInsert) => KdeClipboardPasteShortcut::ShiftInsert,
None => KdeClipboardPasteShortcut::Standard,
}
}
fn kde_clipboard_paste_chord(shortcut: KdeClipboardPasteShortcut) -> (&'static [i32], i32) {
match shortcut {
KdeClipboardPasteShortcut::Standard => (&[EVDEV_KEY_LEFTCTRL], EVDEV_KEY_V),
KdeClipboardPasteShortcut::CtrlShiftV => {
(&[EVDEV_KEY_LEFTCTRL, EVDEV_KEY_LEFTSHIFT], EVDEV_KEY_V)
}
KdeClipboardPasteShortcut::ShiftInsert => (&[EVDEV_KEY_LEFTSHIFT], EVDEV_KEY_INSERT),
}
}
async fn kde_clipboard_contents() -> std::result::Result<String, String> {
let connection = kde_clipboard_connection().await?;
let proxy = kde_clipboard_proxy(&connection).await?;
let output: String = kde_clipboard_dbus_operation(
"getClipboardContents",
proxy.call("getClipboardContents", &()),
)
.await?;
Ok(output)
}
async fn kde_set_clipboard_contents(text: &str) -> std::result::Result<(), String> {
let connection = kde_clipboard_connection().await?;
let proxy = kde_clipboard_proxy(&connection).await?;
let _: () = kde_clipboard_dbus_operation(
"setClipboardContents",
proxy.call("setClipboardContents", &(text)),
)
.await?;
Ok(())
}
async fn kde_clipboard_connection() -> std::result::Result<ZbusConnection, String> {
ZbusConnection::session()
.await
.map_err(|error| format!("failed to connect to session bus for KDE clipboard: {error}"))
}
async fn kde_clipboard_proxy(
connection: &ZbusConnection,
) -> std::result::Result<ZbusProxy<'_>, String> {
kde_clipboard_dbus_operation(
"proxy creation",
ZbusProxy::new(
connection,
KDE_KLIPPER_SERVICE,
KDE_KLIPPER_PATH,
KDE_KLIPPER_INTERFACE,
),
)
.await
}
async fn kde_clipboard_dbus_operation<T, F>(
operation: &'static str,
future: F,
) -> std::result::Result<T, String>
where
F: Future<Output = zbus::Result<T>>,
{
kde_clipboard_dbus_operation_with_timeout(operation, future, KDE_CLIPBOARD_DBUS_TIMEOUT).await
}
async fn kde_clipboard_dbus_operation_with_timeout<T, F>(
operation: &'static str,
future: F,
timeout_duration: Duration,
) -> std::result::Result<T, String>
where
F: Future<Output = zbus::Result<T>>,
{
timeout(timeout_duration, future)
.await
.map_err(|_| format!("KDE clipboard {operation} timed out"))?
.map_err(|error| format!("KDE clipboard {operation} failed: {error}"))
}
fn ydotool_output_error(output: Output) -> String {
command_output_error("ydotool", output)
}
/// X11 keyboard input runs through `xdotool` (XTEST) instead of ydotool.
///
/// ydotool injects raw evdev keycodes into a virtual uinput device. Under X11
/// the server re-interprets those scancodes through the active XKB layout, so
/// named keys and chords land as unrelated glyphs and literal text can mangle
/// symbols/digits (`_` → `%`, `1` → `+`). XTEST resolves keysyms against the
/// live layout, which is what X11 clients actually expect. See issue #58.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum KeyboardCommandBackend {
Wtype,
Xdotool,
Ydotool,
}
struct KeyboardCommandResult {
output: Output,
backend: KeyboardCommandBackend,
}
enum XdotoolAttempt {
Unavailable,
Finished(std::result::Result<Output, String>),
}
async fn run_xdotool(program: &Path, args: &[String]) -> XdotoolAttempt {
run_xdotool_with_timeout(program, args, INPUT_COMMAND_TIMEOUT).await
}
async fn run_xdotool_with_timeout(
program: &Path,
args: &[String],
command_timeout: Duration,
) -> XdotoolAttempt {
let mut command = TokioCommand::new(program);
command.args(args);
command.stdout(Stdio::piped());
command.stderr(Stdio::piped());
command.kill_on_drop(true);
command.process_group(0);
// A busy-file spawn error is transient, not "xdotool is unavailable"; do
// not let it route input to the ydotool fallback.
match crate::command_runner::spawn_retrying_busy(&mut command).await {
Ok(child) => XdotoolAttempt::Finished(
match crate::command_runner::output_child(child, "run xdotool", command_timeout)
.await
.map_err(|error| format!("{error:#}"))
{
Ok(output) if output.status.success() => Ok(output),
Ok(output) => Err(command_output_error("xdotool", output)),
Err(error) => Err(error),
},
),
Err(_) => XdotoolAttempt::Unavailable,
}
}
async fn run_xdotool_or_fallback<F, Fut>(
program: &Path,
args: &[String],
fallback: F,
) -> std::result::Result<KeyboardCommandResult, String>
where
F: FnOnce() -> Fut,
Fut: Future<Output = std::result::Result<Output, String>>,
{
run_xdotool_or_fallback_with_timeout(program, args, INPUT_COMMAND_TIMEOUT, fallback).await
}
async fn run_xdotool_or_fallback_with_timeout<F, Fut>(
program: &Path,
args: &[String],
command_timeout: Duration,
fallback: F,
) -> std::result::Result<KeyboardCommandResult, String>
where
F: FnOnce() -> Fut,
Fut: Future<Output = std::result::Result<Output, String>>,
{
match run_xdotool_with_timeout(program, args, command_timeout).await {
XdotoolAttempt::Unavailable => fallback().await.map(|output| KeyboardCommandResult {
output,
backend: KeyboardCommandBackend::Ydotool,
}),
XdotoolAttempt::Finished(result) => result.map(|output| KeyboardCommandResult {
output,
backend: KeyboardCommandBackend::Xdotool,
}),
}
}
async fn run_xdotool_type_with_input_check<F, Fut>(
program: &Path,
args: &[String],
command_timeout: Duration,
fallback: F,
) -> std::result::Result<(KeyboardCommandResult, Option<String>), String>
where
F: FnOnce() -> Fut,
Fut: Future<Output = std::result::Result<Output, String>>,
{
let guard = tokio::task::spawn_blocking(crate::x11_display::X11TypingGuard::start)
.await
.map_err(|error| format!("{error}"))
.and_then(|result| result.map_err(|error| format!("{error:#}")));
let result =
run_xdotool_or_fallback_with_timeout(program, args, command_timeout, fallback).await?;
if result.backend == KeyboardCommandBackend::Xdotool {
let guard = match guard {
Ok(guard) => guard,
Err(error) => return Ok((result, Some(format!(
"WARNING: X11 layout-change detection was unavailable: {error}. Keyboard focus and grab detection was also unavailable. Field contents remain unverified; inspect the target before submitting or retrying."
)))),
};
// Never replay a dispatched command, even if verification fails.
tokio::task::spawn_blocking(move || guard.verify())
.await
.map_err(|error| format!("X11 input monitor failed after typing; inspect the target before retrying: {error}"))?
.map_err(|error| format!("X11 input verification failed after typing; text may have been sent. Inspect the target before submitting or retrying: {error:#}"))?;
}
Ok((result, None))
}
async fn run_wtype_type_text_or_fallback<F, Fut>(
program: &Path,
text: &str,
mode: WtypeTextMode,
secret: bool,
fallback: F,
) -> std::result::Result<KeyboardCommandResult, String>
where
F: FnOnce() -> Fut,
Fut: Future<Output = std::result::Result<Output, String>>,
{
// Defense in depth: no helper caller may put a secret in argv or bypass
// validation through the missing-program fallback.
validate_wtype_text(text)?;
validate_wtype_privacy(mode, secret)?;
let available = if program.components().count() > 1 {
executable_file(program)
} else {
program.to_str().is_some_and(which_in_path)
};
if !available {
return fallback().await.map(|output| KeyboardCommandResult {
output,
backend: KeyboardCommandBackend::Ydotool,
});
}
let mut command = TokioCommand::new(program);
// wtype's stdin mode uploads a new keymap every 100 characters. On
// Hyprland/Electron those changes can turn literal text into modifier
// keys and shortcuts while wtype still exits successfully. Passing one
// literal argument builds the complete keymap before the first key event.
// `--` prevents option-looking text from becoming wtype commands.
let result = match mode {
WtypeTextMode::Argument => {
command.args(["--", text]);
crate::command_runner::output_with_timeout(
command,
"run wtype",
ydotool_type_timeout(text),
)
.await
}
WtypeTextMode::Stdin => {
command.arg("-");
crate::command_runner::output_with_stdin(
command,
"run wtype",
ydotool_type_timeout(text),
text.as_bytes().to_vec(),
)
.await
}
};
let output = match result {
Ok(output) => output,
// wtype never started, so nothing was typed: same as a missing wtype.
Err(error) if spawn_never_started(&error) => {
return fallback().await.map(|output| KeyboardCommandResult {
output,
backend: KeyboardCommandBackend::Ydotool,
});
}
Err(error) => return Err(format!("{error:#}")),
};
if !output.status.success() {
return Err(command_output_error("wtype", output));
}
Ok(KeyboardCommandResult {
output,
backend: KeyboardCommandBackend::Wtype,
})
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum WtypeTextMode {
Argument,
Stdin,
}
enum WtypeFieldKind {
Password,
NonSecretText,
Ambiguous,
}
fn classify_wtype_field(element: &FocusedElementSummary) -> WtypeFieldKind {
// The overlay's mask predicate includes uncertainty. Only an explicit
// password role confirms a password; terminals can be password prompts.
if element.role.to_ascii_lowercase().contains("password") {
WtypeFieldKind::Password
} else if !element.is_terminal
&& !crate::indicator::is_secret_role(&element.role)
&& focused_element_is_editable(element)
{
WtypeFieldKind::NonSecretText
} else {
WtypeFieldKind::Ambiguous
}
}
fn wtype_text_mode_for_desktop(desktop: Option<&str>) -> WtypeTextMode {
if desktop.is_some_and(|value| {
value
.split(':')
.any(|part| part.eq_ignore_ascii_case("hyprland"))
}) {
WtypeTextMode::Argument
} else {
WtypeTextMode::Stdin
}
}
fn preflight_type_text(text: &str, mode: Option<WtypeTextMode>) -> std::result::Result<(), String> {
if mode.is_some() {
validate_wtype_text(text)?;
} else if text.contains('\0') {
return Err("Literal text was not sent: NUL characters are unsupported.".to_string());
}
Ok(())
}
async fn hyprland_wtype_delivery(
text: &str,
probe: impl Future<Output = Option<FocusProbe>>,
) -> std::result::Result<WtypeTextMode, String> {
let probe = timeout(WTYPE_FOCUS_TIMEOUT, probe).await.ok().flatten();
if let Some(FocusProbe::Found(element)) = probe.as_ref() {
match classify_wtype_field(element) {
WtypeFieldKind::Password => return Err("Literal text was not sent: wtype argument mode is disabled for password fields. Use set_value on an editable field.".to_string()),
WtypeFieldKind::NonSecretText => return Ok(WtypeTextMode::Argument),
WtypeFieldKind::Ambiguous => {}
}
}
// Terminals and generic text roles remain ambiguous even with AT-SPI.
// Keep their stdin route, bounded to a single wtype batch so no mid-text
// keymap refresh is needed. Ambiguous text must never be exposed in argv.
if text.chars().count() <= WTYPE_STDIN_MAX_CHARS {
Ok(WtypeTextMode::Stdin)
} else {
Err(format!("Literal text was not sent: the focused field could not be classified. Hyprland's private stdin fallback accepts at most {WTYPE_STDIN_MAX_CHARS} Unicode characters per call to avoid keymap-refresh corruption. Send smaller calls and verify each one, or enable the target's accessibility support."))
}
}
fn validate_wtype_privacy(mode: WtypeTextMode, secret: bool) -> std::result::Result<(), String> {
if mode == WtypeTextMode::Argument && secret {
return Err("Literal text was not sent: wtype argument mode is disabled for password fields and fields whose secrecy could not be determined. Enable the target application's accessibility support or use set_value on an editable field.".to_string());
}
Ok(())
}
fn focus_requires_secret_protection(probe: &FocusProbe) -> bool {
match probe {
FocusProbe::Found(element) => {
// Electron exposes the Editable state on normal entries without
// implementing EditableText. The interface is not a secrecy test.
let accepts_text = focused_element_is_editable(element) || element.is_terminal;
crate::indicator::is_secret_role(&element.role) || !accepts_text
}
_ => true,
}
}
fn focused_element_is_editable(element: &FocusedElementSummary) -> bool {
element.editable || element.states.iter().any(|state| state == "editable")
}
fn validate_wtype_text(text: &str) -> std::result::Result<(), String> {
if text.contains('\0') {
return Err(
"Literal text was not sent: wtype cannot represent NUL characters.".to_string(),
);
}
if text.len() > WTYPE_MAX_TEXT_BYTES {
return Err(format!(
"Literal text was not sent: wtype accepts at most {WTYPE_MAX_TEXT_BYTES} UTF-8 bytes per call. Use set_value on an editable field or send smaller text segments."
));
}
Ok(())
}
/// True when `xdotool` can drive this session: an X11 session with `DISPLAY`
/// set and the binary present. `COMPUTER_USE_LINUX_FORCE_YDOTOOL_KEYBOARD=1`
/// opts out; `COMPUTER_USE_LINUX_FORCE_XDOTOOL_KEYBOARD=1` forces it on.
fn xdotool_available() -> bool {
which_in_path("xdotool")
}
fn wtype_available() -> bool {
which_in_path("wtype")
}
fn prefer_wtype_keyboard(
force_ydotool: bool,
is_wayland: bool,
compatible_desktop: bool,
available: bool,
) -> bool {
!force_ydotool && is_wayland && compatible_desktop && available
}
/// Where `press_key` sends a chord.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ChordRoute {
Portal,
Wtype,
/// xdotool on X11, otherwise ydotool.
Command,
}
/// wtype replaces the portal only when the session has no RemoteDesktop
/// portal at all, and the portal is neither forced nor already holding the
/// user's consent. Wherever the portal exists it stays preferred.
fn chord_route_from(
prefer_portal: bool,
portal_pinned: bool,
wtype: bool,
portal_missing: bool,
) -> ChordRoute {
if !prefer_portal {
ChordRoute::Command
} else if !portal_pinned && wtype && portal_missing {
ChordRoute::Wtype
} else {
ChordRoute::Portal
}
}
/// wtype arguments for a chord: hold each modifier with `-M`, tap the key by
/// keysym name with `-k`, then release the modifiers in reverse with `-m`.
/// wtype uploads its own keymap, so keysyms do not depend on the layout.
/// `-M` only changes the modifier state, so a lone modifier is tapped as its
/// left-hand keysym, like the portal route.
fn wtype_key_args(key: &str) -> Option<Vec<String>> {
let parts = key
.split('+')
.map(str::trim)
.filter(|part| !part.is_empty())
.collect::<Vec<_>>();
let (key_part, modifier_parts) = parts.split_last()?;
// Validate through the same evdev grammar so every backend accepts
// exactly the same input set.
key_chord(key)?;
let modifiers = modifier_parts
.iter()
.map(|part| wtype_modifier_name(part))
.collect::<Option<Vec<_>>>()?;
let mut args = Vec::new();
for modifier in &modifiers {
args.extend(["-M".to_string(), modifier.to_string()]);
}
let keysym = match wtype_modifier_keysym(key_part).filter(|_| modifier_parts.is_empty()) {
Some(bare) => bare.to_string(),
None => xdotool_keysym_name(key_part)?,
};
args.extend(["-k".to_string(), keysym]);
for modifier in modifiers.iter().rev() {
args.extend(["-m".to_string(), modifier.to_string()]);
}
Some(args)
}
fn wtype_modifier_name(key: &str) -> Option<&'static str> {
match normalize_key(key).as_str() {
"ctrl" | "control" => Some("ctrl"),
"alt" | "option" => Some("alt"),
"shift" => Some("shift"),
"meta" | "super" | "cmd" | "command" => Some("logo"),
_ => None,
}
}
/// Standalone keysym for a bare modifier press (`press_key "Super"`).
fn wtype_modifier_keysym(key: &str) -> Option<&'static str> {
match normalize_key(key).as_str() {
"ctrl" | "control" => Some("Control_L"),
"alt" | "option" => Some("Alt_L"),
"shift" => Some("Shift_L"),
"meta" | "super" | "cmd" | "command" => Some("Super_L"),
_ => None,
}
}
/// Pause, in milliseconds, after wtype uploads its keymap and before the
/// first key. Each wtype run is a new virtual keyboard, and on Sway the
/// focused client gets a fresh `wl_keyboard.enter` for it. A key pressed
/// before that enter arrives only in its held-key list, so the client never
/// sees the press.
const WTYPE_KEY_SETTLE_MS: u64 = 50;
async fn run_wtype_key(args: &[String]) -> std::result::Result<(), String> {
let mut command = TokioCommand::new("wtype");
command
.args(["-s", &WTYPE_KEY_SETTLE_MS.to_string()])
.args(args);
let output =
crate::command_runner::output_with_timeout(command, "run wtype", INPUT_COMMAND_TIMEOUT)
.await
.map_err(|error| format!("{error:#}"))?;
if !output.status.success() {
return Err(command_output_error("wtype", output));
}
Ok(())
}
/// Default per-character delay for `xdotool type`, in milliseconds (xdotool's
/// own default). `--delay 0` lets XTEST key events race each other on some X
/// servers (Cinnamon on Mint 22, issue #147): every character arrives, but in
/// a random order. A small delay keeps them ordered.
/// `COMPUTER_USE_LINUX_XDOTOOL_TYPE_DELAY_MS` overrides it.
const XDOTOOL_TYPE_DELAY_MS: u64 = 12;
const XDOTOOL_TYPE_DELAY_ENV: &str = "COMPUTER_USE_LINUX_XDOTOOL_TYPE_DELAY_MS";
fn xdotool_type_delay_ms() -> u64 {
env::var(XDOTOOL_TYPE_DELAY_ENV)
.ok()
.and_then(|value| value.trim().parse::<u64>().ok())
.unwrap_or(XDOTOOL_TYPE_DELAY_MS)
}
fn xdotool_type_args_with_delay(text: &str, delay_ms: u64) -> Vec<String> {
// XTEST maps LF to Linefeed, which editors ignore. CR maps to Return.
// Fold CRLF first so a Windows line ending still inserts one newline.
let text = text.replace("\r\n", "\n").replace('\n', "\r");
vec![
"type".to_string(),
"--clearmodifiers".to_string(),
"--delay".to_string(),
delay_ms.to_string(),
"--".to_string(),
text,
]
}
/// `xdotool type` spends about `delay_ms` per character, so long inputs need
/// more than the flat input timeout.
fn xdotool_type_timeout(text: &str, delay_ms: u64) -> Duration {
let chars = text.chars().count() as u64;
INPUT_COMMAND_TIMEOUT.saturating_add(Duration::from_millis(chars.saturating_mul(delay_ms)))
}
fn which_in_path(binary: &str) -> bool {
env::var_os("PATH").is_some_and(|path| executable_in(&path, binary))
}
/// True when a directory of `path` (a `PATH` value) holds an executable
/// `binary`.
fn executable_in(path: &std::ffi::OsStr, binary: &str) -> bool {
env::split_paths(path).any(|dir| executable_file(&dir.join(binary)))
}
/// True when the program never started: exec found no file, no permission,
/// no usable format or interpreter, or the file stayed busy after
/// `spawn_retrying_busy` gave up.
fn spawn_never_started(error: &anyhow::Error) -> bool {
error
.chain()
.filter_map(|cause| cause.downcast_ref::<std::io::Error>())
.any(|error| {
matches!(
error.raw_os_error(),
Some(libc::ENOENT | libc::EACCES | libc::ENOEXEC | libc::ETXTBSY)
)
})
}
/// A regular file with an execute bit set, which exec needs.
fn executable_file(path: &Path) -> bool {
use std::os::unix::fs::PermissionsExt;
std::fs::metadata(path)
.is_ok_and(|metadata| metadata.is_file() && metadata.permissions().mode() & 0o111 != 0)
}
/// Map our key grammar onto an `xdotool key` spec such as `ctrl+a`, `Return`,
/// or `shift+F5`. Returns `None` for keys the grammar does not accept, so the
/// caller keeps its existing "never silently dropped" error.
fn xdotool_key_spec(key: &str) -> Option<String> {
let parts = key
.split('+')
.map(str::trim)
.filter(|part| !part.is_empty())
.collect::<Vec<_>>();
let (key_part, modifier_parts) = parts.split_last()?;
// Validate through the same evdev grammar so both backends accept exactly
// the same input set.
key_chord(key)?;
let mut spec = Vec::new();
for part in modifier_parts {
spec.push(xdotool_modifier_name(part)?.to_string());
}
if modifier_parts.is_empty() {
if let Some(bare) = xdotool_modifier_keysym(key_part) {
return Some(bare.to_string());
}
}
spec.push(xdotool_keysym_name(key_part)?);
Some(spec.join("+"))
}
fn xdotool_modifier_name(key: &str) -> Option<&'static str> {
match normalize_key(key).as_str() {
"ctrl" | "control" => Some("ctrl"),
"alt" | "option" => Some("alt"),
"shift" => Some("shift"),
"meta" | "super" | "cmd" | "command" => Some("super"),
_ => None,
}
}
/// Standalone keysym for a bare modifier press (`press_key "Super"`).
fn xdotool_modifier_keysym(key: &str) -> Option<&'static str> {
match normalize_key(key).as_str() {
"ctrl" | "control" => Some("ctrl"),
"alt" | "option" => Some("alt"),
"shift" => Some("shift"),
"meta" | "super" | "cmd" | "command" => Some("super"),
_ => None,
}
}
fn xdotool_keysym_name(key: &str) -> Option<String> {
let normalized = normalize_key(key);
let named = match normalized.as_str() {
"enter" | "return" => "Return",
"escape" | "esc" => "Escape",
"tab" => "Tab",
"backspace" => "BackSpace",
"delete" | "del" => "Delete",
"space" => "space",
"home" => "Home",
"end" => "End",
"pageup" | "page_up" => "Page_Up",
"pagedown" | "page_down" => "Page_Down",
"arrowleft" | "left" => "Left",
"arrowright" | "right" => "Right",
"arrowup" | "up" => "Up",
"arrowdown" | "down" => "Down",
"f1" => "F1",
"f2" => "F2",
"f3" => "F3",
"f4" => "F4",
"f5" => "F5",
"f6" => "F6",
"f7" => "F7",
"f8" => "F8",
"f9" => "F9",
"f10" => "F10",
"f11" => "F11",
"f12" => "F12",
value if value.len() == 1 && value.as_bytes()[0].is_ascii_alphanumeric() => {
return Some(value.to_string());
}
_ => return None,
};
Some(named.to_string())
}
fn command_output_error(command: &str, output: Output) -> String {
let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string();
let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string();
let detail = if stderr.is_empty() { stdout } else { stderr };
if detail.is_empty() {
format!("{command} exited with {}", output.status)
} else {
detail
}
}
async fn ydotool_backend_available() -> bool {
ydotool_backend_available_from(
ydotool::connectable_socket_path().is_ok(),
ydotool::ensure_supported_async().await.is_ok(),
)
}
fn ydotool_backend_available_from(socket_available: bool, cli_supported: bool) -> bool {
socket_available && cli_supported
}
fn should_prefer_portal_backend_by_default(is_wayland: bool, ydotool_available: bool) -> bool {
is_wayland && !ydotool_available
}
fn prefer_portal_text_backend(
force_ydotool: bool,
prefer_xdotool: bool,
is_wayland: bool,
is_kde: bool,
force_portal: bool,
prefer_wtype: bool,
) -> bool {
!force_ydotool && !prefer_xdotool && is_wayland && !is_kde && (force_portal || !prefer_wtype)
}
fn mouse_button_code(button: Option<&str>) -> String {
match button.unwrap_or("left").to_ascii_lowercase().as_str() {
"right" => "0xC1",
"middle" => "0xC2",
"side" => "0xC3",
"extra" => "0xC4",
"forward" => "0xC5",
"back" => "0xC6",
_ => "0xC0",
}
.to_string()
}
/// Parse a chord like `Ctrl+Shift+P` into raw evdev codes: the held
/// modifiers plus the final key. A bare modifier (`Super`) parses as a
/// chord with no held modifiers whose key is the modifier itself.
fn key_chord(key: &str) -> Option<(Vec<u16>, u16)> {
let parts = key
.split('+')
.map(str::trim)
.filter(|part| !part.is_empty())
.collect::<Vec<_>>();
let (key_part, modifier_parts) = parts.split_last()?;
if modifier_parts.is_empty() {
if let Some(modifier) = modifier_keycode(key_part) {
return Some((Vec::new(), modifier));
}
}
let mut modifiers = Vec::new();
for part in modifier_parts {
modifiers.push(modifier_keycode(part)?);
}
let keycode = keycode(key_part)?;
Some((modifiers, keycode))
}
/// Portal form of [`key_chord`]. Modifiers and named keys go out as keysyms
/// so the compositor resolves them against the active keymap: with Caps Lock
/// and Control swapped, `Ctrl` presses whichever key produces `Control_L`
/// (issue #191). Letters and digits stay physical keycodes: mutter drops a
/// keysym missing from the current layout group, so `Ctrl+A` as a keysym
/// would vanish under a non-Latin layout, while the US keycode is what GTK and
/// Qt match shortcuts against there.
fn portal_key_chord(key: &str) -> Option<(Vec<PortalKey>, PortalKey)> {
let parts = key
.split('+')
.map(str::trim)
.filter(|part| !part.is_empty())
.collect::<Vec<_>>();
let (key_part, modifier_parts) = parts.split_last()?;
let (_, keycode) = key_chord(key)?;
let mut modifiers = Vec::new();
for part in modifier_parts {
modifiers.push(PortalKey::Keysym(modifier_keysym(part)?));
}
let key = modifier_keysym(key_part)
.filter(|_| modifier_parts.is_empty())
.or_else(|| named_keysym(key_part))
.map_or(PortalKey::Keycode(i32::from(keycode)), PortalKey::Keysym);
Some((modifiers, key))
}
fn modifier_keysym(key: &str) -> Option<i32> {
let keysym = match normalize_key(key).as_str() {
"ctrl" | "control" => xkey::Control_L,
"alt" | "option" => xkey::Alt_L,
"shift" => xkey::Shift_L,
"meta" | "super" | "cmd" | "command" => xkey::Super_L,
_ => return None,
};
Some(keysym as i32)
}
fn named_keysym(key: &str) -> Option<i32> {
let keysym = match normalize_key(key).as_str() {
"enter" | "return" => xkey::Return,
"escape" | "esc" => xkey::Escape,
"tab" => xkey::Tab,
"backspace" => xkey::BackSpace,
"delete" | "del" => xkey::Delete,
"space" => xkey::space,
"home" => xkey::Home,
"end" => xkey::End,
"pageup" | "page_up" => xkey::Page_Up,
"pagedown" | "page_down" => xkey::Page_Down,
"arrowleft" | "left" => xkey::Left,
"arrowright" | "right" => xkey::Right,
"arrowup" | "up" => xkey::Up,
"arrowdown" | "down" => xkey::Down,
"f1" => xkey::F1,
"f2" => xkey::F2,
"f3" => xkey::F3,
"f4" => xkey::F4,
"f5" => xkey::F5,
"f6" => xkey::F6,
"f7" => xkey::F7,
"f8" => xkey::F8,
"f9" => xkey::F9,
"f10" => xkey::F10,
"f11" => xkey::F11,
"f12" => xkey::F12,
_ => return None,
};
Some(keysym as i32)
}
fn key_sequence(key: &str) -> Option<Vec<String>> {
let (modifiers, keycode) = key_chord(key)?;
let mut events = Vec::new();
for modifier in &modifiers {
events.push(format!("{modifier}:1"));
}
events.push(format!("{keycode}:1"));
events.push(format!("{keycode}:0"));
for modifier in modifiers.iter().rev() {
events.push(format!("{modifier}:0"));
}
Some(events)
}
fn ydotool_key_args(key_events: Vec<String>, has_modifiers: bool) -> Vec<String> {
let mut args = vec!["key".to_string()];
if has_modifiers {
args.extend(["-d".to_string(), "100".to_string()]);
}
args.extend(key_events);
args
}
fn modifier_keycode(key: &str) -> Option<u16> {
match normalize_key(key).as_str() {
"ctrl" | "control" => Some(29),
"alt" | "option" => Some(56),
"shift" => Some(42),
"meta" | "super" | "cmd" | "command" => Some(125),
_ => None,
}
}
fn keycode(key: &str) -> Option<u16> {
match normalize_key(key).as_str() {
"enter" | "return" => Some(28),
"escape" | "esc" => Some(1),
"tab" => Some(15),
"backspace" => Some(14),
"delete" | "del" => Some(111),
"space" => Some(57),
"home" => Some(102),
"end" => Some(107),
"pageup" | "page_up" => Some(104),
"pagedown" | "page_down" => Some(109),
"arrowleft" | "left" => Some(105),
"arrowright" | "right" => Some(106),
"arrowup" | "up" => Some(103),
"arrowdown" | "down" => Some(108),
"f1" => Some(59),
"f2" => Some(60),
"f3" => Some(61),
"f4" => Some(62),
"f5" => Some(63),
"f6" => Some(64),
"f7" => Some(65),
"f8" => Some(66),
"f9" => Some(67),
"f10" => Some(68),
"f11" => Some(87),
"f12" => Some(88),
value if value.len() == 1 => keycode_for_ascii(value.as_bytes()[0] as char),
_ => None,
}
}
fn normalize_key(key: &str) -> String {
key.trim().to_ascii_lowercase().replace(['-', ' '], "")
}
fn keycode_for_ascii(value: char) -> Option<u16> {
match value {
'a' => Some(30),
'b' => Some(48),
'c' => Some(46),
'd' => Some(32),
'e' => Some(18),
'f' => Some(33),
'g' => Some(34),
'h' => Some(35),
'i' => Some(23),
'j' => Some(36),
'k' => Some(37),
'l' => Some(38),
'm' => Some(50),
'n' => Some(49),
'o' => Some(24),
'p' => Some(25),
'q' => Some(16),
'r' => Some(19),
's' => Some(31),
't' => Some(20),
'u' => Some(22),
'v' => Some(47),
'w' => Some(17),
'x' => Some(45),
'y' => Some(21),
'z' => Some(44),
'1' => Some(2),
'2' => Some(3),
'3' => Some(4),
'4' => Some(5),
'5' => Some(6),
'6' => Some(7),
'7' => Some(8),
'8' => Some(9),
'9' => Some(10),
'0' => Some(11),
_ => None,
}
}
fn list_process_apps() -> Vec<AppCandidate> {
let output = Command::new("ps")
.args(["-eo", "pid=,comm=,args="])
.output();
let Ok(output) = output else {
return Vec::new();
};
if !output.status.success() {
return Vec::new();
}
String::from_utf8_lossy(&output.stdout)
.lines()
.filter_map(parse_process_line)
.filter(|app| looks_like_desktop_app(&app.name, &app.command))
.take(50)
.collect()
}
fn parse_process_line(line: &str) -> Option<AppCandidate> {
let trimmed = line.trim();
let mut parts = trimmed.splitn(3, char::is_whitespace);
let pid = parts.next()?.parse().ok()?;
let name = parts.next()?.to_string();
let command = parts.next().unwrap_or("").trim().to_string();
Some(AppCandidate { name, pid, command })
}
fn looks_like_desktop_app(name: &str, command: &str) -> bool {
let haystack = format!("{name} {command}").to_ascii_lowercase();
[
"codex",
"electron",
"chrome",
"chromium",
"firefox",
"brave",
"code",
"gnome-terminal",
"ptyxis",
"kgx",
"nautilus",
"slack",
"discord",
"spotify",
"obsidian",
]
.iter()
.any(|needle| haystack.contains(needle))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::atspi_tree::{AccessibilityAction, Bounds};
use crate::windows::{WindowBounds, GNOME_SHELL_EXTENSION_BACKEND, NIRI_BACKEND};
use std::os::unix::fs::PermissionsExt;
#[test]
fn mcp_server_version_matches_cargo() {
let info = rmcp::ServerHandler::get_info(&ComputerUseLinux::default());
assert_eq!(info.server_info.version, env!("CARGO_PKG_VERSION"));
}
/// Restores an environment variable when a test ends, including on panic.
struct EnvVarGuard {
key: &'static str,
original: Option<std::ffi::OsString>,
}
impl EnvVarGuard {
fn set(key: &'static str, value: &str) -> Self {
let original = std::env::var_os(key);
std::env::set_var(key, value);
Self { key, original }
}
fn unset(key: &'static str) -> Self {
let original = std::env::var_os(key);
std::env::remove_var(key);
Self { key, original }
}
}
impl Drop for EnvVarGuard {
fn drop(&mut self) {
match &self.original {
Some(value) => std::env::set_var(self.key, value),
None => std::env::remove_var(self.key),
}
}
}
/// The run_shell tests change the same process-wide variable, so they
/// take turns instead of racing each other under the parallel test runner.
static SHELL_ENV_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
/// Runs pointer dispatch tests in a child process so backend overrides
/// cannot race other tests. All command programs are harmless stubs; the
/// private datagram socket and invalid D-Bus address cannot reach a desktop.
struct PointerChildHarness {
dir: std::path::PathBuf,
socket_path: std::path::PathBuf,
_socket: std::os::unix::net::UnixDatagram,
}
impl PointerChildHarness {
fn new(prefix: &str) -> Self {
// The longest socket a run names: the scroll test's missing.sock.
let dir = crate::test_sockets::socket_fixture(
&format!(
"{prefix}-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
),
"missing.sock",
);
std::fs::create_dir(&dir).unwrap();
std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).unwrap();
let socket_path = dir.join("input.sock");
let socket = std::os::unix::net::UnixDatagram::bind(&socket_path).unwrap();
for program in ["ydotool", "xdotool", "systemctl", "gnome-screenshot"] {
let path = dir.join(program);
std::fs::write(
&path,
r#"#!/bin/sh
case "${0##*/}" in
systemctl) exit 0 ;;
gnome-screenshot) printf 'capture\n' >> "$CUL_POINTER_TEST_CAPTURE_LOG"; exit 1 ;;
ydotool)
case "$1" in help|--help) printf '%s\n' click mousemove type key debug; exit 0 ;; esac
# Like ydotool, fail when no daemon socket exists.
[ -S "$YDOTOOL_SOCKET" ] || { echo "ydotool: no daemon socket at $YDOTOOL_SOCKET" >&2; exit 2; }
# Capability probes use their own socket. Record only dispatched input.
[ "$YDOTOOL_SOCKET" = "$CUL_POINTER_TEST_SOCKET" ] || exit 0 ;;
esac
printf '%s %s\n' "${0##*/}" "$*" >> "$CUL_POINTER_TEST_LOG"
"#,
)
.unwrap();
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o700)).unwrap();
}
Self {
dir,
socket_path,
_socket: socket,
}
}
/// Run the child test `child` in `mode` on a Wayland session with the
/// uinput pointer disabled and no backend forced; `configure` changes
/// that.
fn run(&self, child: &str, mode: &str, configure: impl FnOnce(&mut Command)) -> ChildRun {
let log = self.dir.join(format!("{mode}.log"));
let capture_log = self.dir.join(format!("{mode}-capture.log"));
let mut command = Command::new(std::env::current_exe().unwrap());
command
.args(["--exact", child, "--nocapture"])
.env_clear()
.env("PATH", &self.dir)
.env("HOME", &self.dir)
.env("CUL_POINTER_TEST_MODE", mode)
.env("CUL_POINTER_TEST_LOG", &log)
.env("CUL_POINTER_TEST_CAPTURE_LOG", &capture_log)
.env("COMPUTER_USE_LINUX_SCREENSHOT_BACKEND", "gnome-screenshot")
.env("CUL_POINTER_TEST_SOCKET", &self.socket_path)
.env("YDOTOOL_SOCKET", &self.socket_path)
.env("XDG_RUNTIME_DIR", &self.dir)
.env(
"DBUS_SESSION_BUS_ADDRESS",
format!("unix:path={}/no-bus", self.dir.display()),
)
.env("DISPLAY", ":9876")
.env("WAYLAND_DISPLAY", "no-wayland")
.env("XDG_SESSION_TYPE", "wayland")
.env("CU_DISABLE_ABS_POINTER", "1")
.env("COMPUTER_USE_LINUX_FORCE_YDOTOOL_POINTER", "0")
.env("COMPUTER_USE_LINUX_FORCE_PORTAL_POINTER", "0");
for key in [
"DESKTOP_SESSION",
"HYPRLAND_INSTANCE_SIGNATURE",
"XAUTHORITY",
"XDG_SESSION_DESKTOP",
"XDG_CURRENT_DESKTOP",
] {
command.env(key, "pointer-test");
}
configure(&mut command);
let output = crate::command_runner::output_blocking_with_timeout(
&mut command,
"pointer backend regression",
Duration::from_secs(15),
)
.unwrap();
assert!(
output.status.success(),
"{mode}: {}{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
ChildRun {
commands: std::fs::read_to_string(&log).unwrap_or_default(),
captured: capture_log.exists(),
}
}
}
struct ChildRun {
/// Input commands the stubs recorded.
commands: String,
/// Whether the child asked the screenshot stub for a capture.
captured: bool,
}
impl Drop for PointerChildHarness {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.dir);
}
}
#[test]
fn pointer_coordinate_backend_dispatch_is_safe() {
let harness = PointerChildHarness::new("cul-pointer-safety");
for mode in [
"wayland-default",
"wayland-uncached",
"wayland-ydotool",
"wayland-portal",
"x11",
] {
let run = harness.run(
"server::tests::pointer_coordinate_backend_child",
mode,
|command| match mode {
"wayland-ydotool" => {
command.env("COMPUTER_USE_LINUX_FORCE_YDOTOOL_POINTER", "1");
}
"wayland-portal" => {
command.env("COMPUTER_USE_LINUX_FORCE_PORTAL_POINTER", "1");
}
"x11" => {
command
.env("WAYLAND_DISPLAY", "")
.env("XDG_SESSION_TYPE", "x11");
}
_ => {}
},
);
assert!(
!run.captured,
"{mode}: a pointer warning triggered screenshot capture"
);
let commands = run.commands;
match mode {
"wayland-portal" => assert!(commands.is_empty(), "{commands}"),
"x11" => {
assert!(
commands.contains("xdotool mousemove -- 5536 700 click"),
"{commands}"
);
assert!(
commands.contains("xdotool mousemove -- 100 100"),
"{commands}"
);
assert!(commands.contains("xdotool mouseup 1"), "{commands}");
assert!(!commands.contains("ydotool"), "{commands}");
}
_ => assert_eq!(commands, "ydotool mousemove --wheel -- 0 -5\n"),
}
}
}
/// An untargeted scroll keeps the portal start error when its ydotool
/// fallback fails too (#389).
#[test]
fn untargeted_scroll_keeps_portal_start_error() {
let harness = PointerChildHarness::new("cul-scroll-portal-error");
// Without a ydotool daemon the portal is preferred, cannot start on
// the invalid bus, and the untargeted scroll falls back to ydotool.
let missing_socket = harness.dir.join("missing.sock");
let run = harness.run(
"server::tests::pointer_dispatch_child",
"portal-start-failure",
|command| {
command.env("YDOTOOL_SOCKET", &missing_socket);
},
);
assert!(!run.captured);
}
/// A coordinate scroll on Wayland sends the move and the wheel through
/// the uinput pointer in kernel signs, with no ydotool daemon running.
#[test]
fn uinput_scroll_sends_kernel_signed_wheel_without_ydotool() {
let harness = PointerChildHarness::new("cul-uinput-scroll");
let missing_socket = harness.dir.join("missing.sock");
let run = harness.run(
"server::tests::pointer_dispatch_child",
"uinput-scroll-direction",
|command| {
command
.env("CU_DISABLE_ABS_POINTER", "0")
.env("YDOTOOL_SOCKET", &missing_socket);
},
);
assert!(!run.captured);
assert!(run.commands.is_empty(), "{}", run.commands);
}
/// An element call naming a replaced snapshot is refused before any
/// window is raised or input is sent, on every element tool.
#[test]
fn element_calls_naming_a_replaced_snapshot_send_nothing() {
let harness = PointerChildHarness::new("cul-stale-snapshot");
let run = harness.run(
"server::tests::pointer_dispatch_child",
"stale-snapshot",
|_| {},
);
assert!(!run.captured);
assert!(run.commands.is_empty(), "{}", run.commands);
}
/// A uinput device sized for an earlier desktop is not reused: the
/// compositor spreads its range across the current layout, so every
/// point would land scaled and its result would carry no note (#389).
#[test]
fn stale_uinput_pointer_is_replaced() {
let harness = PointerChildHarness::new("cul-uinput-stale");
let run = harness.run(
"server::tests::pointer_dispatch_child",
"uinput-stale",
|command| {
command.env("CU_DISABLE_ABS_POINTER", "0");
},
);
// Unit-test builds refuse to create a real device, so each stale
// action fails closed. Only the click with no device asks for a
// capture, which the stub refuses. Nothing reaches ydotool.
assert!(run.captured);
assert!(run.commands.is_empty(), "{}", run.commands);
}
/// A harness whose xrandr and swaymsg report two monitors of different
/// heights, 2560x1440 at 0,0 and 1920x1080 at 2560,0, which leave
/// 2560..4480 x 1080..1440 of a 4480x1440 capture uncovered. Each layout
/// read is logged to the returned file. xdotool fails any command that
/// moves to 5001,1301, so a failed dispatch can be checked.
fn monitor_gap_harness(prefix: &str) -> (PointerChildHarness, std::path::PathBuf) {
let harness = PointerChildHarness::new(prefix);
let xrandr = harness.dir.join("xrandr");
std::fs::write(
&xrandr,
r#"#!/bin/sh
printf '%s\n' "$*" >> "$CUL_POINTER_TEST_LAYOUT_LOG"
printf 'Monitors: 2\n 0: +*DP-1 2560/597x1440/336+0+0 DP-1\n 1: +HDMI-1 1920/527x1080/296+2560+0 HDMI-1\n'
"#,
)
.unwrap();
std::fs::set_permissions(&xrandr, std::fs::Permissions::from_mode(0o700)).unwrap();
let swaymsg = harness.dir.join("swaymsg");
std::fs::write(
&swaymsg,
r#"#!/bin/sh
printf 'swaymsg %s\n' "$*" >> "$CUL_POINTER_TEST_LAYOUT_LOG"
printf '[{"active":true,"rect":{"x":0,"y":0,"width":2560,"height":1440},"scale":1.0},{"active":true,"rect":{"x":2560,"y":0,"width":1920,"height":1080},"scale":1.0}]'
"#,
)
.unwrap();
std::fs::set_permissions(&swaymsg, std::fs::Permissions::from_mode(0o700)).unwrap();
let xdotool = harness.dir.join("xdotool");
std::fs::write(
&xdotool,
r#"#!/bin/sh
printf '%s %s\n' "${0##*/}" "$*" >> "$CUL_POINTER_TEST_LOG"
case "$*" in *"-- 5001 1301"*) echo "xdotool: test failure" >&2; exit 1 ;; esac
"#,
)
.unwrap();
std::fs::set_permissions(&xdotool, std::fs::Permissions::from_mode(0o700)).unwrap();
let layout_log = harness.dir.join("layout.log");
(harness, layout_log)
}
/// A coordinate in the gap of [`monitor_gap_harness`] gets a note on the
/// X11 and uinput routes; a coordinate on either monitor does not. The
/// note reads the layout once per capture and takes no capture (#391).
fn run_monitor_gap_child(mode: &str) {
let (harness, layout_log) = monitor_gap_harness("cul-monitor-gap");
let run = harness.run("server::tests::pointer_dispatch_child", mode, |command| {
command.env("CUL_POINTER_TEST_LAYOUT_LOG", &layout_log);
if mode == "x11-monitor-gap" {
command
.env("WAYLAND_DISPLAY", "")
.env("XDG_SESSION_TYPE", "x11");
} else {
command
.env("CU_DISABLE_ABS_POINTER", "0")
.env("SWAYSOCK", harness.dir.join("sway.sock"));
}
});
assert!(!run.captured, "{mode}: a note took a capture");
// The child caches two captures and acts on each. On Wayland the
// note never runs xrandr, which could start Xwayland.
let reads = if mode == "x11-monitor-gap" {
"--listactivemonitors\n--listactivemonitors\n"
} else {
"swaymsg -t get_outputs -r\nswaymsg -t get_outputs -r\n"
};
assert_eq!(
std::fs::read_to_string(&layout_log).unwrap_or_default(),
reads,
"{mode}"
);
let sent = if mode == "x11-monitor-gap" {
"xdotool mousemove -- 3000 1200 click"
} else {
// The uinput route emits the wheel from the absolute pointer
// device itself, so no wheel command is spawned.
assert!(
!run.commands.contains("mousemove --wheel"),
"{mode}: {}",
run.commands
);
return;
};
assert!(run.commands.contains(sent), "{mode}: {}", run.commands);
}
#[test]
fn x11_monitor_gap_coordinate_gets_a_note() {
run_monitor_gap_child("x11-monitor-gap");
}
#[test]
fn uinput_monitor_gap_coordinate_gets_a_note() {
run_monitor_gap_child("uinput-monitor-gap");
}
/// A drag checks the two endpoints its route sent. With one endpoint off
/// the capture and the other in the gap of [`monitor_gap_harness`], the
/// result names both; a drag between two on-monitor points gets no note
/// (#398).
fn run_drag_gap_child(mode: &str) {
let (harness, layout_log) = monitor_gap_harness("cul-drag-gap");
let run = harness.run("server::tests::pointer_dispatch_child", mode, |command| {
command.env("CUL_POINTER_TEST_LAYOUT_LOG", &layout_log);
if mode == "uinput-drag-gap" {
command
.env("CU_DISABLE_ABS_POINTER", "0")
.env("SWAYSOCK", harness.dir.join("sway.sock"));
} else {
command
.env("WAYLAND_DISPLAY", "")
.env("XDG_SESSION_TYPE", "x11");
}
if mode == "ydotool-drag-gap" {
command.env("COMPUTER_USE_LINUX_FORCE_YDOTOOL_POINTER", "1");
}
});
assert!(!run.captured, "{mode}: a note took a capture");
// One capture, one layout read; never xrandr on Wayland.
let read = if mode == "uinput-drag-gap" {
"swaymsg -t get_outputs -r\n"
} else {
"--listactivemonitors\n"
};
assert_eq!(
std::fs::read_to_string(&layout_log).unwrap_or_default(),
read,
"{mode}"
);
match mode {
"x11-drag-gap" => {
assert!(
run.commands
.starts_with("xdotool mousemove -- 5000 500\nxdotool mousedown 1\n"),
"{}",
run.commands
);
assert!(!run.commands.contains("ydotool"), "{}", run.commands);
}
"ydotool-drag-gap" => {
assert!(
run.commands.starts_with(
"ydotool mousemove --absolute -- 5000 500\nydotool click 0x40\n"
),
"{}",
run.commands
);
assert!(!run.commands.contains("xdotool"), "{}", run.commands);
}
_ => assert!(run.commands.is_empty(), "{}", run.commands),
}
}
#[test]
fn x11_drag_endpoints_get_off_screen_and_gap_notes() {
run_drag_gap_child("x11-drag-gap");
}
#[test]
fn ydotool_x11_drag_endpoints_get_off_screen_and_gap_notes() {
run_drag_gap_child("ydotool-drag-gap");
}
#[test]
fn uinput_drag_endpoints_get_off_screen_and_gap_notes() {
run_drag_gap_child("uinput-drag-gap");
}
#[tokio::test]
async fn pointer_dispatch_child() {
let Ok(mode) = std::env::var("CUL_POINTER_TEST_MODE") else {
return;
};
let backend = ComputerUseLinux::default();
match mode.as_str() {
"portal-start-failure" => {
let scroll: ScrollParams =
serde_json::from_value(serde_json::json!({"direction":"down"})).unwrap();
let result = backend.dispatch_scroll(scroll).await.0 .0;
assert!(!result.ok, "{}", result.message);
let (ydotool_error, portal_error) = result
.message
.split_once(
" The scroll fell back to ydotool because the remote desktop portal could not start: ",
)
.unwrap_or_else(|| panic!("{}", result.message));
assert!(
ydotool_error.contains("no daemon socket"),
"{}",
result.message
);
assert!(!portal_error.trim().is_empty(), "{}", result.message);
}
"uinput-stale" => {
use crate::abs_pointer::{tests::RecordedEvents, AbsPointer};
let pointer = |width, height| {
Some(AbsPointer::with_events(
RecordedEvents::default(),
width,
height,
))
};
backend.cache_desktop_size(1920, 1080);
let click: ClickParams =
serde_json::from_value(serde_json::json!({"x":2500,"y":500})).unwrap();
let scroll: ScrollParams = serde_json::from_value(
serde_json::json!({"x":2500,"y":500,"direction":"down"}),
)
.unwrap();
let replace_failed = "was not sent: cached absolute-pointer geometry was stale for the 1920x1080 raw capture and the device could not be recreated";
// The message names the route the retry takes (#396); the
// last click below takes it.
let retry = "The stale device was removed. A retry sizes a new device to a fresh screenshot; if that fails, it is not sent, since ydotool cannot position the pointer accurately on Wayland.";
*backend.abs_pointer.lock().unwrap() = pointer(3840, 1080);
let result = backend.dispatch_click(click.clone()).await.0 .0;
assert!(!result.ok, "{}", result.message);
assert!(
result.message.contains(replace_failed),
"{}",
result.message
);
assert!(result.message.ends_with(retry), "{}", result.message);
assert!(backend.abs_pointer.lock().unwrap().is_none());
*backend.abs_pointer.lock().unwrap() = pointer(3840, 1080);
let result = backend.dispatch_scroll(scroll).await.0 .0;
assert!(!result.ok, "{}", result.message);
assert!(
result.message.contains(replace_failed),
"{}",
result.message
);
assert!(result.message.ends_with(retry), "{}", result.message);
assert!(backend.abs_pointer.lock().unwrap().is_none());
*backend.abs_pointer.lock().unwrap() = pointer(3840, 1080);
let drag = DragParams {
start_x: 100,
start_y: 100,
end_x: 200,
end_y: 200,
..Default::default()
};
let result = backend.dispatch_drag(drag).await.0;
assert!(!result.ok, "{}", result.message);
assert!(
result.message.contains(replace_failed),
"{}",
result.message
);
assert!(result.message.ends_with(retry), "{}", result.message);
assert!(backend.abs_pointer.lock().unwrap().is_none());
// A KWin element click names the device, not the mapping.
*backend.abs_pointer.lock().unwrap() = pointer(3840, 1080);
let prepared = PreparedKwinElementClick {
element_index: 4,
point: KwinElementClickPoint {
logical: (10, 10),
capture: (10, 10),
},
capture_size: (1920, 1080),
};
let output = backend
.kwin_abs_pointer_ready(prepared, &None)
.await
.unwrap_err();
assert_eq!(
output.message,
"click was not sent: KWin mapped element_index 4, but the uinput absolute pointer could not be used: cached absolute-pointer geometry was stale for the 1920x1080 raw capture and the device could not be recreated: unit tests do not create real uinput devices. No pointer event was emitted. The stale device was removed. A retry of element_index 4 sizes a new device to a fresh screenshot; if that fails, it is not sent, since ydotool cannot position the pointer accurately on Wayland."
);
assert!(backend.abs_pointer.lock().unwrap().is_none());
// Replacing a stale device needs no capture.
let capture_log = std::env::var("CUL_POINTER_TEST_CAPTURE_LOG").unwrap();
assert!(!Path::new(&capture_log).exists());
// With no device to replace, a failed capture still leaves
// the other routes; here the Wayland refusal.
let result = backend.dispatch_click(click.clone()).await.0 .0;
assert!(!result.ok, "{}", result.message);
assert!(
result
.message
.contains("cannot position the pointer accurately on Wayland"),
"{}",
result.message
);
// A device that matches the capture clamps an off-capture
// point and says where it landed.
*backend.abs_pointer.lock().unwrap() = pointer(1920, 1080);
let result = backend.dispatch_click(click).await.0 .0;
assert!(result.ok, "{}", result.message);
assert!(
result
.message
.contains("Requested coordinate 2500,500 was clamped to 1919,500"),
"{}",
result.message
);
}
"uinput-scroll-direction" => {
use crate::abs_pointer::{tests::RecordedEvents, AbsPointer};
use evdev::RelativeAxisCode;
let events = RecordedEvents::default();
backend.cache_desktop_size(1920, 1080);
*backend.abs_pointer.lock().unwrap() =
Some(AbsPointer::with_events(events.clone(), 1920, 1080));
for direction in ["left", "right", "up", "down"] {
let scroll: ScrollParams = serde_json::from_value(
serde_json::json!({"x":500,"y":400,"direction":direction}),
)
.unwrap();
let result = backend.dispatch_scroll(scroll).await.0 .0;
assert!(result.ok, "{direction}: {}", result.message);
assert_eq!(
result.message,
"Pointer positioned and wheel sent through the uinput absolute pointer."
);
}
// The move comes first, then the wheel, from one device.
let abs = evdev::EventType::ABSOLUTE.0;
let rel = evdev::EventType::RELATIVE.0;
assert_eq!(
events.all_events()[..4],
[
(abs, evdev::AbsoluteAxisCode::ABS_X.0, 500),
(abs, evdev::AbsoluteAxisCode::ABS_Y.0, 400),
(rel, RelativeAxisCode::REL_HWHEEL.0, -5),
(rel, RelativeAxisCode::REL_WHEEL.0, 0),
]
);
// Each wheel call writes both axes; the unused one is 0.
let wheel: Vec<(u16, i32)> = events
.wheel_events()
.into_iter()
.filter(|(_, value)| *value != 0)
.collect();
assert_eq!(
wheel,
[
(RelativeAxisCode::REL_HWHEEL.0, -5),
(RelativeAxisCode::REL_HWHEEL.0, 5),
(RelativeAxisCode::REL_WHEEL.0, 5),
(RelativeAxisCode::REL_WHEEL.0, -5),
]
);
}
"stale-snapshot" => {
let node = AccessibilityNode {
index: 0,
parent_index: None,
depth: 1,
object_ref: ":1.10/org/a11y/atspi/accessible/7".to_string(),
role: "entry".to_string(),
name: Some("Name".to_string()),
description: None,
child_count: 0,
bounds: Some(crate::atspi_tree::Bounds {
x: 100,
y: 100,
width: 80,
height: 20,
}),
states: vec!["enabled".to_string(), "editable".to_string()],
actions: Vec::new(),
value: None,
text: None,
supports_editable_text: true,
};
let first = backend.cache_snapshot(std::slice::from_ref(&node), None);
backend.cache_snapshot(std::slice::from_ref(&node), None);
let refused = |action: &str, ok: bool, message: &str| {
assert!(!ok, "{action}: {message}");
assert!(
message.starts_with(&format!(
"{action} was not sent: snapshot_id {first:?} is not the cached snapshot"
)),
"{message}"
);
};
// The window target is refused before it is raised.
let click: ClickParams = serde_json::from_value(serde_json::json!({
"element_index": 0, "snapshot_id": first, "window_id": 7
}))
.unwrap();
let result = backend.dispatch_click(click).await.0 .0;
refused("click", result.ok, &result.message);
let click: ClickParams = serde_json::from_value(serde_json::json!({
"role": "entry", "snapshot_id": first
}))
.unwrap();
let result = backend.dispatch_click(click).await.0 .0;
refused("click", result.ok, &result.message);
let scroll: ScrollParams = serde_json::from_value(serde_json::json!({
"element_index": 0, "snapshot_id": first, "direction": "down"
}))
.unwrap();
let result = backend.dispatch_scroll(scroll).await.0 .0;
refused("scroll", result.ok, &result.message);
let set_value: SetValueParams = serde_json::from_value(serde_json::json!({
"element_index": 0, "snapshot_id": first, "value": "typed"
}))
.unwrap();
let (result, verified, _) = backend.dispatch_set_value(set_value).await;
refused("set_value", result.ok, &result.message);
assert_eq!(verified, None);
let action: ActionParams = serde_json::from_value(serde_json::json!({
"element_index": 0, "snapshot_id": first
}))
.unwrap();
let result = backend.perform_element_action(&action, None).await.0 .0;
refused("perform_action", result.ok, &result.message);
// Scroll checks before raising its window too, and a lone x
// still takes the element's point.
let scroll: ScrollParams = serde_json::from_value(serde_json::json!({
"element_index": 0, "snapshot_id": first, "direction": "down",
"window_id": 7
}))
.unwrap();
let result = backend.dispatch_scroll(scroll).await.0 .0;
refused("scroll", result.ok, &result.message);
let scroll: ScrollParams = serde_json::from_value(serde_json::json!({
"element_index": 0, "snapshot_id": first, "direction": "down", "x": 500
}))
.unwrap();
let result = backend.dispatch_scroll(scroll).await.0 .0;
refused("scroll", result.ok, &result.message);
// The current id, and a full point that wins over the index,
// pass the snapshot check and reach the input routes, which
// refuse here: this test session has no usable pointer.
let current = backend.cached_snapshot_id();
let passed = |action: &str, message: &str| {
assert!(
!message.starts_with(&format!("{action} was not sent: snapshot_id")),
"{message}"
);
};
for click in [
serde_json::json!({"element_index": 0, "snapshot_id": current}),
serde_json::json!({"element_index": 0, "snapshot_id": first, "x": 140, "y": 110}),
] {
let click: ClickParams = serde_json::from_value(click).unwrap();
passed("click", &backend.dispatch_click(click).await.0 .0.message);
}
for scroll in [
serde_json::json!({"element_index": 0, "snapshot_id": current, "direction": "down"}),
serde_json::json!({"element_index": 0, "snapshot_id": first, "direction": "down", "x": 140, "y": 110}),
] {
let scroll: ScrollParams = serde_json::from_value(scroll).unwrap();
passed(
"scroll",
&backend.dispatch_scroll(scroll).await.0 .0.message,
);
}
let set_value: SetValueParams = serde_json::from_value(serde_json::json!({
"element_index": 0, "snapshot_id": current, "value": "typed"
}))
.unwrap();
passed(
"set_value",
&backend.dispatch_set_value(set_value).await.0.message,
);
}
"x11-monitor-gap" | "uinput-monitor-gap" => {
use crate::abs_pointer::{tests::RecordedEvents, AbsPointer};
let uinput = mode == "uinput-monitor-gap";
let click = |x: i32, y: i32| -> ClickParams {
serde_json::from_value(serde_json::json!({"x":x,"y":y})).unwrap()
};
backend.cache_desktop_size(4480, 1440);
if uinput {
*backend.abs_pointer.lock().unwrap() = Some(AbsPointer::with_events(
RecordedEvents::default(),
4480,
1440,
));
}
let gap = "WARNING: coordinate 3000,1200 is on no monitor of the captured desktop (4480x1440); the input landed at a monitor edge or on no visible pixel.";
let result = backend.dispatch_click(click(3000, 1200)).await.0 .0;
assert!(result.ok, "{}", result.message);
assert!(result.message.ends_with(gap), "{}", result.message);
for (x, y) in [(3000, 500), (100, 1300)] {
let result = backend.dispatch_click(click(x, y)).await.0 .0;
assert!(result.ok, "{}", result.message);
assert!(!result.message.contains("WARNING"), "{}", result.message);
}
let scroll: ScrollParams = serde_json::from_value(
serde_json::json!({"x":3000,"y":1200,"direction":"down"}),
)
.unwrap();
let result = backend.dispatch_scroll(scroll).await.0 .0;
assert!(result.ok, "{}", result.message);
assert!(result.message.ends_with(gap), "{}", result.message);
let result = backend.dispatch_click(click(5000, 1300)).await.0 .0;
assert!(result.ok, "{}", result.message);
let off_capture = if uinput {
// The device clamps the point into the gap.
"Requested coordinate 5000,1300 was clamped to 4479,1300 by the uinput absolute pointer. WARNING: coordinate 4479,1300 is on no monitor of the captured desktop (4480x1440); the input landed at a monitor edge or on no visible pixel."
} else {
// The X server clamps the point into the gap.
"WARNING: coordinate 5000,1300 is outside the captured desktop (4480x1440); the X server keeps the pointer on the screen, but the nearest point there, 4479,1300, is on no monitor, so the input landed at a monitor edge or on no visible pixel."
};
assert!(result.message.ends_with(off_capture), "{}", result.message);
if !uinput {
// A failed dispatch sent nothing, so it gets no warning.
let result = backend.dispatch_click(click(5001, 1301)).await.0 .0;
assert!(!result.ok, "{}", result.message);
assert!(!result.message.contains("WARNING"), "{}", result.message);
}
// A new capture reads the layout again.
backend.cache_desktop_size(4480, 1440);
let result = backend.dispatch_click(click(3000, 1200)).await.0 .0;
assert!(result.message.ends_with(gap), "{}", result.message);
// Where a retry goes after a failed device replacement.
let route = if uinput {
"it is not sent, since ydotool cannot position the pointer accurately on Wayland"
} else {
"it goes through ydotool"
};
assert_eq!(backend.pointer_retry_route().await, route);
if uinput {
std::env::set_var("COMPUTER_USE_LINUX_FORCE_PORTAL_POINTER", "1");
assert_eq!(
backend.pointer_retry_route().await,
"it goes through the remote desktop portal"
);
}
}
"x11-drag-gap" | "ydotool-drag-gap" | "uinput-drag-gap" => {
use crate::abs_pointer::{tests::RecordedEvents, AbsPointer};
let uinput = mode == "uinput-drag-gap";
let drag = |start: (i32, i32), end: (i32, i32)| DragParams {
start_x: start.0,
start_y: start.1,
end_x: end.0,
end_y: end.1,
..Default::default()
};
backend.cache_desktop_size(4480, 1440);
if uinput {
*backend.abs_pointer.lock().unwrap() = Some(AbsPointer::with_events(
RecordedEvents::default(),
4480,
1440,
));
}
let gap = |name: &str| {
format!("WARNING: {name} coordinate 3000,1200 is on no monitor of the captured desktop (4480x1440); the input landed at a monitor edge or on no visible pixel.")
};
// Off the capture, onto the right monitor, then into the gap.
let result = backend
.dispatch_drag(drag((5000, 500), (3000, 1200)))
.await
.0;
assert!(result.ok, "{}", result.message);
let start = if uinput {
"Requested start coordinate 5000,500 was clamped to 4479,500 by the uinput absolute pointer."
} else {
"WARNING: start coordinate 5000,500 is outside the captured desktop (4480x1440); the X server keeps the pointer on the screen, so the input landed at an edge instead."
};
assert!(
result.message.ends_with(&format!("{start} {}", gap("end"))),
"{}",
result.message
);
// From the gap, then off the capture, which lands in the gap.
let result = backend
.dispatch_drag(drag((3000, 1200), (5000, 1300)))
.await
.0;
assert!(result.ok, "{}", result.message);
let end = if uinput {
"Requested end coordinate 5000,1300 was clamped to 4479,1300 by the uinput absolute pointer. WARNING: end coordinate 4479,1300 is on no monitor of the captured desktop (4480x1440); the input landed at a monitor edge or on no visible pixel."
} else {
"WARNING: end coordinate 5000,1300 is outside the captured desktop (4480x1440); the X server keeps the pointer on the screen, but the nearest point there, 4479,1300, is on no monitor, so the input landed at a monitor edge or on no visible pixel."
};
assert!(
result.message.ends_with(&format!("{} {end}", gap("start"))),
"{}",
result.message
);
// The path crosses the gap, but both endpoints are on a
// monitor.
let result = backend
.dispatch_drag(drag((2000, 1400), (4400, 1000)))
.await
.0;
assert!(result.ok, "{}", result.message);
assert!(!result.message.contains("WARNING"), "{}", result.message);
assert!(!result.message.contains("clamped"), "{}", result.message);
if mode == "x11-drag-gap" {
// A failed drag gets no note.
let result = backend
.dispatch_drag(drag((5001, 1301), (3000, 1200)))
.await
.0;
assert!(!result.ok, "{}", result.message);
assert!(!result.message.contains("WARNING"), "{}", result.message);
}
}
other => panic!("unknown mode {other}"),
}
}
/// Send newline-delimited JSON-RPC frames to an in-process server and
/// collect every response frame until the server closes the stream.
async fn mcp_exchange(frames: &[serde_json::Value]) -> Vec<serde_json::Value> {
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
let (server_transport, client_transport) = tokio::io::duplex(1 << 20);
let server = tokio::spawn(async move {
if let Ok(running) = ComputerUseLinux::default().serve(server_transport).await {
let _ = running.waiting().await;
}
});
let (read, mut write) = tokio::io::split(client_transport);
let mut input = String::new();
for frame in frames {
input.push_str(&frame.to_string());
input.push('\n');
}
write.write_all(input.as_bytes()).await.unwrap();
let mut reader = BufReader::new(read);
let mut responses = Vec::new();
let expected = frames
.iter()
.filter(|frame| frame.get("id").is_some())
.count();
while responses.len() < expected {
let mut line = String::new();
let read = tokio::time::timeout(Duration::from_secs(10), reader.read_line(&mut line))
.await
.expect("server should answer in time")
.unwrap();
if read == 0 {
break;
}
let frame: serde_json::Value = serde_json::from_str(&line).unwrap();
if frame.get("id").is_some() {
responses.push(frame);
}
}
drop(write);
drop(reader);
let _ = tokio::time::timeout(Duration::from_secs(10), server).await;
responses
}
#[tokio::test]
async fn bare_discover_probe_falls_back_to_initialize() {
// Hosts probe with `server/discover` before `initialize`. rmcp 2 treated
// any non-initialize first request as fatal and closed the stream.
let responses = mcp_exchange(&[
serde_json::json!({"jsonrpc":"2.0","id":1,"method":"server/discover","params":{}}),
serde_json::json!({"jsonrpc":"2.0","id":2,"method":"initialize","params":{
"protocolVersion":"2025-06-18","capabilities":{},
"clientInfo":{"name":"test-client","version":"1"}}}),
serde_json::json!({"jsonrpc":"2.0","method":"notifications/initialized"}),
serde_json::json!({"jsonrpc":"2.0","id":3,"method":"tools/list","params":{}}),
])
.await;
assert_eq!(responses.len(), 3, "{responses:?}");
assert!(responses[0].get("error").is_some(), "{:?}", responses[0]);
assert_eq!(responses[1]["result"]["protocolVersion"], "2025-06-18");
assert!(!responses[2]["result"]["tools"]
.as_array()
.unwrap()
.is_empty());
}
#[tokio::test]
async fn modern_discover_lifecycle_lists_tools() {
let meta = serde_json::json!({
"io.modelcontextprotocol/protocolVersion": "2026-07-28",
"io.modelcontextprotocol/clientInfo": {"name": "test-client", "version": "1"},
"io.modelcontextprotocol/clientCapabilities": {}
});
let responses = mcp_exchange(&[
serde_json::json!({"jsonrpc":"2.0","id":1,"method":"server/discover","params":{"_meta":meta}}),
serde_json::json!({"jsonrpc":"2.0","id":2,"method":"tools/list","params":{"_meta":meta}}),
])
.await;
assert_eq!(responses.len(), 2, "{responses:?}");
assert!(responses[0]["result"]["supportedVersions"]
.as_array()
.unwrap()
.iter()
.any(|version| version == "2026-07-28"));
let listed = &responses[1]["result"];
assert_eq!(listed["resultType"], "complete");
assert!(listed.get("ttlMs").is_some() && listed.get("cacheScope").is_some());
assert!(!listed["tools"].as_array().unwrap().is_empty());
}
#[tokio::test]
async fn pointer_coordinate_backend_child() {
let Ok(mode) = std::env::var("CUL_POINTER_TEST_MODE") else {
return;
};
let backend = ComputerUseLinux::default();
if mode != "wayland-uncached" {
backend.cache_desktop_size(9376, 1600);
assert!(backend
.off_screen_note_for_point(9376, 700, PointerRoute::X11)
.await
.is_some());
}
// No layout source answers here, so only the capture rectangle counts.
assert!(backend
.off_screen_note_for_point(5536, 700, PointerRoute::X11)
.await
.is_none());
let click: ClickParams =
serde_json::from_value(serde_json::json!({"x":5536,"y":700})).unwrap();
let scroll: ScrollParams =
serde_json::from_value(serde_json::json!({"x":5536,"y":700,"direction":"down"}))
.unwrap();
let drag = DragParams {
start_x: 100,
start_y: 100,
end_x: 200,
end_y: 200,
..Default::default()
};
let results = [
backend.dispatch_click(click).await.0 .0,
backend.dispatch_scroll(scroll).await.0 .0,
backend.dispatch_drag(drag).await.0,
];
for result in results {
if mode == "x11" {
assert!(result.ok, "{}", result.message);
} else {
assert!(!result.ok, "{}", result.message);
let expected = if mode == "wayland-portal" {
"portal could not start"
} else {
"cannot position the pointer accurately on Wayland"
};
assert!(result.message.contains(expected), "{}", result.message);
}
}
let scroll: ScrollParams =
serde_json::from_value(serde_json::json!({"direction":"down"})).unwrap();
let result = backend.dispatch_scroll(scroll).await.0 .0;
assert_eq!(result.ok, mode != "wayland-portal", "{}", result.message);
}
#[test]
fn completion_tool_is_explicitly_opt_in_and_has_side_effect_annotations() {
let server = ComputerUseLinux::default();
assert!(!server
.router_with_completion(false)
.list_all()
.iter()
.any(|t| t.name == "complete_interaction"));
let tools = server.router_with_completion(true).list_all();
let tool = tools
.iter()
.find(|t| t.name == "complete_interaction")
.unwrap();
let value = serde_json::to_value(tool).unwrap();
assert_eq!(value["annotations"]["readOnlyHint"], false);
assert_eq!(value["annotations"]["destructiveHint"], false);
assert_eq!(value["annotations"]["idempotentHint"], false);
assert_eq!(value["annotations"]["openWorldHint"], true);
}
#[tokio::test]
async fn completion_notification_handles_missing_and_failed_backends() {
for path in ["/nonexistent/computer-use-notify", "/bin/false"] {
let result =
send_completion_notification(Path::new(path), Duration::from_secs(2)).await;
assert!(result.ok);
assert_eq!(result.cue, "skipped");
}
let result =
send_completion_notification(Path::new("/bin/true"), Duration::from_secs(2)).await;
assert_eq!(result.cue, "notification");
}
#[tokio::test]
async fn completion_notification_timeout_is_bounded() {
let dir = std::env::temp_dir().join(format!(
"computer-use-notify-test-{}-{:?}",
std::process::id(),
std::time::SystemTime::now()
));
std::fs::create_dir_all(&dir).unwrap();
let program = dir.join("notify");
std::fs::write(&program, "#!/bin/sh\nexec sleep 30\n").unwrap();
std::fs::set_permissions(&program, std::fs::Permissions::from_mode(0o700)).unwrap();
let started = std::time::Instant::now();
let output = send_completion_notification(&program, Duration::from_millis(30)).await;
assert_eq!(output.cue, "skipped");
assert!(started.elapsed() < Duration::from_secs(3));
std::fs::remove_dir_all(dir).unwrap();
}
#[test]
fn exported_tool_schemas_omit_unsigned_integer_formats() {
let tools = ComputerUseLinux::default().mcp_tool_router().list_all();
let value = serde_json::to_value(tools).unwrap();
let mut unsupported = Vec::new();
collect_unsigned_integer_formats(&value, "$", &mut unsupported);
assert!(
unsupported.is_empty(),
"unsupported unsigned integer formats: {unsupported:?}"
);
}
fn collect_unsigned_integer_formats(
value: &serde_json::Value,
path: &str,
unsupported: &mut Vec<String>,
) {
match value {
serde_json::Value::Object(object) => {
if matches!(
object.get("format").and_then(serde_json::Value::as_str),
Some("uint" | "uint8" | "uint16" | "uint32" | "uint64" | "usize")
) {
unsupported.push(path.to_string());
}
for (key, nested) in object {
collect_unsigned_integer_formats(nested, &format!("{path}/{key}"), unsupported);
}
}
serde_json::Value::Array(items) => {
for (index, nested) in items.iter().enumerate() {
collect_unsigned_integer_formats(
nested,
&format!("{path}/{index}"),
unsupported,
);
}
}
_ => {}
}
}
fn node(index: u32, bounds: Option<Bounds>) -> AccessibilityNode {
node_with_actions(index, bounds, Vec::new())
}
#[test]
fn shell_only_tree_explains_missing_renderer_content() {
let mut frame = node(0, None);
frame.role = "frame".to_string();
let nodes = [frame];
let note = shell_only_accessibility_tree_note(&nodes, true, false).unwrap();
assert!(note.contains("--force-renderer-accessibility"));
assert!(note.contains("If this is an Electron/Chromium app"));
assert!(shell_only_accessibility_tree_note(&nodes, false, false).is_none());
assert!(shell_only_accessibility_tree_note(&nodes, true, true).is_none());
assert!(shell_only_accessibility_tree_note(&[], true, false).is_none());
}
#[test]
fn content_and_editable_trees_do_not_get_renderer_hint() {
let mut content = node(0, None);
assert!(shell_only_accessibility_tree_note(&[content.clone()], true, false).is_none());
content.role = "frame".to_string();
content.supports_editable_text = true;
assert!(shell_only_accessibility_tree_note(&[content], true, false).is_none());
}
#[test]
fn native_menu_chrome_does_not_hide_the_renderer_hint() {
let mut frame = node(0, None);
frame.role = "frame".to_string();
let mut menu = node(1, None);
menu.role = "menu item".to_string();
menu.actions.push(AccessibilityAction {
index: 0,
name: "click".to_string(),
description: String::new(),
keybinding: String::new(),
});
assert!(
shell_only_accessibility_tree_note(&[frame.clone(), menu.clone()], true, false)
.is_some()
);
menu.supports_editable_text = true;
assert!(shell_only_accessibility_tree_note(&[frame, menu], true, false).is_none());
}
#[test]
fn native_menu_buttons_are_chrome_only_beneath_a_menu() {
let mut frame = node(0, None);
frame.role = "frame".to_string();
let mut panel = node(1, None);
panel.role = "panel".to_string();
panel.parent_index = Some(0);
let mut menu = node(2, None);
menu.role = "menu bar".to_string();
menu.parent_index = Some(1);
let mut button = node(3, None);
button.role = "button".to_string();
button.parent_index = Some(2);
assert!(shell_only_accessibility_tree_note(
&[frame.clone(), panel.clone(), menu.clone(), button.clone()],
true,
false
)
.is_some());
button.parent_index = Some(1);
assert!(
shell_only_accessibility_tree_note(&[frame, panel, menu, button], true, false)
.is_none()
);
}
#[test]
fn review_hint_keeps_raw_menu_ancestry_after_compaction() {
let mut frame = node(0, None);
frame.role = "frame".to_string();
frame.depth = 1;
let mut panel = node(1, None);
panel.role = "panel".to_string();
panel.name = None;
panel.depth = 2;
panel.parent_index = Some(0);
let mut menu = node(2, None);
menu.role = "menu bar".to_string();
menu.name = None;
menu.depth = 3;
menu.parent_index = Some(1);
let mut button = node(3, None);
button.role = "button".to_string();
button.depth = 4;
button.parent_index = Some(2);
let nodes = vec![frame, panel, menu, button];
let (compacted, note) = compact_accessibility_tree_with_note(nodes.clone(), true, false);
assert_eq!(
compacted.len(),
2,
"unnamed panel and menu bar should be compacted away"
);
assert_eq!(compacted[1].parent_index, Some(0));
assert!(
shell_only_accessibility_tree_note(&compacted, true, false).is_none(),
"compacted data cannot recover the removed menu context"
);
assert!(
note.is_some(),
"raw menu ancestry must determine the observation hint"
);
assert!(
compact_accessibility_tree_with_note(nodes.clone(), false, false)
.1
.is_none()
);
assert!(compact_accessibility_tree_with_note(nodes, true, true)
.1
.is_none());
}
fn node_with_actions(
index: u32,
bounds: Option<Bounds>,
actions: Vec<AccessibilityAction>,
) -> AccessibilityNode {
AccessibilityNode {
index,
parent_index: None,
depth: 0,
object_ref: format!(":1.{index}/org/a11y/atspi/accessible/{index}"),
role: "push button".to_string(),
name: Some(format!("Button {index}")),
description: None,
child_count: 0,
bounds,
states: Vec::new(),
actions,
value: None,
text: None,
supports_editable_text: false,
}
}
fn click_action() -> AccessibilityAction {
AccessibilityAction {
index: 0,
name: "Click".to_string(),
description: "Clicks the element".to_string(),
keybinding: String::new(),
}
}
fn solid_png(width: u32, height: u32) -> Vec<u8> {
let img = image::RgbaImage::from_pixel(width, height, image::Rgba([32, 128, 192, 255]));
let mut out = Vec::new();
image::DynamicImage::ImageRgba8(img)
.write_to(&mut std::io::Cursor::new(&mut out), image::ImageFormat::Png)
.unwrap();
out
}
#[test]
fn targeted_app_state_crops_before_screenshot_payload_resize() {
let raw = RawScreenshotCapture {
mime_type: "image/png".to_string(),
bytes: solid_png(400, 200),
source: "test".to_string(),
width: 400,
height: 200,
};
let capture = prepare_app_state_screenshot(
raw,
Some((50, 20, 200, 100)),
true,
ScreenshotPayloadOptions {
max_width: Some(100),
max_height: Some(100),
max_bytes: Some(1024 * 1024),
..Default::default()
},
)
.unwrap();
assert_eq!(
(capture.coordinate_width, capture.coordinate_height),
(200, 100)
);
assert_eq!((capture.width, capture.height), (100, 50));
}
#[test]
fn unresolved_app_state_target_refuses_full_desktop_screenshot() {
let raw = RawScreenshotCapture {
mime_type: "image/png".to_string(),
bytes: solid_png(400, 200),
source: "test".to_string(),
width: 400,
height: 200,
};
let error =
prepare_app_state_screenshot(raw, None, true, ScreenshotPayloadOptions::default())
.unwrap_err();
assert!(error.to_string().contains("requires a resolved window"));
}
#[test]
fn targeted_app_state_crops_only_visible_part_of_offscreen_window() {
let raw = RawScreenshotCapture {
mime_type: "image/png".to_string(),
bytes: solid_png(400, 200),
source: "test".to_string(),
width: 400,
height: 200,
};
let capture = prepare_app_state_screenshot(
raw,
Some((-50, -40, 100, 100)),
true,
ScreenshotPayloadOptions::default(),
)
.unwrap();
assert_eq!(
(capture.coordinate_width, capture.coordinate_height),
(50, 60)
);
}
fn sample_app_state_output(screenshot: Option<&ScreenshotCapture>) -> GetAppStateOutput {
GetAppStateOutput {
app_name_or_bundle_identifier: None,
window_context: None,
window_error: None,
window_permissions_hint: None,
backend: "linux-atspi".to_string(),
screenshot: screenshot.map(ScreenshotSummary::from),
screenshot_error: None,
snapshot_id: None,
accessibility_tree: TreeNodes::default(),
accessibility_tree_spill: None,
accessibility_tree_raw_count: 0,
tree_scoped: true,
accessibility_tree_truncated: false,
accessibility_error: None,
readiness: crate::diagnostics::ReadinessReport {
can_register_mcp_tools: true,
can_build_accessibility_tree: true,
can_query_windows: true,
can_focus_apps: true,
can_focus_windows: true,
can_send_development_input: true,
can_capture_screenshots: true,
screenshot_capture_status: crate::diagnostics::ScreenshotCaptureStatus::Verified,
recommended_next_step: String::new(),
blockers: Vec::new(),
warnings: Vec::new(),
},
diagnostics: None,
message: "ok".to_string(),
}
}
#[test]
fn app_state_result_emits_screenshot_as_image_block_not_inline_base64() {
// Issue #145: the base64 payload must ride a structured image block so
// hosts count it as vision tokens instead of a 100KB text blob.
let raw = RawScreenshotCapture {
mime_type: "image/png".to_string(),
bytes: solid_png(64, 32),
source: "test".to_string(),
width: 64,
height: 32,
};
let capture =
prepare_app_state_screenshot(raw, None, false, ScreenshotPayloadOptions::default())
.unwrap();
let payload = data_url_payload(&capture.data_url);
let output = sample_app_state_output(Some(&capture));
let result = app_state_result(output, Some(capture)).unwrap();
assert_eq!(result.content.len(), 2);
let image = result.content[0]
.as_image()
.expect("first block is an image");
assert_eq!(image.mime_type, "image/png");
assert_eq!(image.data, payload);
let text = &result.content[1]
.as_text()
.expect("second block is text")
.text;
assert!(
!text.contains(&payload),
"base64 leaked into the text block"
);
assert!(!text.contains("data_url"));
let structured = result.structured_content.expect("structured content");
assert_eq!(structured["screenshot"]["width"], 64);
assert_eq!(structured["screenshot"]["mime_type"], "image/png");
assert!(structured["screenshot"].get("data_url").is_none());
}
#[test]
fn app_state_result_without_screenshot_is_text_only() {
let result = app_state_result(sample_app_state_output(None), None).unwrap();
assert_eq!(result.is_error, Some(false));
assert_eq!(result.content.len(), 1);
assert!(result.content[0].as_text().is_some());
assert_eq!(
result.structured_content.unwrap()["screenshot"],
serde_json::Value::Null
);
}
/// A server whose diagnostics probe labels each report with its run number
/// in `recommended_next_step`, so a result shows which run it came from.
/// Each run, and the live accessibility check, read `accessibility_enabled`.
/// Each run reads `windows_available` for whether a window backend answers.
pub(super) fn server_with_labelled_diagnostics(
accessibility_enabled: fn() -> bool,
windows_available: fn() -> bool,
) -> (ComputerUseLinux, Arc<std::sync::atomic::AtomicUsize>) {
server_with_labelled_reports(accessibility_enabled, move || {
crate::diagnostics_impl::tests::doctor_report_fixture(
accessibility_enabled(),
windows_available(),
)
})
}
/// Like `server_with_labelled_diagnostics`, with each probe's report built
/// by `report`.
pub(super) fn server_with_labelled_reports(
accessibility_enabled: fn() -> bool,
report: impl Fn() -> DoctorReport + Send + Sync + 'static,
) -> (ComputerUseLinux, Arc<std::sync::atomic::AtomicUsize>) {
use std::sync::atomic::{AtomicUsize, Ordering};
let runs = Arc::new(AtomicUsize::new(0));
let counter = Arc::clone(&runs);
let probe = move || {
let run = counter.fetch_add(1, Ordering::SeqCst) + 1;
let mut report = report();
report.readiness.recommended_next_step = format!("probe run {run}");
report
};
let server = ComputerUseLinux {
diagnostics: Arc::new(DiagnosticsCache::new(
Arc::new(probe),
Duration::from_secs(3600),
)),
accessibility_check: AccessibilityCheck(accessibility_enabled),
..Default::default()
};
(server, runs)
}
pub(super) async fn app_state_value(
server: &ComputerUseLinux,
params: serde_json::Value,
) -> serde_json::Value {
let params: GetAppStateParams = serde_json::from_value(params).unwrap();
server
.app_state(params)
.await
.unwrap()
.structured_content
.unwrap()
}
#[test]
fn readiness_is_refreshed_only_when_an_observation_contradicts_it() {
use crate::windowing::registry::{
GNOME_SHELL_EXTENSION_BACKEND, GNOME_SHELL_INTROSPECT_BACKEND, KWIN_BACKEND,
};
use Contradiction::{Failure, Success};
let cached = |can_build_accessibility_tree: bool, can_query_windows: bool| {
crate::diagnostics_impl::tests::doctor_report_fixture(
can_build_accessibility_tree,
can_query_windows,
)
};
let backends = |health: &[(&str, bool)]| {
let mut report = cached(true, true);
for (backend, ok) in health {
report.windowing.backends.insert(
backend.to_string(),
crate::diagnostics::Check {
ok: *ok,
detail: String::new(),
},
);
}
report
};
let observed = |screenshot_failed, window, tree| AppStateObservation {
screenshot_failed,
window,
failed_window_backends: &[],
tree,
};
let resolved_after_failures = |failed_window_backends| AppStateObservation {
screenshot_failed: false,
window: WindowLookup::Resolved,
failed_window_backends,
tree: TreeRead::Nodes,
};
let cases = [
(
"everything worked as the readiness says",
cached(true, true),
observed(false, WindowLookup::Resolved, TreeRead::Nodes),
None,
),
(
"a screenshot failed",
cached(true, true),
observed(true, WindowLookup::NotRequested, TreeRead::Nodes),
Some(Failure),
),
(
"a window lookup failed",
cached(true, true),
observed(false, WindowLookup::Failed, TreeRead::Nodes),
Some(Failure),
),
(
"a window lookup failed where the readiness says it would",
cached(true, false),
observed(false, WindowLookup::Failed, TreeRead::Nodes),
Some(Failure),
),
(
"the tree read failed or found no nodes",
cached(true, true),
observed(false, WindowLookup::NotRequested, TreeRead::Failed),
Some(Failure),
),
(
"the tree was skipped because this call found accessibility off",
cached(false, true),
observed(
false,
WindowLookup::NotRequested,
TreeRead::AccessibilityOff,
),
None,
),
(
"a failure alongside the expected skip",
cached(false, true),
observed(true, WindowLookup::NotRequested, TreeRead::AccessibilityOff),
Some(Failure),
),
(
"a window resolved where the readiness says none can be listed",
cached(false, false),
observed(false, WindowLookup::Resolved, TreeRead::AccessibilityOff),
Some(Success),
),
(
"no window was requested where the readiness says none can be listed",
cached(true, false),
observed(false, WindowLookup::NotRequested, TreeRead::Nodes),
None,
),
(
"a backend the cached report calls healthy failed before another answered",
backends(&[(GNOME_SHELL_EXTENSION_BACKEND, true)]),
resolved_after_failures(&[GNOME_SHELL_EXTENSION_BACKEND]),
Some(Failure),
),
(
"backends the cached report calls unhealthy failed before KWin answered",
backends(&[
(GNOME_SHELL_EXTENSION_BACKEND, false),
(GNOME_SHELL_INTROSPECT_BACKEND, false),
(KWIN_BACKEND, true),
]),
resolved_after_failures(&[
GNOME_SHELL_EXTENSION_BACKEND,
GNOME_SHELL_INTROSPECT_BACKEND,
]),
None,
),
];
for (case, cached, observed, expected) in cases {
assert_eq!(
readiness_contradiction(&cached.readiness, &cached.windowing, observed),
expected,
"{case}"
);
}
}
#[tokio::test]
async fn app_state_calls_that_find_what_the_cache_predicts_reuse_one_report() {
use std::sync::atomic::Ordering;
// Accessibility is off in the fixture and in the live check, so every
// call skips the tree, as the cached report predicts, without touching
// the desktop.
let (server, runs) = server_with_labelled_diagnostics(|| false, || true);
for call in 1..=3 {
let state =
app_state_value(&server, serde_json::json!({"include_screenshot": false})).await;
assert!(
state["accessibility_error"]
.as_str()
.unwrap()
.contains("accessibility is disabled"),
"{state}"
);
assert_eq!(
state["readiness"]["recommended_next_step"], "probe run 1",
"call {call} reports the first call's report"
);
}
assert_eq!(runs.load(Ordering::SeqCst), 1, "only the first call probed");
}
#[tokio::test]
async fn a_failed_action_makes_the_next_app_state_probe_again() {
use std::sync::atomic::Ordering;
// Accessibility is off as the cached report says, so the observation
// itself never asks for a probe.
let (server, runs) = server_with_labelled_diagnostics(|| false, || true);
server.diagnostics.report(false).await;
// No observation has cached a tree, so the index is stale and the
// action fails before it reaches the desktop.
let params = serde_json::from_value(serde_json::json!({"element_index": 3})).unwrap();
let result = server.perform_action(Parameters(params)).await.unwrap();
assert_eq!(result.is_error, Some(true));
let state =
app_state_value(&server, serde_json::json!({"include_screenshot": false})).await;
assert_eq!(
runs.load(Ordering::SeqCst),
2,
"the failed action forced a probe"
);
assert_eq!(state["readiness"]["recommended_next_step"], "probe run 2");
}
#[tokio::test]
async fn verbose_app_state_always_probes_and_refreshes_the_cache() {
use std::sync::atomic::Ordering;
let (server, runs) = server_with_labelled_diagnostics(|| false, || true);
server.diagnostics.report(false).await;
let verbose = app_state_value(
&server,
serde_json::json!({"include_screenshot": false, "verbose": true}),
)
.await;
assert_eq!(runs.load(Ordering::SeqCst), 2);
assert_eq!(verbose["readiness"]["recommended_next_step"], "probe run 2");
assert_eq!(
verbose["diagnostics"]["readiness"]["recommended_next_step"],
"probe run 2"
);
let next = app_state_value(&server, serde_json::json!({"include_screenshot": false})).await;
assert_eq!(runs.load(Ordering::SeqCst), 2);
assert_eq!(
next["readiness"]["recommended_next_step"], "probe run 2",
"the next call reuses the verbose call's report"
);
}
/// Runs `child` in a process whose busctl and gsettings answer the
/// accessibility probes from CUL_A11Y_ENABLED, which starts as true. The
/// AT-SPI bus they name has no socket, so every tree read fails.
fn run_accessibility_child(child: &str) {
let dir = std::env::temp_dir().join(format!(
"cul-a11y-{child}-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir(&dir).unwrap();
std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).unwrap();
struct Cleanup(std::path::PathBuf);
impl Drop for Cleanup {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.0);
}
}
let _cleanup = Cleanup(dir.clone());
// systemctl keeps session discovery off the desktop.
for program in ["busctl", "gsettings", "systemctl"] {
let path = dir.join(program);
std::fs::write(
&path,
r#"#!/bin/sh
case "${0##*/} $*" in
systemctl*) exit 1 ;;
*GetAddress) echo "s \"unix:path=$HOME/no-a11y-bus\"" ;;
*IsEnabled|*toolkit-accessibility) echo "$CUL_A11Y_ENABLED" ;;
*) echo false ;;
esac
"#,
)
.unwrap();
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o700)).unwrap();
}
let mut command = Command::new(std::env::current_exe().unwrap());
command
.args([
"--exact",
&format!("server::tests::{child}"),
"--ignored",
"--nocapture",
])
.env_clear()
.env("PATH", &dir)
.env("HOME", &dir)
.env("XDG_RUNTIME_DIR", &dir)
.env("DISPLAY", ":9876")
.env("WAYLAND_DISPLAY", "")
.env("XDG_SESSION_TYPE", "x11")
.env("CUL_A11Y_CHILD", "1")
.env("CUL_A11Y_ENABLED", "true")
.env(
"DBUS_SESSION_BUS_ADDRESS",
format!("unix:path={}/no-bus", dir.display()),
);
for key in [
"DESKTOP_SESSION",
"HYPRLAND_INSTANCE_SIGNATURE",
"XAUTHORITY",
"XDG_SESSION_DESKTOP",
"XDG_CURRENT_DESKTOP",
] {
command.env(key, "a11y-child-test");
}
let output = crate::command_runner::output_blocking_with_timeout(
&mut command,
child,
Duration::from_secs(15),
)
.unwrap();
assert!(
String::from_utf8_lossy(&output.stdout).contains("running 1 test"),
"{child} must execute its assertions"
);
assert!(
output.status.success(),
"{}{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
}
#[test]
fn a_failed_app_state_reports_fresh_readiness_and_the_next_call_probes_again() {
run_accessibility_child("failed_app_state_child");
}
#[tokio::test]
#[ignore = "isolated environment subprocess; invoked by a_failed_app_state_reports_fresh_readiness_and_the_next_call_probes_again"]
async fn failed_app_state_child() {
use std::sync::atomic::Ordering;
assert_eq!(env::var("CUL_A11Y_CHILD").as_deref(), Ok("1"));
let (server, runs) = server_with_labelled_diagnostics(
crate::diagnostics_impl::can_build_accessibility_tree_now,
|| true,
);
server.diagnostics.report(false).await;
let state =
app_state_value(&server, serde_json::json!({"include_screenshot": false})).await;
let error = state["accessibility_error"].as_str().unwrap();
assert!(error.contains("failed to connect to AT-SPI bus"), "{error}");
assert_eq!(
runs.load(Ordering::SeqCst),
2,
"the failed call probed once"
);
assert_eq!(state["readiness"]["recommended_next_step"], "probe run 2");
assert!(state["diagnostics"].is_null());
// Every tree read fails here, so a second call would probe on its
// failure either way. Read what its start would read instead.
let (_, probed) = server.diagnostics.report_and_probed(false).await;
assert!(probed, "the next call probes again to see a recovery");
}
#[test]
fn app_state_sees_accessibility_switched_off_and_on_within_the_ttl() {
run_accessibility_child("accessibility_switch_child");
}
#[tokio::test]
#[ignore = "isolated environment subprocess; invoked by app_state_sees_accessibility_switched_off_and_on_within_the_ttl"]
async fn accessibility_switch_child() {
use std::sync::atomic::Ordering;
assert_eq!(env::var("CUL_A11Y_CHILD").as_deref(), Ok("1"));
let (server, runs) = server_with_labelled_diagnostics(
crate::diagnostics_impl::can_build_accessibility_tree_now,
|| true,
);
let params = serde_json::json!({"include_screenshot": false});
let cached = server.diagnostics.report(false).await;
assert!(cached.readiness.can_build_accessibility_tree);
env::set_var("CUL_A11Y_ENABLED", "false");
let off = app_state_value(&server, params.clone()).await;
assert_eq!(
runs.load(Ordering::SeqCst),
2,
"switching off forced a probe"
);
assert_eq!(off["readiness"]["recommended_next_step"], "probe run 2");
assert_eq!(off["readiness"]["can_build_accessibility_tree"], false);
assert!(
off["accessibility_error"]
.as_str()
.unwrap()
.contains("accessibility is disabled"),
"{}",
off["accessibility_error"]
);
let still_off = app_state_value(&server, params.clone()).await;
assert_eq!(
runs.load(Ordering::SeqCst),
2,
"a call that finds accessibility off, as cached, keeps the cache"
);
assert_eq!(
still_off["readiness"]["recommended_next_step"],
"probe run 2"
);
env::set_var("CUL_A11Y_ENABLED", "true");
let on = app_state_value(&server, params).await;
assert_eq!(
runs.load(Ordering::SeqCst),
3,
"switching on forced a probe"
);
assert_eq!(on["readiness"]["recommended_next_step"], "probe run 3");
assert_eq!(on["readiness"]["can_build_accessibility_tree"], true);
assert!(
!on["accessibility_error"]
.as_str()
.unwrap_or_default()
.contains("accessibility is disabled"),
"the tree read must run: {}",
on["accessibility_error"]
);
}
#[test]
fn gnome_window_crop_scales_logical_bounds_to_capture_pixels() {
let bounds = WindowBounds {
x: Some(6),
y: Some(36),
width: 1357,
height: 1144,
};
let monitors = [(0, 0, 1920, 1200)];
assert_eq!(
logical_window_crop_rect(&bounds, &monitors, 2560, 1600).unwrap(),
(8, 48, 1810, 1526)
);
}
#[test]
fn portal_points_map_capture_pixels_back_to_logical_window_space() {
let scaled = WindowCoordinateMap {
capture_rect: (8, 48, 1810, 1526),
full_capture_rect: (8, 48, 1810, 1526),
portal_rect: Some((6, 36, 1357, 1144)),
};
assert_eq!(scaled.portal_point(913, 811), Some((684, 608)));
let clipped = WindowCoordinateMap {
capture_rect: (0, 0, 50, 60),
full_capture_rect: (-50, -40, 100, 100),
portal_rect: Some((-50, -40, 100, 100)),
};
assert_eq!(clipped.portal_point(0, 0), Some((0, 0)));
}
#[test]
fn scaled_kwin_bounds_keep_capture_and_portal_spaces_distinct() {
let bounds = WindowBounds {
x: Some(1000),
y: Some(100),
width: 800,
height: 600,
};
let logical_rect = window_crop_rect(&bounds).unwrap();
let full_capture_rect =
logical_window_crop_rect(&bounds, &[(0, 0, 1920, 1080)], 3840, 2160).unwrap();
let mapping = WindowCoordinateMap {
capture_rect: full_capture_rect,
full_capture_rect,
portal_rect: Some(logical_rect),
};
assert_eq!(mapping.capture_rect, (2000, 200, 1600, 1200));
assert_eq!(mapping.portal_point(2400, 500), Some((1200, 250)));
}
#[test]
fn kwin_mapping_uses_the_workspace_geometry_origin() {
let bounds = WindowBounds {
x: Some(1100),
y: Some(50),
width: 800,
height: 600,
};
let logical_rect = window_crop_rect(&bounds).unwrap();
let full_capture_rect =
logical_window_crop_rect(&bounds, &[(100, -50, 1920, 1080)], 3840, 2160).unwrap();
let mapping = WindowCoordinateMap {
capture_rect: full_capture_rect,
full_capture_rect,
portal_rect: Some(logical_rect),
};
assert_eq!(mapping.capture_rect, (2000, 200, 1600, 1200));
assert_eq!(mapping.portal_point(2400, 500), Some((1300, 200)));
}
#[test]
fn gnome_window_crop_accounts_for_negative_monitor_origins() {
let bounds = WindowBounds {
x: Some(-900),
y: Some(100),
width: 400,
height: 300,
};
let monitors = [(-1000, 0, 1000, 800), (0, 0, 1200, 800)];
assert_eq!(
logical_window_crop_rect(&bounds, &monitors, 2200, 800).unwrap(),
(100, 100, 400, 300)
);
}
#[test]
fn readonly_targeted_screenshot_requires_focused_visible_window() {
let mut window = window_info(1, Some("Target"), None, None, None);
assert!(ensure_readonly_screenshot_target_is_visible(&window).is_err());
window.focused = true;
assert!(ensure_readonly_screenshot_target_is_visible(&window).is_ok());
window.hidden = true;
assert!(ensure_readonly_screenshot_target_is_visible(&window).is_err());
}
#[test]
fn wayland_display_is_enough_to_select_portal_fallback() {
assert!(session_is_wayland(None, Some("wayland-1")));
assert!(session_is_wayland(Some(" "), Some("wayland-1")));
assert!(session_is_wayland(Some("wayland"), None));
assert!(!session_is_wayland(Some("x11"), None));
assert!(!session_is_wayland(None, Some(" ")));
}
#[test]
fn incompatible_ydotool_socket_does_not_suppress_portal_fallback() {
let incompatible_ydotool = ydotool_backend_available_from(true, false);
let compatible_ydotool = ydotool_backend_available_from(true, true);
assert!(should_prefer_portal_backend_by_default(
true,
incompatible_ydotool
));
assert!(!should_prefer_portal_backend_by_default(
true,
compatible_ydotool
));
assert!(!should_prefer_portal_backend_by_default(
false,
incompatible_ydotool
));
}
#[test]
fn xdotool_keyboard_override_policy_matches_documented_precedence() {
assert!(prefer_xdotool_keyboard(false, true, true, true, true));
assert!(!prefer_xdotool_keyboard(true, true, true, true, true));
assert!(!prefer_xdotool_keyboard(false, true, true, false, true));
assert!(!prefer_xdotool_keyboard(false, false, true, true, true));
assert!(prefer_xdotool_keyboard(false, false, false, true, true));
}
#[test]
fn wayland_prefers_wtype_unless_ydotool_is_forced() {
assert!(prefer_wtype_keyboard(false, true, true, true));
assert!(!prefer_wtype_keyboard(true, true, true, true));
assert!(!prefer_wtype_keyboard(false, false, true, true));
assert!(!prefer_wtype_keyboard(false, true, false, true));
assert!(!prefer_wtype_keyboard(false, true, true, false));
}
#[test]
fn chords_use_wtype_only_where_no_remote_desktop_portal_exists() {
// (prefer_portal, portal_pinned, wtype, portal_missing)
assert_eq!(chord_route_from(true, false, true, true), ChordRoute::Wtype);
// The portal exists: it stays preferred even when wtype is present.
assert_eq!(
chord_route_from(true, false, true, false),
ChordRoute::Portal
);
// Forced portal or an existing consented session keeps the portal.
assert_eq!(chord_route_from(true, true, true, true), ChordRoute::Portal);
// Without wtype the portal reports its own error.
assert_eq!(
chord_route_from(true, false, false, true),
ChordRoute::Portal
);
// ydotool, xdotool and forced ydotool keep their command route.
assert_eq!(
chord_route_from(false, false, true, true),
ChordRoute::Command
);
}
#[test]
fn wtype_chords_hold_modifiers_around_one_keysym() {
let args = |key| wtype_key_args(key).map(|args| args.join(" "));
assert_eq!(args("Return").as_deref(), Some("-k Return"));
assert_eq!(
args("Ctrl+Shift+T").as_deref(),
Some("-M ctrl -M shift -k t -m shift -m ctrl")
);
assert_eq!(args("cmd+l").as_deref(), Some("-M logo -k l -m logo"));
assert_eq!(args("Alt+F4").as_deref(), Some("-M alt -k F4 -m alt"));
assert_eq!(args("Super").as_deref(), Some("-k Super_L"));
assert_eq!(args("Page Down").as_deref(), Some("-k Page_Down"));
assert!(wtype_key_args("Ctrl+Hyper+L").is_none());
assert!(wtype_key_args("Ctrl+F13").is_none());
assert!(wtype_key_args("").is_none());
}
#[test]
fn literal_text_portal_policy_preserves_layout_aware_routes_and_overrides() {
// GNOME has no wtype route. ydotool availability must not suppress
// literal portal text, unlike pointer and physical chord policies.
assert!(prefer_portal_text_backend(
false, false, true, false, false, false
));
// A compatible compositor with wtype needs no portal prompt.
assert!(!prefer_portal_text_backend(
false, false, true, false, false, true
));
assert!(prefer_portal_text_backend(
false, false, true, false, true, true
));
// Explicit raw/XTEST overrides, KDE clipboard, and X11 keep precedence.
assert!(!prefer_portal_text_backend(
true, false, true, false, true, false
));
assert!(!prefer_portal_text_backend(
false, true, true, false, true, false
));
assert!(!prefer_portal_text_backend(
false, false, true, true, true, false
));
assert!(!prefer_portal_text_backend(
false, false, false, false, true, false
));
}
#[tokio::test]
async fn raw_text_rejects_unsupported_characters_before_backend_access() {
assert!(validate_ydotool_text("probe-0710_a/b\t\nXYZ").is_ok());
for text in [
"prefixä",
"prefix中",
"prefix\r",
"prefix\0",
"prefix\u{7f}",
] {
// This exits before CLI probing, socket discovery or input, even
// though a supported prefix appears before the invalid character.
let error = run_ydotool_type_text(text).await.unwrap_err();
assert!(error.contains("no text was sent"), "{error}");
}
}
#[test]
fn raw_text_success_explains_layout_dependence() {
let result = action_result("type_text", Ok(Vec::new()), None);
assert!(result.ok);
assert!(result.message.contains("US physical key positions"));
assert!(result.message.contains("verify the field contents"));
}
#[test]
fn window_crop_happens_before_screenshot_payload_resize() {
let (cropped, width, height) = crop_png(&solid_png(400, 200), 50, 20, 200, 100).unwrap();
let capture = prepare_screenshot_payload(
RawScreenshotCapture {
mime_type: "image/png".to_string(),
bytes: cropped,
source: "test".to_string(),
width,
height,
},
ScreenshotPayloadOptions {
max_width: Some(100),
max_height: Some(100),
max_bytes: Some(1024 * 1024),
..Default::default()
},
)
.unwrap();
assert_eq!(
(capture.coordinate_width, capture.coordinate_height),
(200, 100)
);
assert_eq!((capture.width, capture.height), (100, 50));
assert!(capture.resized);
}
fn window_info(
window_id: u64,
title: Option<&str>,
app_id: Option<&str>,
wm_class: Option<&str>,
pid: Option<u32>,
) -> WindowInfo {
WindowInfo {
window_id,
title: title.map(str::to_string),
app_id: app_id.map(str::to_string),
wm_class: wm_class.map(str::to_string),
pid,
bounds: Some(WindowBounds {
x: Some(10),
y: Some(20),
width: 800,
height: 600,
}),
workspace: Some(0),
focused: false,
hidden: false,
client_type: Some("wayland".to_string()),
backend: GNOME_SHELL_EXTENSION_BACKEND.to_string(),
terminal: None,
}
}
#[test]
fn window_backend_names_the_backend_that_answered() {
let mut window = window_info(1, Some("Editor"), None, None, None);
window.backend = NIRI_BACKEND.to_string();
assert_eq!(window_backend(std::iter::once(&window)), NIRI_BACKEND);
assert_eq!(window_backend(std::iter::empty()), UNAVAILABLE_BACKEND);
}
#[test]
fn relative_click_coordinates_use_capture_space_rect() {
let mut params = ClickParams {
x: Some(7),
y: Some(9),
relative: Some(true),
..Default::default()
};
apply_window_relative_click_coordinates(&mut params, (133, 267, 1067, 800)).unwrap();
assert_eq!((params.x, params.y), (Some(140), Some(276)));
}
#[test]
fn relative_click_origin_is_screenshot_crop_not_raw_surface_buffer() {
// GNOME Wayland CSD probe: frame [760,406,400,300], buffer
// [746,394,428,329]. The screenshot's green target is at [850,543].
// Crop-local [90,137] and desktop [850,543] must denote the same point;
// substituting the buffer/shadow origin would miss by [14,12].
let mut params = ClickParams {
x: Some(90),
y: Some(137),
relative: Some(true),
..Default::default()
};
apply_window_relative_click_coordinates(&mut params, (760, 406, 400, 300)).unwrap();
assert_eq!((params.x, params.y), (Some(850), Some(543)));
}
#[test]
fn relative_click_coordinates_require_xy() {
let mut params = ClickParams {
x: Some(7),
relative: Some(true),
..Default::default()
};
let error =
apply_window_relative_click_coordinates(&mut params, (100, 200, 800, 600)).unwrap_err();
assert!(error.contains("both x and y"));
assert_eq!((params.x, params.y), (Some(7), None));
}
#[test]
fn relative_click_coordinates_must_stay_inside_bounds() {
for (x, y) in [(-1, 9), (7, -1), (800, 9), (7, 600)] {
let mut params = ClickParams {
x: Some(x),
y: Some(y),
relative: Some(true),
..Default::default()
};
let error = apply_window_relative_click_coordinates(&mut params, (100, 200, 800, 600))
.unwrap_err();
assert!(error.contains("inside target-window bounds"));
assert_eq!((params.x, params.y), (Some(x), Some(y)));
}
}
#[test]
fn absolute_pointer_note_reports_the_emitted_coordinate() {
assert_eq!(
abs_pointer_clamp_note(
"coordinate",
crate::abs_pointer::PointerLanding {
requested: (1920, 1080),
emitted: (1919, 1079),
}
),
Some(
"Requested coordinate 1920,1080 was clamped to 1919,1079 by the uinput absolute pointer."
.to_string()
)
);
assert_eq!(
abs_pointer_clamp_note(
"coordinate",
crate::abs_pointer::PointerLanding {
requested: (640, 480),
emitted: (640, 480),
}
),
None
);
}
#[tokio::test]
async fn failed_absolute_pointer_action_drops_the_device() {
use crate::abs_pointer::{tests::RecordedEvents, AbsPointer, PointerButton};
// Call 1 is the press. The device may now hold the button, so the
// next action must build a fresh one instead of reusing it.
let pointer = Arc::new(Mutex::new(Some(AbsPointer::with_events(
RecordedEvents::failing_on(&[1]),
100,
100,
))));
let result = run_abs_pointer(Arc::clone(&pointer), |pointer| {
pointer.click(10, 10, PointerButton::Left, 1)
})
.await;
assert!(result.is_err());
assert!(pointer.lock().unwrap().is_none());
let pointer = Arc::new(Mutex::new(Some(AbsPointer::with_events(
RecordedEvents::default(),
100,
100,
))));
let result = run_abs_pointer(Arc::clone(&pointer), |pointer| {
pointer.click(10, 10, PointerButton::Left, 1)
})
.await;
assert!(result.is_ok());
assert!(pointer.lock().unwrap().is_some());
}
#[test]
fn accessibility_filter_candidates_prefer_title_and_skip_synthetic_app_id() {
let window = window_info(
42,
Some("CU ATSPI GTK Test"),
Some("window:46"),
Some("cu_atspi_gtk_test.py"),
Some(2914326),
);
let candidates = accessibility_filter_candidates(Some(&window));
assert_eq!(
candidates,
vec![
"CU ATSPI GTK Test".to_string(),
"cu_atspi_gtk_test.py".to_string(),
]
);
}
#[test]
fn accessibility_filter_candidates_include_reverse_dns_leaf() {
let window = window_info(
42,
Some("Untitled — KWrite"),
Some("org.kde.kwrite"),
Some("kwrite"),
None,
);
let candidates = accessibility_filter_candidates(Some(&window));
assert!(candidates.contains(&"kwrite".to_string()));
assert!(!candidates.contains(&"desktop".to_string()));
}
fn accessible_app(object_ref: &str, name: &str) -> AccessibleAppSummary {
AccessibleAppSummary {
object_ref: object_ref.to_string(),
name: Some(name.to_string()),
pid: None,
role: "application".to_string(),
child_count: 1,
bounds: None,
}
}
#[test]
fn explicit_app_id_unique_root_uses_current_object_ref() {
let apps = vec![
accessible_app(":1.40/org/a11y/atspi/accessible/root", "KWrite"),
accessible_app(":1.41/org/a11y/atspi/accessible/root", "Kate"),
];
for app_id in ["kwrite", "org.kde.kwrite"] {
assert_eq!(
select_accessibility_app_id_filter(Some(&apps), app_id),
":1.40/org/a11y/atspi/accessible/root"
);
}
}
#[test]
fn explicit_app_id_missing_root_or_list_failure_is_unmatchable() {
let apps = vec![accessible_app(
":1.41/org/a11y/atspi/accessible/root",
"Kate",
)];
assert_eq!(
select_accessibility_app_id_filter(Some(&apps), "kwrite"),
UNMATCHABLE_ACCESSIBILITY_APP_FILTER
);
assert_eq!(
select_accessibility_app_id_filter(None, "kwrite"),
UNMATCHABLE_ACCESSIBILITY_APP_FILTER
);
for app_id in ["window:46", ".desktop"] {
assert_eq!(
select_accessibility_app_id_filter(Some(&apps), app_id),
UNMATCHABLE_ACCESSIBILITY_APP_FILTER
);
}
}
#[test]
fn explicit_app_id_ambiguous_leaf_is_unmatchable() {
let apps = vec![
accessible_app(":1.40/org/a11y/atspi/accessible/root", "KWrite"),
accessible_app(":1.42/org/a11y/atspi/accessible/root", "com.example.kwrite"),
];
assert_eq!(
select_accessibility_app_id_filter(Some(&apps), "kwrite"),
UNMATCHABLE_ACCESSIBILITY_APP_FILTER
);
}
#[test]
fn explicit_full_app_id_cannot_prove_ambiguous_root_namespaces() {
let apps = vec![
accessible_app(":1.40/org/a11y/atspi/accessible/root", "KWrite"),
accessible_app(":1.42/org/a11y/atspi/accessible/root", "kwrite"),
];
assert_eq!(
select_accessibility_app_id_filter(Some(&apps), "org.kde.kwrite"),
UNMATCHABLE_ACCESSIBILITY_APP_FILTER
);
}
#[tokio::test]
async fn app_id_authoritative_window_rejection_cannot_use_accessibility_fallback() {
let backend = ComputerUseLinux::default();
let windows = vec![
window_info(1, Some("KWrite"), Some("org.kde.kwrite"), None, None),
window_info(2, Some("Other"), Some("com.example.kwrite"), None, None),
];
for app_id in ["kwrite", "org.missing.kwrite"] {
let params: GetAppStateParams =
serde_json::from_value(serde_json::json!({ "app_id": app_id })).unwrap();
let target = params.window_target();
assert!(resolve_window_target(&windows, &target).is_err());
let filter = backend
.resolve_accessibility_app_filter_with(¶ms, None, true, || async {
Ok(vec![accessible_app(":1.40/root", "KWrite")])
})
.await;
assert_eq!(
filter.as_deref(),
Some(UNMATCHABLE_ACCESSIBILITY_APP_FILTER),
"{app_id}"
);
}
}
#[tokio::test]
async fn app_id_unavailable_window_backend_preserves_exact_accessibility_fallback() {
let backend = ComputerUseLinux::default();
for app_id in ["KWRITE", "org.kde.kwrite"] {
let params: GetAppStateParams =
serde_json::from_value(serde_json::json!({ "app_id": app_id })).unwrap();
let filter = backend
.resolve_accessibility_app_filter_with(¶ms, None, false, || async {
Ok(vec![accessible_app(":1.40/root", "KWrite")])
})
.await;
assert_eq!(filter.as_deref(), Some(":1.40/root"));
}
}
#[tokio::test]
async fn app_id_conflicting_resolved_window_cannot_use_pid_or_explicit_name() {
let backend = ComputerUseLinux::default();
let window = window_info(1, Some("Kate"), Some("org.kde.kate"), None, Some(123));
for explicit_name in [None, Some("Kate")] {
let params: GetAppStateParams = serde_json::from_value(serde_json::json!({
"window_id": 1,
"app_id": "org.kde.kwrite",
"app_name_or_bundle_identifier": explicit_name,
}))
.unwrap();
assert!(
resolve_window_target(std::slice::from_ref(&window), ¶ms.window_target())
.is_ok()
);
let filter = backend
.resolve_accessibility_app_filter_with(¶ms, Some(&window), true, || async {
let mut app = accessible_app(":1.41/root", "Kate");
app.pid = Some(123);
Ok(vec![app])
})
.await;
assert_eq!(
filter.as_deref(),
Some(UNMATCHABLE_ACCESSIBILITY_APP_FILTER)
);
}
}
#[test]
fn app_id_accessibility_fallback_rejects_arbitrary_name_substrings() {
let apps = vec![accessible_app(":1.40/root", "KWrite")];
assert_eq!(
select_accessibility_app_id_filter(Some(&apps), "write"),
UNMATCHABLE_ACCESSIBILITY_APP_FILTER
);
}
#[test]
fn app_id_rejection_cannot_regain_a_pid_only_snapshot() {
assert_eq!(
accessibility_snapshot_target_pid(
Some(UNMATCHABLE_ACCESSIBILITY_APP_FILTER),
Some(123)
),
None
);
assert_eq!(
accessibility_snapshot_target_pid(Some(":1.40/root"), Some(123)),
Some(123)
);
assert_eq!(
accessibility_snapshot_target_pid(None, Some(123)),
Some(123)
);
}
#[tokio::test]
async fn pid_targets_leave_root_selection_to_traversal_and_keep_explicit_filters() {
let backend = ComputerUseLinux::default();
let window = window_info(
42,
Some("Qt app"),
Some("org.example.qt-app"),
Some("qt-app"),
Some(123),
);
for fields in [
serde_json::json!({ "pid": 123 }),
serde_json::json!({ "window_id": 42 }),
serde_json::json!({ "app_id": "org.example.qt-app" }),
serde_json::json!({
"window_id": 42,
"app_name_or_bundle_identifier": ":1.41/root",
}),
] {
let params: GetAppStateParams = serde_json::from_value(fields).unwrap();
let expected = params.app_name_or_bundle_identifier.clone();
let filter = backend
.resolve_accessibility_app_filter_with(¶ms, Some(&window), true, || async {
panic!("PID-scoped traversal discovers every application root")
})
.await;
assert_eq!(filter, expected);
assert_eq!(
accessibility_snapshot_target_pid(filter.as_deref(), Some(123)),
Some(123)
);
}
let params =
serde_json::from_value(serde_json::json!({ "window_id": 42, "pid": 456 })).unwrap();
let mut pidless = window;
pidless.pid = None;
let filter = backend
.resolve_accessibility_app_filter_with(¶ms, Some(&pidless), true, || async {
panic!("a pidless window keeps its own name filter")
})
.await;
assert_eq!(filter.as_deref(), Some("Qt app"));
pidless.title = None;
pidless.app_id = None;
pidless.wm_class = None;
let filter = backend
.resolve_accessibility_app_filter_with(¶ms, Some(&pidless), true, || async {
panic!("an unidentified window cannot select another application")
})
.await;
assert_eq!(
filter.as_deref(),
Some(UNMATCHABLE_ACCESSIBILITY_APP_FILTER)
);
}
#[test]
fn compact_accessibility_tree_reparents_actionable_descendants() {
let nodes = vec![
AccessibilityNode {
index: 0,
parent_index: None,
depth: 0,
object_ref: ":1.0/root".to_string(),
role: "application".to_string(),
name: Some("demo-app".to_string()),
description: None,
child_count: 1,
bounds: None,
states: Vec::new(),
actions: Vec::new(),
value: None,
text: None,
supports_editable_text: false,
},
AccessibilityNode {
index: 1,
parent_index: Some(0),
depth: 1,
object_ref: ":1.1/frame".to_string(),
role: "frame".to_string(),
name: Some("Demo Frame".to_string()),
description: None,
child_count: 1,
bounds: None,
states: Vec::new(),
actions: Vec::new(),
value: None,
text: None,
supports_editable_text: false,
},
AccessibilityNode {
index: 2,
parent_index: Some(1),
depth: 2,
object_ref: ":1.2/filler".to_string(),
role: "filler".to_string(),
name: None,
description: None,
child_count: 1,
bounds: None,
states: Vec::new(),
actions: Vec::new(),
value: None,
text: None,
supports_editable_text: false,
},
AccessibilityNode {
index: 3,
parent_index: Some(2),
depth: 3,
object_ref: ":1.3/button".to_string(),
role: "button".to_string(),
name: Some("Run".to_string()),
description: None,
child_count: 0,
bounds: Some(Bounds {
x: 10,
y: 20,
width: 100,
height: 40,
}),
states: Vec::new(),
actions: vec![AccessibilityAction {
index: 0,
name: "Click".to_string(),
description: "Clicks the button".to_string(),
keybinding: String::new(),
}],
value: None,
text: None,
supports_editable_text: false,
},
];
let compacted = compact_accessibility_tree(nodes);
assert_eq!(compacted.len(), 3);
assert_eq!(compacted[0].role, "application");
assert_eq!(compacted[1].role, "frame");
assert_eq!(compacted[2].role, "button");
assert_eq!(compacted[2].parent_index, Some(1));
assert_eq!(compacted[1].child_count, 1);
}
#[test]
fn compact_accessibility_tree_drops_structural_noise() {
let nodes = vec![
AccessibilityNode {
index: 0,
parent_index: None,
depth: 0,
object_ref: ":1.0/root".to_string(),
role: "application".to_string(),
name: Some("demo-app".to_string()),
description: None,
child_count: 2,
bounds: None,
states: Vec::new(),
actions: Vec::new(),
value: None,
text: None,
supports_editable_text: false,
},
AccessibilityNode {
index: 1,
parent_index: Some(0),
depth: 1,
object_ref: ":1.1/frame".to_string(),
role: "frame".to_string(),
name: Some("Demo Frame".to_string()),
description: None,
child_count: 2,
bounds: None,
states: Vec::new(),
actions: Vec::new(),
value: None,
text: None,
supports_editable_text: false,
},
AccessibilityNode {
index: 2,
parent_index: Some(1),
depth: 2,
object_ref: ":1.2/tab".to_string(),
role: "page tab".to_string(),
name: Some("Hidden".to_string()),
description: None,
child_count: 0,
bounds: None,
states: Vec::new(),
actions: Vec::new(),
value: None,
text: None,
supports_editable_text: false,
},
AccessibilityNode {
index: 3,
parent_index: Some(1),
depth: 2,
object_ref: ":1.3/separator".to_string(),
role: "separator".to_string(),
name: None,
description: None,
child_count: 0,
bounds: None,
states: Vec::new(),
actions: Vec::new(),
value: None,
text: None,
supports_editable_text: false,
},
];
let compacted = compact_accessibility_tree(nodes);
assert_eq!(compacted.len(), 3);
assert_eq!(compacted[2].role, "page tab");
assert_eq!(compacted[2].name.as_deref(), Some("Hidden"));
}
fn tree_node(index: u32, parent: Option<u32>, depth: u32, role: &str) -> AccessibilityNode {
AccessibilityNode {
index,
parent_index: parent,
depth,
object_ref: format!(":1.{index}/node"),
role: role.to_string(),
name: None,
description: None,
child_count: 0,
bounds: None,
states: Vec::new(),
actions: Vec::new(),
value: None,
text: None,
supports_editable_text: false,
}
}
fn placeholder_text(children: usize) -> crate::atspi_tree::AccessibilityText {
crate::atspi_tree::AccessibilityText {
character_count: children as i32,
caret_offset: Some(0),
content: Some("\u{fffc}".repeat(children)),
truncated: false,
selections: Vec::new(),
}
}
fn named_action(index: i32, name: &str) -> AccessibilityAction {
AccessibilityAction {
index,
name: name.to_string(),
description: String::new(),
keybinding: String::new(),
}
}
#[test]
fn compaction_drops_containers_whose_text_is_only_child_placeholders() {
// Chromium sections report one U+FFFC per child as their text.
let mut section = tree_node(2, Some(1), 2, "section");
section.text = Some(placeholder_text(1));
let mut button = tree_node(3, Some(2), 3, "push button");
button.name = Some("Save".to_string());
button.actions = vec![named_action(0, "click")];
let nodes = vec![
tree_node(0, None, 0, "application"),
tree_node(1, Some(0), 1, "frame"),
section,
button,
];
let compacted = compact_accessibility_tree(nodes);
let roles: Vec<_> = compacted.iter().map(|node| node.role.as_str()).collect();
assert_eq!(roles, ["application", "frame", "push button"]);
assert_eq!(compacted[2].parent_index, Some(1));
}
fn chromium_generic_actions() -> Vec<AccessibilityAction> {
vec![
named_action(0, "doDefault"),
named_action(1, "showContextMenu"),
]
}
#[test]
fn compaction_drops_chromium_containers_with_only_generic_actions() {
// Chromium lists doDefault and showContextMenu on every element.
let mut panel = tree_node(2, Some(1), 2, "panel");
panel.actions = chromium_generic_actions();
panel.text = Some(placeholder_text(1));
let mut button = tree_node(3, Some(2), 3, "push button");
button.name = Some("Save".to_string());
button.actions = vec![named_action(0, "press"), named_action(1, "showContextMenu")];
let nodes = vec![
tree_node(0, None, 0, "application"),
tree_node(1, Some(0), 1, "frame"),
panel,
button,
];
let compacted = compact_accessibility_tree(nodes);
let roles: Vec<_> = compacted.iter().map(|node| node.role.as_str()).collect();
assert_eq!(roles, ["application", "frame", "push button"]);
}
#[test]
fn interactive_tree_keeps_controls_and_results_with_their_indices() {
let mut nodes = vec![
tree_node(0, None, 0, "application"),
tree_node(1, Some(0), 1, "frame"),
tree_node(2, Some(1), 2, "section"),
tree_node(3, Some(2), 3, "heading"),
tree_node(4, Some(2), 3, "section"),
tree_node(5, Some(4), 4, "link"),
tree_node(6, Some(5), 5, "static"),
tree_node(7, Some(4), 4, "entry"),
tree_node(8, Some(2), 3, "statusbar"),
tree_node(9, Some(2), 3, "progress bar"),
tree_node(10, Some(2), 3, "paragraph"),
tree_node(11, Some(8), 4, "static"),
tree_node(12, Some(2), 3, "notification"),
tree_node(13, Some(2), 3, "button"),
];
for node in &mut nodes {
node.actions = chromium_generic_actions();
}
nodes[3].name = Some("Results".to_string());
nodes[5].actions = vec![named_action(0, "jump"), named_action(1, "showContextMenu")];
nodes[5].name = Some("Docs".to_string());
// Text inside the link only forwards clicks to it.
nodes[6].actions = vec![named_action(0, "clickAncestor")];
nodes[6].name = Some("Docs".to_string());
nodes[7].supports_editable_text = true;
nodes[9].value = Some(crate::atspi_tree::AccessibilityValue {
current: 40.0,
minimum: 0.0,
maximum: 100.0,
minimum_increment: 1.0,
text: None,
});
nodes[10].name = Some("Body copy".to_string());
// The status region's text sits in a child, as in Chromium; an empty
// notification (node 12) announces nothing, and a button with no
// specific action (node 13) is still a control.
nodes[8].child_count = 1;
nodes[11].name = Some("Saved".to_string());
// get_app_state compacts first, then filters the compacted tree.
let kept = interactive_accessibility_tree(compact_accessibility_tree(nodes));
let summary: Vec<_> = kept
.iter()
.map(|node| (node.role.as_str(), node.parent_index, node.child_count))
.collect();
// Compaction renumbers: the frame is 1 and the status bar 6.
assert_eq!(
summary,
[
("application", None, 1),
("frame", Some(0), 6),
("heading", Some(1), 0),
("link", Some(1), 0),
("entry", Some(1), 0),
("statusbar", Some(1), 1),
("progress bar", Some(1), 0),
("static", Some(6), 0),
("button", Some(1), 0),
]
);
assert_eq!(kept[7].name.as_deref(), Some("Saved"));
}
#[test]
fn interactive_indices_resolve_to_the_same_cached_nodes() {
let backend = ComputerUseLinux::default();
let mut nodes: Vec<_> = (0..4)
.map(|index| tree_node(index, index.checked_sub(1), index, "section"))
.collect();
nodes[3].role = "push button".to_string();
nodes[3].actions = vec![named_action(0, "click")];
let kept = interactive_accessibility_tree(nodes);
backend.cache_snapshot(&kept, None);
let node = backend
.resolve_cached_node(
Some(3),
&ElementSelector::default(),
ElementResolvePurpose::Action,
)
.unwrap();
assert_eq!(node.object_ref, ":1.3/node");
assert!(backend
.resolve_cached_node(
Some(2),
&ElementSelector::default(),
ElementResolvePurpose::Action
)
.is_err());
}
#[test]
fn output_schema_lets_compact_nodes_leave_out_default_fields() {
let schema = rmcp::handler::server::tool::schema_for_output::<GetAppStateOutput>();
let required = |name: &str| -> Vec<String> {
schema["$defs"][name]["required"]
.as_array()
.unwrap()
.iter()
.map(|field| field.as_str().unwrap().to_string())
.collect()
};
assert_eq!(
required("CompactNode"),
["index", "depth", "object_ref", "role"]
);
assert_eq!(required("CompactText"), ["character_count"]);
}
#[test]
fn post_action_tree_keeps_the_snapshot_window_scope_of_the_same_app() {
let scope = Some(("PlanetScale - Chromium".to_string(), Some(7)));
// An untargeted perform_action or set_value on the same app keeps it.
assert_eq!(
post_action_window_title(None, None, Some(7), scope.clone()).as_deref(),
Some("PlanetScale - Chromium")
);
// A pid-only target of the same app keeps it too.
let pid_only = WindowTarget {
pid: Some(7),
..WindowTarget::default()
};
assert_eq!(
post_action_window_title(Some(&pid_only), None, Some(7), scope.clone()).as_deref(),
Some("PlanetScale - Chromium")
);
// Another app, or a snapshot with no pid, does not inherit it.
assert_eq!(
post_action_window_title(None, None, Some(8), scope.clone()),
None
);
assert_eq!(
post_action_window_title(None, None, None, Some(("x".to_string(), None))),
None
);
// A target that names its own window uses that window.
let titled = WindowTarget {
title: Some("Other".to_string()),
..WindowTarget::default()
};
assert_eq!(
post_action_window_title(Some(&titled), None, Some(7), scope).as_deref(),
Some("Other")
);
}
#[test]
fn compact_tree_pages_and_spills_in_the_snapshot_encoding() {
let mut node = tree_node(0, None, 0, "frame");
node.actions = vec![named_action(0, "click")];
let page = tree_page(&[node.clone()], NodeDetail::Compact, 0);
let json = serde_json::to_value(&page).unwrap();
assert_eq!(json["nodes"][0]["actions"], serde_json::json!(["click"]));
assert!(json["nodes"][0].get("value").is_none());
let page = tree_page(&[node], NodeDetail::Full, 0);
let json = serde_json::to_value(&page).unwrap();
assert_eq!(json["nodes"][0]["actions"][0]["name"], "click");
assert!(json["nodes"][0]["value"].is_null());
}
#[test]
fn window_title_narrows_only_title_or_window_id_targets() {
let params = |value: serde_json::Value| -> GetAppStateParams {
serde_json::from_value(value).unwrap()
};
let window = WindowInfo {
title: Some("PlanetScale - Chromium".to_string()),
..kwin_window(42)
};
assert_eq!(
accessibility_window_title(¶ms(serde_json::json!({"pid": 7})), Some(&window)),
None,
"a pid target keeps every window of the app"
);
assert_eq!(
accessibility_window_title(
¶ms(serde_json::json!({"title": "PlanetScale"})),
Some(&window)
)
.as_deref(),
Some("PlanetScale - Chromium")
);
assert_eq!(
accessibility_window_title(¶ms(serde_json::json!({"title": "PlanetScale"})), None)
.as_deref(),
Some("PlanetScale")
);
assert_eq!(
accessibility_window_title(
¶ms(serde_json::json!({"window_id": 42})),
Some(&window)
)
.as_deref(),
Some("PlanetScale - Chromium")
);
}
#[test]
fn kde_clipboard_restore_delay_uses_minimum_for_short_text() {
assert_eq!(
kde_clipboard_restore_delay("short"),
Duration::from_millis(KDE_CLIPBOARD_RESTORE_MIN_DELAY_MS)
);
}
#[test]
fn kde_clipboard_restore_delay_scales_and_caps_long_text() {
let scaled_text = "x".repeat(1_000);
assert_eq!(
kde_clipboard_restore_delay(&scaled_text),
Duration::from_millis(4_000)
);
let capped_text = "x".repeat(10_000);
assert_eq!(
kde_clipboard_restore_delay(&capped_text),
Duration::from_millis(KDE_CLIPBOARD_RESTORE_MAX_DELAY_MS)
);
}
#[tokio::test]
async fn failed_kde_clipboard_write_restores_the_old_contents_and_is_not_replayed() {
for restore_works in [true, false] {
let writes = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
let recorded = std::sync::Arc::clone(&writes);
let error = replace_kde_clipboard("secret text", "old contents", move |value| {
recorded.lock().unwrap().push(value.clone());
async move {
if value == "secret text" || !restore_works {
Err("klipper call timed out".to_string())
} else {
Ok(())
}
}
})
.await
.unwrap_err();
// The write may have landed although it failed, so the user's
// clipboard is put back, and the text is not typed another way.
assert_eq!(*writes.lock().unwrap(), ["secret text", "old contents"]);
assert!(!error.can_fallback_to_ydotool);
assert!(!error.clear_portal_keyboard_session);
assert!(error.message.contains("not replayed"), "{}", error.message);
assert_eq!(
error.message.contains("previous contents were restored"),
restore_works,
"{}",
error.message
);
}
}
#[test]
fn kde_clipboard_maps_each_paste_shortcut_to_its_chord() {
assert_eq!(
kde_clipboard_paste_chord(KdeClipboardPasteShortcut::Standard),
(&[EVDEV_KEY_LEFTCTRL][..], EVDEV_KEY_V)
);
assert_eq!(
kde_clipboard_paste_chord(KdeClipboardPasteShortcut::CtrlShiftV),
(&[EVDEV_KEY_LEFTCTRL, EVDEV_KEY_LEFTSHIFT][..], EVDEV_KEY_V)
);
assert_eq!(
kde_clipboard_paste_chord(KdeClipboardPasteShortcut::ShiftInsert),
(&[EVDEV_KEY_LEFTSHIFT][..], EVDEV_KEY_INSERT)
);
}
fn focused(role: &str, is_terminal: bool) -> FocusedElementSummary {
FocusedElementSummary {
role: role.to_string(),
name: None,
editable: false,
states: vec!["focused".to_string()],
is_terminal,
}
}
#[test]
fn kde_clipboard_focus_fallback_uses_the_role_enum_not_the_name() {
let ctrl_shift_v = Some(TerminalPasteShortcut::CtrlShiftV);
let shift_insert = Some(TerminalPasteShortcut::ShiftInsert);
// No AT-SPI answer (xterm exposes none) keeps the terminal chord.
assert_eq!(
kde_clipboard_shortcut_for_focus(shift_insert, None),
KdeClipboardPasteShortcut::ShiftInsert
);
// A terminal widget keeps it, whatever its localized role name.
assert_eq!(
kde_clipboard_shortcut_for_focus(ctrl_shift_v, Some(&focused("терминал", true))),
KdeClipboardPasteShortcut::CtrlShiftV
);
// A focused search field inside the terminal window pastes with Ctrl+V.
assert_eq!(
kde_clipboard_shortcut_for_focus(ctrl_shift_v, Some(&focused("text", false))),
KdeClipboardPasteShortcut::Standard
);
assert_eq!(
kde_clipboard_shortcut_for_focus(None, None),
KdeClipboardPasteShortcut::Standard
);
}
#[test]
fn kde_terminal_keeps_chord_for_non_text_focus() {
let ctrl_shift_v = Some(TerminalPasteShortcut::CtrlShiftV);
let shift_insert = Some(TerminalPasteShortcut::ShiftInsert);
assert_eq!(
kde_clipboard_shortcut_for_focus(ctrl_shift_v, Some(&focused("filler", false))),
KdeClipboardPasteShortcut::CtrlShiftV
);
assert_eq!(
kde_clipboard_shortcut_for_focus(ctrl_shift_v, Some(&focused("panel", false))),
KdeClipboardPasteShortcut::CtrlShiftV
);
assert_eq!(
kde_clipboard_shortcut_for_focus(shift_insert, Some(&focused("filler", false))),
KdeClipboardPasteShortcut::ShiftInsert
);
assert_eq!(
kde_clipboard_shortcut_for_focus(ctrl_shift_v, Some(&focused("text", false))),
KdeClipboardPasteShortcut::Standard
);
let editable_filler = FocusedElementSummary {
editable: true,
..focused("filler", false)
};
assert_eq!(
kde_clipboard_shortcut_for_focus(ctrl_shift_v, Some(&editable_filler)),
KdeClipboardPasteShortcut::Standard
);
}
#[test]
fn kde_clipboard_routes_xterm_family_to_shift_insert() {
for wm_class in ["xterm", "UXTerm", "urxvt", "rxvt-unicode", "koi8rxterm"] {
let window = window_info(1, Some("user@host"), None, Some(wm_class), Some(100));
assert_eq!(
kde_clipboard_terminal_shortcut(&WindowTarget::default(), Some(&window)),
Some(TerminalPasteShortcut::ShiftInsert),
"{wm_class} should paste with Shift+Insert"
);
}
}
#[test]
fn kde_clipboard_routes_explicit_tty_targets_to_terminal_paste() {
let target = WindowTarget {
tty: Some("/dev/pts/11".to_string()),
..Default::default()
};
assert_eq!(
kde_clipboard_terminal_shortcut(&target, None),
Some(TerminalPasteShortcut::CtrlShiftV)
);
}
#[test]
fn kde_clipboard_prefers_resolved_window_over_stale_terminal_selector() {
let target = WindowTarget {
tty: Some("/dev/pts/11".to_string()),
..Default::default()
};
let window = window_info(
1,
Some("Browser"),
Some("firefox"),
Some("firefox"),
Some(100),
);
assert_eq!(
kde_clipboard_terminal_shortcut(&target, Some(&window)),
None
);
}
#[tokio::test]
async fn kde_clipboard_async_routing_prefers_focus_over_stale_terminal_selector() {
let target = WindowTarget {
tty: Some("/dev/pts/11".to_string()),
..Default::default()
};
let window = window_info(
1,
Some("Browser"),
Some("firefox"),
Some("firefox"),
Some(100),
);
let focus = WindowFocusResult {
requested_window: window.clone(),
focused_window: Some(window),
exact_window_focused: true,
app_focused: true,
backend: KWIN_BACKEND.to_string(),
note: "test".to_string(),
};
assert_eq!(
kde_clipboard_paste_shortcut(&target, Some(&focus)).await,
KdeClipboardPasteShortcut::Standard
);
}
#[test]
fn kde_clipboard_routes_konsole_identity_to_terminal_paste() {
let window = window_info(
1,
Some("unflappable-donkey"),
Some("org.kde.konsole"),
Some("konsole"),
Some(100),
);
assert_eq!(
kde_clipboard_terminal_shortcut(&WindowTarget::default(), Some(&window)),
Some(TerminalPasteShortcut::CtrlShiftV)
);
}
#[test]
fn kde_clipboard_does_not_classify_terminal_words_in_browser_titles() {
let window = window_info(
1,
Some("xterm documentation"),
Some("firefox"),
Some("firefox"),
Some(100),
);
assert_eq!(
kde_clipboard_terminal_shortcut(&WindowTarget::default(), Some(&window)),
None
);
}
#[test]
fn kde_clipboard_pty_metadata_does_not_change_editor_paste() {
let mut window = window_info(
1,
Some("xterm integration test"),
Some("example-ide"),
Some("ExampleIde"),
Some(100),
);
window.terminal = Some(crate::terminal::TerminalWindowContext {
tty: "/dev/pts/11".to_string(),
root_process: crate::terminal::TerminalProcess {
pid: 200,
command_name: "bash".to_string(),
command_line: "bash".to_string(),
cwd: None,
},
active_process: None,
process_count: 1,
confidence: "high".to_string(),
match_reason: "one child PTY".to_string(),
});
let terminal = kde_clipboard_terminal_shortcut(&WindowTarget::default(), Some(&window));
assert_eq!(terminal, None);
assert_eq!(
kde_clipboard_shortcut_for_focus(terminal, None),
KdeClipboardPasteShortcut::Standard
);
}
#[tokio::test]
async fn kde_clipboard_dbus_operation_times_out_when_pending() {
let error = kde_clipboard_dbus_operation_with_timeout(
"proxy creation",
std::future::pending::<zbus::Result<()>>(),
Duration::from_millis(1),
)
.await
.unwrap_err();
assert_eq!(error, "KDE clipboard proxy creation timed out");
}
#[test]
fn cached_element_index_resolves_to_bounds_center() {
let backend = ComputerUseLinux::default();
backend.cache_nodes(&[node(
7,
Some(Bounds {
x: 10,
y: 20,
width: 100,
height: 40,
}),
)]);
let point = backend
.resolve_optional_target_point(None, None, Some(7))
.unwrap()
.unwrap();
assert_eq!(point, (60, 40));
}
#[test]
fn coordinate_target_overrides_cached_element_index() {
let backend = ComputerUseLinux::default();
backend.cache_nodes(&[node(
7,
Some(Bounds {
x: 10,
y: 20,
width: 100,
height: 40,
}),
)]);
let point = backend
.resolve_optional_target_point(Some(200), Some(300), Some(7))
.unwrap()
.unwrap();
assert_eq!(point, (200, 300));
}
#[test]
fn cached_element_index_requires_positive_bounds() {
let backend = ComputerUseLinux::default();
backend.cache_nodes(&[node(
7,
Some(Bounds {
x: 10,
y: 20,
width: 0,
height: 40,
}),
)]);
let error = backend
.resolve_optional_target_point(None, None, Some(7))
.unwrap_err();
assert!(error.contains("No clickable bounds cached for element_index 7"));
}
#[test]
fn cached_element_index_ignores_sentinel_bounds() {
let backend = ComputerUseLinux::default();
backend.cache_nodes(&[node(
7,
Some(Bounds {
x: i32::MIN,
y: i32::MIN,
width: 1,
height: 1,
}),
)]);
let error = backend
.resolve_optional_target_point(None, None, Some(7))
.unwrap_err();
assert!(error.contains("No clickable bounds cached for element_index 7"));
}
#[test]
fn empty_node_cache_clears_stale_element_index() {
let backend = ComputerUseLinux::default();
backend.cache_nodes(&[node(
7,
Some(Bounds {
x: 10,
y: 20,
width: 100,
height: 40,
}),
)]);
backend.cache_nodes(&[]);
let error = backend
.resolve_optional_target_point(None, None, Some(7))
.unwrap_err();
assert!(error.contains("No clickable bounds cached for element_index 7"));
}
#[test]
fn click_target_falls_back_to_primary_action_without_bounds() {
let backend = ComputerUseLinux::default();
backend.cache_nodes(&[node_with_actions(
7,
None,
vec![AccessibilityAction {
index: 0,
name: "Click".to_string(),
description: "Clicks the button".to_string(),
keybinding: String::new(),
}],
)]);
let target = backend
.resolve_click_target(&ClickParams {
element_index: Some(7),
..Default::default()
})
.unwrap();
match target {
ClickTarget::PrimaryAction {
object_ref,
action_name,
action_index,
} => {
assert_eq!(object_ref, ":1.7/org/a11y/atspi/accessible/7");
assert_eq!(action_name.as_deref(), Some("Click"));
assert_eq!(action_index, 0);
}
ClickTarget::Coordinates(_, _)
| ClickTarget::ElementBounds { .. }
| ClickTarget::ElementPoint { .. } => {
panic!("expected AT-SPI primary-action fallback")
}
}
}
#[test]
fn element_activation_ignores_hidpi_and_zero_origin_bounds() {
for (x, y) in [(75, 144), (0, 0)] {
let backend = ComputerUseLinux::default();
backend.cache_nodes(&[node_with_actions(
7,
Some(Bounds {
x,
y,
width: 320,
height: 120,
}),
vec![AccessibilityAction {
index: 0,
name: "click".into(),
description: String::new(),
keybinding: String::new(),
}],
)]);
assert!(matches!(
backend
.resolve_click_target(&ClickParams {
element_index: Some(7),
..Default::default()
})
.unwrap(),
ClickTarget::PrimaryAction {
action_index: 0,
..
}
));
// Explicit coordinates and gestures retain pointer semantics.
for params in [
ClickParams {
x: Some(470),
y: Some(408),
element_index: Some(7),
..Default::default()
},
ClickParams {
button: Some("right".into()),
element_index: Some(7),
..Default::default()
},
ClickParams {
click_count: Some(2),
element_index: Some(7),
..Default::default()
},
] {
let explicit = params.x.is_some();
match backend.resolve_click_target(¶ms).unwrap() {
ClickTarget::Coordinates(470, 408) if explicit => {}
ClickTarget::ElementBounds { center, .. } if !explicit => {
assert_eq!(center, (x + 160, y + 60));
}
_ => panic!("unexpected click target"),
}
}
}
}
#[test]
fn element_click_does_not_replace_pointer_with_non_activation_action() {
let backend = ComputerUseLinux::default();
backend.cache_nodes(&[node_with_actions(
7,
Some(Bounds {
x: 10,
y: 20,
width: 20,
height: 20,
}),
vec![AccessibilityAction {
index: 0,
name: "show-menu".into(),
description: String::new(),
keybinding: String::new(),
}],
)]);
assert!(matches!(
backend
.resolve_click_target(&ClickParams {
element_index: Some(7),
..Default::default()
})
.unwrap(),
ClickTarget::ElementBounds {
center: (20, 30),
..
}
));
}
fn kwin_cached_element_nodes() -> Vec<AccessibilityNode> {
let mut app = node(0, None);
app.role = "application".to_string();
let mut frame = node(
1,
Some(Bounds {
x: 0,
y: 0,
width: 800,
height: 600,
}),
);
frame.parent_index = Some(0);
frame.depth = 1;
frame.role = "frame".to_string();
let mut menu = node(
2,
Some(Bounds {
x: 0,
y: 0,
width: 64,
height: 33,
}),
);
menu.parent_index = Some(1);
menu.depth = 2;
menu.role = "menu".to_string();
vec![app, frame, menu]
}
fn kwin_window(window_id: u64) -> WindowInfo {
let mut window = window_info(
window_id,
Some("Editor"),
Some("editor"),
Some("editor"),
Some(4242),
);
window.backend = KWIN_BACKEND.to_string();
window
}
#[test]
fn kwin_cached_dialog_and_alert_ancestors_keep_window_identity() {
for role in ["frame", "window", "dialog", "alert"] {
let backend = ComputerUseLinux::default();
let mut nodes = kwin_cached_element_nodes();
nodes[1].role = role.to_string();
backend.cache_snapshot_with_window(&nodes, Some(4242), None, Some(&kwin_window(42)));
let target = backend
.cached_kwin_element_target(&nodes[2], (32, 16), false, None)
.unwrap_or_else(|error| panic!("{role}: {error}"));
assert_eq!(target.window_id, Some(42));
assert_eq!(target.pid, Some(4242));
assert_eq!((target.frame.width, target.frame.height), (800, 600));
}
let mut nodes = kwin_cached_element_nodes();
nodes[1].role = "panel".to_string();
assert!(depth_one_kwin_element_frame(&nodes, &nodes[2], (32, 16), false).is_err());
}
#[test]
fn kwin_cached_element_requires_depth_one_frame_and_exact_window_identity() {
let backend = ComputerUseLinux::default();
let nodes = kwin_cached_element_nodes();
backend.cache_snapshot_with_window(&nodes, Some(4242), None, Some(&kwin_window(42)));
let target = backend
.cached_kwin_element_target(&nodes[2], (32, 16), false, None)
.unwrap();
assert_eq!(target.window_id, Some(42));
assert_eq!(target.pid, Some(4242));
assert_eq!((target.frame.x, target.frame.y), (0, 0));
assert_eq!((target.frame.width, target.frame.height), (800, 600));
let global = backend
.cached_kwin_element_target(&nodes[2], (1148, 487), true, None)
.unwrap();
assert!(global.global_point);
assert_eq!(global.center, (1148, 487));
let missing = node(7, nodes[2].bounds.clone());
let error = depth_one_kwin_element_frame(&nodes, &missing, (32, 16), false).unwrap_err();
assert!(error.contains("depth-1"), "{error}");
let error = depth_one_kwin_element_frame(&nodes, &nodes[2], (900, 16), false).unwrap_err();
assert!(error.contains("outside"), "{error}");
let requested = kwin_window(42);
let focused = kwin_window(43);
let app_only_focus = WindowFocusResult {
requested_window: requested,
focused_window: Some(focused),
exact_window_focused: false,
app_focused: true,
backend: KWIN_BACKEND.to_string(),
note: String::new(),
};
let error = backend
.cached_kwin_element_target(&nodes[2], (32, 16), false, Some(&app_only_focus))
.unwrap_err();
assert!(error.contains("not app-only focus"), "{error}");
}
#[test]
fn kwin_adapter_bypasses_explicit_relative_native_and_non_kwin_paths() {
assert!(!kwin_element_adapter_required(
false,
true,
Some(KWIN_BACKEND),
None
));
assert!(!kwin_element_adapter_required(
true,
false,
Some(GNOME_SHELL_EXTENSION_BACKEND),
None
));
assert!(!kwin_element_adapter_required(
true,
true,
Some(KWIN_BACKEND),
Some(GNOME_SHELL_EXTENSION_BACKEND)
));
assert!(kwin_element_adapter_required(
true,
false,
Some(KWIN_BACKEND),
None
));
assert!(kwin_element_adapter_required(true, true, None, None));
let backend = ComputerUseLinux::default();
let mut nodes = kwin_cached_element_nodes();
nodes[2].actions = vec![click_action()];
backend.cache_snapshot_with_window(&nodes, Some(4242), None, Some(&kwin_window(42)));
assert!(matches!(
backend.resolve_click_target(&ClickParams {
element_index: Some(2),
..Default::default()
}),
Ok(ClickTarget::PrimaryAction { .. })
));
assert!(matches!(
backend.resolve_click_target(&ClickParams {
element_index: Some(2),
x: Some(470),
y: Some(408),
..Default::default()
}),
Ok(ClickTarget::Coordinates(470, 408))
));
let mut relative = ClickParams {
element_index: Some(2),
x: Some(7),
y: Some(9),
relative: Some(true),
window_id: Some(42),
..Default::default()
};
apply_window_relative_click_coordinates(&mut relative, (133, 267, 1067, 800)).unwrap();
assert!(matches!(
backend.resolve_click_target(&relative),
Ok(ClickTarget::Coordinates(140, 276))
));
}
#[test]
fn atspi_coordinate_target_uses_cached_element_and_supplied_point() {
let backend = ComputerUseLinux::default();
let mut target = node_with_actions(
7,
Some(Bounds {
x: 0,
y: 0,
width: 64,
height: 33,
}),
vec![click_action()],
);
target.role = "menu item".to_string();
target.name = Some("Görünüm".to_string());
backend.cache_nodes(&[target]);
for params in [
ClickParams {
element_index: Some(7),
x: Some(32),
y: Some(16),
coordinate_space: ClickCoordinateSpace::Atspi,
..Default::default()
},
ClickParams {
role: Some("menu item".to_string()),
name: Some("Görünüm".to_string()),
x: Some(32),
y: Some(16),
coordinate_space: ClickCoordinateSpace::Atspi,
..Default::default()
},
] {
match backend.resolve_click_target(¶ms).unwrap() {
ClickTarget::ElementPoint { node, point } => {
assert_eq!(node.index, 7);
assert_eq!(point, (32, 16));
}
_ => panic!("explicit AT-SPI coordinates must retain pointer semantics"),
}
}
}
#[tokio::test]
async fn explicit_atspi_coordinates_reject_non_kwin_backend_before_target_resolution() {
for name in ["gnome", "niri", "x11"] {
let backend = ComputerUseLinux::default();
backend.last_snapshot.lock().unwrap().window_backend = Some(name.to_string());
let output = backend
.dispatch_click(ClickParams {
element_index: Some(7),
x: Some(32),
y: Some(16),
coordinate_space: ClickCoordinateSpace::Atspi,
..Default::default()
})
.await
.0
.0;
assert!(!output.ok);
assert!(
output.message.contains("require a KWin Wayland"),
"{name}: {}",
output.message
);
}
}
#[test]
fn atspi_coordinate_params_require_unambiguous_geometry() {
let valid = ClickParams {
element_index: Some(7),
x: Some(32),
y: Some(16),
coordinate_space: ClickCoordinateSpace::Atspi,
..Default::default()
};
assert_eq!(validate_click_coordinate_space(&valid), Ok(()));
let relative = ClickParams {
relative: Some(true),
..valid.clone()
};
assert!(validate_click_coordinate_space(&relative)
.unwrap_err()
.contains("cannot use relative"));
let missing_y = ClickParams {
y: None,
..valid.clone()
};
assert!(validate_click_coordinate_space(&missing_y)
.unwrap_err()
.contains("both x and y"));
let missing_element = ClickParams {
element_index: None,
..valid
};
assert!(validate_click_coordinate_space(&missing_element)
.unwrap_err()
.contains("require element_index or a semantic selector"));
let raw_capture = ClickParams {
x: Some(1148),
y: Some(487),
..Default::default()
};
assert_eq!(validate_click_coordinate_space(&raw_capture), Ok(()));
}
#[test]
fn kwin_prepared_click_routes_capture_to_uinput_and_logical_to_portal() {
let prepared = PreparedKwinElementClick {
element_index: 2,
point: KwinElementClickPoint {
logical: (278, 352),
capture: (417, 528),
},
capture_size: (3840, 2160),
};
assert_eq!(prepared.uinput_point(), (417, 528));
assert_eq!(prepared.portal_point(), (278, 352));
let output = kwin_element_click_dispatch_output(
ActionOutput {
ok: true,
implemented: true,
action: "click".to_string(),
message: "Action sent through the test pointer.".to_string(),
received: None,
},
Some(prepared),
);
assert!(output.ok);
assert!(output.message.contains("capture 417,528"));
assert!(output.message.contains("portal logical 278,352"));
assert!(output.message.contains("Dispatch only"));
assert!(output.message.contains("UI activation was not verified"));
}
/// The KWin mapping text must not replace the notes each backend route
/// already appended (#389), and names no backend of its own (#396).
#[test]
fn kwin_element_click_mapping_keeps_backend_notes() {
let prepared = PreparedKwinElementClick {
element_index: 2,
point: KwinElementClickPoint {
logical: (2560, 1440),
capture: (3840, 2160),
},
capture_size: (3840, 2160),
};
// uinput route: the edge-clamp note.
let uinput = kwin_element_click_dispatch_output(
abs_pointer_result(
"click",
Ok(crate::abs_pointer::PointerLanding {
requested: (3840, 2160),
emitted: (3839, 2159),
}),
None,
),
Some(prepared),
);
assert!(
uinput
.message
.contains("Requested coordinate 3840,2160 was clamped to 3839,2159"),
"{}",
uinput.message
);
assert!(
uinput.message.contains("capture 3840,2160"),
"{}",
uinput.message
);
// The backend message already names the backend; the mapping does
// not name it again (#396).
assert!(
uinput.message.starts_with(
"Action sent through the uinput absolute pointer. Requested coordinate 3840,2160 was clamped to 3839,2159 by the uinput absolute pointer. KWin element coordinate mapping: capture 3840,2160;"
),
"{}",
uinput.message
);
// Portal route: the mapping follows the backend message and does
// not name the backend again.
let portal = kwin_element_click_dispatch_output(
ActionOutput {
ok: true,
implemented: true,
action: "click".to_string(),
message: "Action sent through the remote desktop portal.".to_string(),
received: None,
},
Some(prepared),
);
assert_eq!(
portal.message,
"Action sent through the remote desktop portal. KWin element coordinate mapping: capture 3840,2160; portal logical 2560,1440. Dispatch only: UI activation was not verified."
);
}
#[test]
fn kwin_adapter_failures_are_not_sent_and_include_recovery_guidance() {
let backend = ComputerUseLinux::default();
backend.cache_snapshot_with_window(
&kwin_cached_element_nodes(),
Some(4242),
None,
Some(&kwin_window(42)),
);
let output = backend
.kwin_element_click_failure(
2,
"mapped element point is outside every active KWin output",
None,
)
.0;
assert!(!output.ok);
assert!(output.message.contains("click was not sent"));
assert!(output.message.contains("No pointer event was emitted"));
assert!(output.message.contains("window_id 42"));
assert!(output.message.contains("screenshot x/y"));
}
/// A KWin element click whose mapping worked but whose uinput device
/// could not be replaced names the device and the retry route, not a
/// mapping failure (#396).
#[test]
fn kwin_pointer_replacement_failure_names_the_device() {
let failure = AbsPointerFailure {
detail: "cached absolute-pointer geometry was stale for the 3840x2160 raw capture and the device could not be recreated: uinput unavailable".to_string(),
device_dropped: true,
};
let output = kwin_abs_pointer_not_sent(
2,
&failure,
"it goes through the remote desktop portal",
None,
);
assert!(!output.ok);
assert_eq!(
output.message,
"click was not sent: KWin mapped element_index 2, but the uinput absolute pointer could not be used: cached absolute-pointer geometry was stale for the 3840x2160 raw capture and the device could not be recreated: uinput unavailable. No pointer event was emitted. The stale device was removed. A retry of element_index 2 sizes a new device to a fresh screenshot; if that fails, it goes through the remote desktop portal."
);
// A poisoned lock dropped no device, so there is no retry to name.
let output = kwin_abs_pointer_not_sent(
2,
&AbsPointerFailure::lock_poisoned(),
"it goes through the remote desktop portal",
None,
);
assert!(
output
.message
.ends_with("absolute pointer lock was poisoned. No pointer event was emitted."),
"{}",
output.message
);
}
/// The note checks capture pixels against the monitors scaled onto the
/// capture, and words what each route did (#391).
#[tokio::test]
async fn monitor_gap_note_follows_the_route() {
let backend = ComputerUseLinux::default();
let known = |backend: &ComputerUseLinux, monitors: Vec<(i32, i32, i32, i32)>| {
backend.capture_layout.lock().unwrap().monitors = CaptureMonitors::Logical(monitors);
};
// A 2x capture of a logical 2240x720 layout with a gap under the
// right monitor.
backend.cache_desktop_size(4480, 1440);
known(&backend, vec![(0, 0, 1280, 720), (1280, 0, 960, 540)]);
for route in [
PointerRoute::X11,
PointerRoute::Uinput,
PointerRoute::Portal,
] {
for (x, y) in [(100, 1300), (3000, 500), (4479, 1079)] {
assert_eq!(backend.off_screen_note_for_point(x, y, route).await, None);
}
}
assert_eq!(
backend
.off_screen_note_for_point(3000, 1200, PointerRoute::Portal)
.await
.as_deref(),
Some("WARNING: coordinate 3000,1200 is on no monitor of the captured desktop (4480x1440).")
);
assert_eq!(
backend
.off_screen_note_for_point(2560, 1080, PointerRoute::X11)
.await
.as_deref(),
Some("WARNING: coordinate 2560,1080 is on no monitor of the captured desktop (4480x1440); the input landed at a monitor edge or on no visible pixel.")
);
// Off the capture, X11 clamps to the screen first.
assert_eq!(
backend
.off_screen_note_for_point(5000, 500, PointerRoute::X11)
.await
.as_deref(),
Some("WARNING: coordinate 5000,500 is outside the captured desktop (4480x1440); the X server keeps the pointer on the screen, so the input landed at an edge instead.")
);
assert_eq!(
backend
.off_screen_note_for_point(5000, 1300, PointerRoute::X11)
.await
.as_deref(),
Some("WARNING: coordinate 5000,1300 is outside the captured desktop (4480x1440); the X server keeps the pointer on the screen, but the nearest point there, 4479,1300, is on no monitor, so the input landed at a monitor edge or on no visible pixel.")
);
// Off the capture, the uinput clamp note speaks for itself.
assert_eq!(
backend
.off_screen_note_for_point(-1, 10, PointerRoute::Uinput)
.await,
None
);
assert_eq!(
backend
.off_screen_note_for_point(-1, 10, PointerRoute::Portal)
.await
.as_deref(),
Some("WARNING: coordinate -1,10 is outside the captured desktop (4480x1440).")
);
// A layout that does not scale evenly onto the capture is ignored.
known(&backend, vec![(0, 0, 1280, 720), (1280, 0, 1280, 540)]);
assert_eq!(
backend
.off_screen_note_for_point(3000, 1200, PointerRoute::X11)
.await,
None
);
// So is an unknown one, and a new capture drops the old layout.
backend.capture_layout.lock().unwrap().monitors = CaptureMonitors::Unknown;
assert_eq!(
backend
.off_screen_note_for_point(3000, 1200, PointerRoute::X11)
.await,
None
);
known(&backend, vec![(0, 0, 1280, 720), (1280, 0, 960, 540)]);
backend.cache_desktop_size(4480, 1440);
assert_eq!(
backend.capture_layout.lock().unwrap().monitors,
CaptureMonitors::Unread
);
}
/// The portal sends no drag with an endpoint off its monitors, so its
/// notes name each endpoint that stopped it and claim no landing (#398).
#[tokio::test]
async fn portal_drag_notes_name_each_endpoint() {
let backend = ComputerUseLinux::default();
backend.cache_desktop_size(4480, 1440);
backend.capture_layout.lock().unwrap().monitors =
CaptureMonitors::Logical(vec![(0, 0, 1280, 720), (1280, 0, 960, 540)]);
assert_eq!(
backend
.drag_off_screen_notes((3000, 1200), (5000, 500), PointerRoute::Portal)
.await,
[
"WARNING: start coordinate 3000,1200 is on no monitor of the captured desktop (4480x1440).",
"WARNING: end coordinate 5000,500 is outside the captured desktop (4480x1440).",
]
);
assert_eq!(
backend
.drag_off_screen_notes((100, 1300), (-1, 10), PointerRoute::Portal)
.await,
["WARNING: end coordinate -1,10 is outside the captured desktop (4480x1440)."]
);
// The path from 2000,1400 to 4400,1000 crosses the gap; the portal
// moves that motion onto a monitor, so only endpoints count.
assert!(backend
.drag_off_screen_notes((2000, 1400), (4400, 1000), PointerRoute::Portal)
.await
.is_empty());
}
/// An X11 capture is the whole root window, which can hold framebuffer
/// area no monitor shows, so RandR rectangles are used as they are.
#[tokio::test]
async fn x11_monitor_gap_note_uses_root_pixels() {
let backend = ComputerUseLinux::default();
backend.cache_desktop_size(4480, 3600);
backend.capture_layout.lock().unwrap().monitors =
CaptureMonitors::Root(vec![(0, 0, 1280, 720), (1280, 720, 960, 1080)]);
for (x, y) in [(100, 100), (1500, 1600), (2239, 1799)] {
assert_eq!(
backend
.off_screen_note_for_point(x, y, PointerRoute::X11)
.await,
None,
"{x},{y}"
);
}
for (x, y) in [(100, 1000), (2240, 1000), (4000, 3000)] {
assert!(
backend
.off_screen_note_for_point(x, y, PointerRoute::X11)
.await
.is_some_and(|note| note.contains("is on no monitor")),
"{x},{y}"
);
}
assert_eq!(root_monitor_rects(&[]), None);
assert_eq!(root_monitor_rects(&[(0, 0, 0, 720)]), None);
}
#[test]
fn monitor_rects_scale_onto_the_capture() {
// Negative origins and a 1.25 scale; edges round outward.
assert_eq!(
monitor_rects_in_capture(&[(-1536, 0, 1536, 864), (0, 0, 1024, 575)], (3200, 1080)),
Some(vec![(0, 0, 1920, 1080), (1920, 0, 3200, 719)])
);
assert_eq!(monitor_rects_in_capture(&[], (1920, 1080)), None);
assert_eq!(
monitor_rects_in_capture(&[(0, 0, 0, 1080)], (1920, 1080)),
None
);
assert_eq!(
monitor_rects_in_capture(&[(0, 0, 1920, 1080)], (1920, 1200)),
None
);
let rects = monitor_rects_in_capture(&[(0, 0, 1920, 1080)], (1920, 1080)).unwrap();
assert!(point_on_capture_monitor(&rects, 1919, 1079));
assert!(!point_on_capture_monitor(&rects, 1920, 1079));
}
#[tokio::test]
async fn stale_pointer_recreation_failure_is_fail_closed() {
let backend = ComputerUseLinux::default();
let error = backend
.create_abs_pointer_with(2560, 1440, true, |_, _| anyhow::bail!("uinput unavailable"))
.await
.unwrap_err();
assert!(
error
.detail
.contains("geometry was stale for the 2560x1440"),
"{error:?}"
);
assert!(error.detail.contains("could not be recreated"), "{error:?}");
assert!(error.device_dropped);
assert!(backend.abs_pointer.lock().unwrap().is_none());
// With no stale device, a failed creation leaves the other routes.
let ready = backend
.create_abs_pointer_with(2560, 1440, false, |_, _| {
anyhow::bail!("uinput unavailable")
})
.await;
assert_eq!(ready, Ok(false));
assert!(backend.abs_pointer.lock().unwrap().is_none());
}
#[test]
fn entry_submit_and_slider_jump_are_not_pointer_clicks() {
for (role, action_name) in [
("entry", "activate"),
("spin button", "activate"),
("slider", "jump"),
] {
let backend = ComputerUseLinux::default();
let mut target = node_with_actions(
7,
Some(Bounds {
x: 10,
y: 20,
width: 100,
height: 40,
}),
vec![AccessibilityAction {
index: 0,
name: action_name.into(),
description: String::new(),
keybinding: String::new(),
}],
);
target.role = role.into();
backend.cache_nodes(&[target]);
assert!(matches!(
backend
.resolve_click_target(&ClickParams {
element_index: Some(7),
..Default::default()
})
.unwrap(),
ClickTarget::ElementBounds {
center: (60, 40),
..
}
));
let mut node = backend.last_snapshot.lock().unwrap().nodes[0].clone();
for bounds in [
None,
Some(Bounds {
x: i32::MIN,
y: i32::MIN,
width: 1,
height: 1,
}),
Some(Bounds {
x: 0,
y: 0,
width: 0,
height: 0,
}),
] {
node.bounds = bounds;
backend.cache_nodes(&[node.clone()]);
assert!(backend
.resolve_click_target(&ClickParams {
element_index: Some(7),
..Default::default()
})
.is_err());
}
}
}
#[test]
fn click_target_falls_back_to_primary_action_with_sentinel_bounds() {
let backend = ComputerUseLinux::default();
backend.cache_nodes(&[node_with_actions(
7,
Some(Bounds {
x: i32::MIN,
y: i32::MIN,
width: 1,
height: 1,
}),
vec![AccessibilityAction {
index: 0,
name: "Click".to_string(),
description: "Clicks the button".to_string(),
keybinding: String::new(),
}],
)]);
let target = backend
.resolve_click_target(&ClickParams {
element_index: Some(7),
..Default::default()
})
.unwrap();
match target {
ClickTarget::PrimaryAction {
object_ref,
action_name,
action_index,
} => {
assert_eq!(object_ref, ":1.7/org/a11y/atspi/accessible/7");
assert_eq!(action_name.as_deref(), Some("Click"));
assert_eq!(action_index, 0);
}
ClickTarget::Coordinates(_, _)
| ClickTarget::ElementBounds { .. }
| ClickTarget::ElementPoint { .. } => {
panic!("expected AT-SPI primary-action fallback")
}
}
}
#[test]
fn click_target_requires_bounds_for_non_plain_clicks() {
let backend = ComputerUseLinux::default();
backend.cache_nodes(&[node_with_actions(
7,
None,
vec![AccessibilityAction {
index: 0,
name: "Click".to_string(),
description: "Clicks the button".to_string(),
keybinding: String::new(),
}],
)]);
let error = backend
.resolve_click_target(&ClickParams {
element_index: Some(7),
button: Some("right".to_string()),
..Default::default()
})
.unwrap_err();
assert!(error.contains("No clickable bounds cached for element_index 7"));
}
#[test]
fn absolute_mousemove_uses_coordinate_separator() {
assert_eq!(
absolute_mousemove_args(200, 300),
vec![
"mousemove".to_string(),
"--absolute".to_string(),
"--".to_string(),
"200".to_string(),
"300".to_string(),
]
);
}
#[test]
fn native_x11_pointer_policy_requires_explicit_x11_without_wayland_display() {
assert!(native_x11_xdotool_pointer_session(Some("x11"), None));
assert!(!native_x11_xdotool_pointer_session(
Some("wayland"),
Some("wayland-0")
));
assert!(!native_x11_xdotool_pointer_session(
Some("x11"),
Some("wayland-0")
));
}
#[test]
fn xdotool_pointer_command_is_single_no_sync_move_and_click() {
assert_eq!(
xdotool_pointer_click_args(1550, 930, 3, Some("right")),
Some(vec![
"mousemove".to_string(),
"--".to_string(),
"1550".to_string(),
"930".to_string(),
"click".to_string(),
"--repeat".to_string(),
"3".to_string(),
"3".to_string(),
])
);
}
#[test]
fn xdotool_pointer_policy_requires_a_native_display_without_global_override() {
assert!(prefer_xdotool_pointer(false, Some("x11"), true, None));
assert!(!prefer_xdotool_pointer(true, Some("x11"), true, None));
assert!(!prefer_xdotool_pointer(false, Some("wayland"), true, None));
assert!(!prefer_xdotool_pointer(false, Some("x11"), false, None));
assert!(!prefer_xdotool_pointer(
false,
Some("x11"),
true,
Some("wayland-0")
));
}
#[test]
fn xdotool_pointer_supports_only_standard_buttons() {
assert!(xdotool_pointer_click_args(10, 20, 1, None).is_some());
assert!(xdotool_pointer_click_args(10, 20, 1, Some("middle")).is_some());
assert!(xdotool_pointer_click_args(10, 20, 1, Some("right")).is_some());
}
#[test]
fn extended_pointer_buttons_do_not_construct_xdotool_commands() {
for button in ["side", "extra", "forward", "back"] {
assert_eq!(xdotool_pointer_click_args(10, 20, 1, Some(button)), None);
}
}
#[tokio::test]
async fn pointer_xdotool_spawn_failure_uses_ydotool_fallback() {
let result = run_xdotool_pointer_or_fallback(
Path::new("/definitely/missing/xdotool"),
&[],
|| async { Ok::<_, String>(Vec::new()) },
)
.await
.expect("spawn failure should use fallback");
assert_eq!(result.backend, KeyboardCommandBackend::Ydotool);
}
#[tokio::test]
async fn pointer_xdotool_nonzero_exit_does_not_use_ydotool_fallback() {
let result = run_xdotool_pointer_or_fallback(
Path::new("/bin/sh"),
&["-c".to_string(), "exit 9".to_string()],
|| async { Err::<Vec<Output>, _>("fallback called".to_string()) },
)
.await;
let error = result.expect_err("launched nonzero xdotool must be terminal");
assert!(!error.contains("fallback called"));
}
#[test]
fn xdotool_scroll_uses_wheel_buttons_instead_of_a_valuator_event() {
assert_eq!(
xdotool_pointer_scroll_args(Some((1550, 930)), 5, ScrollDirection::Up),
vec![
"mousemove".to_string(),
"--".to_string(),
"1550".to_string(),
"930".to_string(),
"click".to_string(),
"--delay".to_string(),
"12".to_string(),
"--repeat".to_string(),
"5".to_string(),
"4".to_string(),
]
);
assert_eq!(
xdotool_pointer_scroll_args(Some((-20, 15)), 1, ScrollDirection::Down),
vec![
"mousemove".to_string(),
"--".to_string(),
"-20".to_string(),
"15".to_string(),
"click".to_string(),
"--delay".to_string(),
"12".to_string(),
"--repeat".to_string(),
"1".to_string(),
"5".to_string(),
]
);
assert_eq!(
xdotool_pointer_scroll_args(None, 3, ScrollDirection::Left),
vec![
"click".to_string(),
"--delay".to_string(),
"12".to_string(),
"--repeat".to_string(),
"3".to_string(),
"6".to_string(),
]
);
assert_eq!(
xdotool_pointer_scroll_args(None, 0, ScrollDirection::Right),
vec![
"click".to_string(),
"--delay".to_string(),
"12".to_string(),
"--repeat".to_string(),
"1".to_string(),
"7".to_string(),
]
);
}
#[test]
fn scroll_pages_are_bounded_before_unit_conversion() {
assert_eq!(scroll_units(None).unwrap(), 5);
assert_eq!(scroll_units(Some(1.0)).unwrap(), 5);
assert_eq!(scroll_units(Some(-1.0)).unwrap(), 5);
assert_eq!(scroll_units(Some(0.0)).unwrap(), 1);
assert_eq!(scroll_units(Some(0.1)).unwrap(), 1);
assert_eq!(scroll_units(Some(100.0)).unwrap(), 500);
assert_eq!(scroll_units(Some(-100.0)).unwrap(), 500);
for pages in [
100.000_001,
-100.000_001,
f64::NAN,
f64::INFINITY,
f64::NEG_INFINITY,
] {
assert!(scroll_units(Some(pages)).is_err(), "accepted {pages}");
}
}
#[test]
fn maximum_scroll_timeout_is_bounded() {
let repeat = u32::try_from(scroll_units(Some(MAX_SCROLL_PAGES)).unwrap()).unwrap();
assert_eq!(repeat, 500);
assert_eq!(
xdotool_scroll_timeout(repeat, XDOTOOL_SCROLL_CLICK_DELAY_MS),
Duration::from_secs(16)
);
}
#[test]
fn xdotool_scroll_timeout_covers_the_gap_after_every_click() {
assert_eq!(
xdotool_scroll_timeout(1, 12),
INPUT_COMMAND_TIMEOUT + Duration::from_millis(12)
);
assert_eq!(
xdotool_scroll_timeout(5, 100),
INPUT_COMMAND_TIMEOUT + Duration::from_millis(500)
);
assert_eq!(
xdotool_scroll_timeout(0, 12),
INPUT_COMMAND_TIMEOUT + Duration::from_millis(12)
);
}
/// Left is a negative `REL_HWHEEL`, as on the portal route and X11
/// button 6. Left used to be positive, a scroll right.
#[test]
fn wheel_deltas_use_kernel_signs() {
assert_eq!(wheel_deltas(ScrollDirection::Up, 5), (0, 5));
assert_eq!(wheel_deltas(ScrollDirection::Down, 5), (0, -5));
assert_eq!(wheel_deltas(ScrollDirection::Left, 5), (-5, 0));
assert_eq!(wheel_deltas(ScrollDirection::Right, 5), (5, 0));
}
#[test]
fn wheel_mousemove_uses_coordinate_separator_for_negative_values() {
assert_eq!(
wheel_mousemove_args(0, -3),
vec![
"mousemove".to_string(),
"--wheel".to_string(),
"--".to_string(),
"0".to_string(),
"-3".to_string(),
]
);
}
#[test]
fn pointer_actions_keep_pixel_coordinates_for_ydotool_absolute_moves() {
assert_eq!(
absolute_mousemove_args(1550, 930),
vec![
"mousemove".to_string(),
"--absolute".to_string(),
"--".to_string(),
"1550".to_string(),
"930".to_string(),
]
);
}
#[test]
fn key_chord_splits_modifiers_and_key() {
assert_eq!(key_chord("Ctrl+Shift+P"), Some((vec![29, 42], 25)));
assert_eq!(key_chord("Ctrl+S"), Some((vec![29], 31)));
assert_eq!(key_chord("Enter"), Some((vec![], 28)));
// A bare modifier is a chord with no held modifiers.
assert_eq!(key_chord("Super"), Some((vec![], 125)));
assert_eq!(key_chord("NotAKey"), None);
}
#[test]
fn portal_key_chord_sends_modifiers_and_named_keys_as_keysyms() {
use PortalKey::{Keycode, Keysym};
// Modifiers resolve through the live keymap (#191); letters stay
// physical so non-Latin layout groups still see Ctrl+A.
assert_eq!(
portal_key_chord("Ctrl+A"),
Some((vec![Keysym(xkey::Control_L as i32)], Keycode(30)))
);
assert_eq!(
portal_key_chord("ctrl+shift+t"),
Some((
vec![Keysym(xkey::Control_L as i32), Keysym(xkey::Shift_L as i32)],
Keycode(20)
))
);
assert_eq!(
portal_key_chord("Alt+F4"),
Some((vec![Keysym(xkey::Alt_L as i32)], Keysym(xkey::F4 as i32)))
);
assert_eq!(
portal_key_chord("Meta+Return"),
Some((
vec![Keysym(xkey::Super_L as i32)],
Keysym(xkey::Return as i32)
))
);
assert_eq!(
portal_key_chord("Super"),
Some((vec![], Keysym(xkey::Super_L as i32)))
);
assert_eq!(
portal_key_chord("page-down"),
Some((vec![], Keysym(xkey::Page_Down as i32)))
);
assert_eq!(
portal_key_chord("Ctrl+1"),
Some((vec![Keysym(xkey::Control_L as i32)], Keycode(2)))
);
assert_eq!(portal_key_chord("Ctrl+NotAKey"), None);
assert_eq!(portal_key_chord("Hyper+A"), None);
}
#[test]
fn portal_key_chord_accepts_exactly_the_evdev_grammar() {
for key in [
"enter",
"Escape",
"tab",
"backspace",
"del",
"space",
"home",
"end",
"pageup",
"left",
"right",
"up",
"down",
"f1",
"f12",
"a",
"z",
"0",
"9",
"ctrl",
"alt",
"shift",
"cmd",
"Ctrl+Alt+Delete",
"Shift+Tab",
"ctrl+shift",
"?",
"F13",
"Ctrl+",
"",
] {
assert_eq!(
portal_key_chord(key).is_some(),
key_chord(key).is_some(),
"{key}"
);
}
}
#[test]
fn key_sequence_presses_modifiers_around_key() {
assert_eq!(
key_sequence("Ctrl+Shift+P"),
Some(vec![
"29:1".to_string(),
"42:1".to_string(),
"25:1".to_string(),
"25:0".to_string(),
"42:0".to_string(),
"29:0".to_string(),
])
);
}
#[test]
fn ydotool_modifier_chords_include_an_inter_event_delay() {
let args = ydotool_key_args(key_sequence("Ctrl+T").unwrap(), true);
assert_eq!(args, ["key", "-d", "100", "29:1", "20:1", "20:0", "29:0"]);
}
#[test]
fn key_sequence_presses_bare_modifier() {
assert_eq!(
key_sequence("Super"),
Some(vec!["125:1".to_string(), "125:0".to_string()])
);
}
#[test]
fn xdotool_key_spec_maps_named_keys_to_x11_keysyms() {
assert_eq!(xdotool_key_spec("Return"), Some("Return".to_string()));
assert_eq!(xdotool_key_spec("enter"), Some("Return".to_string()));
assert_eq!(xdotool_key_spec("Escape"), Some("Escape".to_string()));
assert_eq!(xdotool_key_spec("backspace"), Some("BackSpace".to_string()));
assert_eq!(xdotool_key_spec("PageUp"), Some("Page_Up".to_string()));
assert_eq!(xdotool_key_spec("ArrowLeft"), Some("Left".to_string()));
assert_eq!(xdotool_key_spec("f5"), Some("F5".to_string()));
assert_eq!(xdotool_key_spec("space"), Some("space".to_string()));
}
#[test]
fn xdotool_key_spec_maps_chords_with_modifier_prefixes() {
assert_eq!(xdotool_key_spec("ctrl+a"), Some("ctrl+a".to_string()));
assert_eq!(xdotool_key_spec("Ctrl+S"), Some("ctrl+s".to_string()));
assert_eq!(
xdotool_key_spec("Ctrl+Shift+P"),
Some("ctrl+shift+p".to_string())
);
assert_eq!(
xdotool_key_spec("Meta+Return"),
Some("super+Return".to_string())
);
assert_eq!(xdotool_key_spec("Alt+F4"), Some("alt+F4".to_string()));
}
#[test]
fn xdotool_key_spec_maps_bare_modifier_to_single_keysym() {
assert_eq!(xdotool_key_spec("Super"), Some("super".to_string()));
assert_eq!(xdotool_key_spec("ctrl"), Some("ctrl".to_string()));
}
#[test]
fn xdotool_type_keeps_a_per_character_delay_so_xtest_events_stay_ordered() {
// Issue #147: `--delay 0` delivers characters out of order on some X
// servers. The default must stay non-zero.
const { assert!(XDOTOOL_TYPE_DELAY_MS > 0) };
let text = "x".repeat(10_000);
let args = xdotool_type_args_with_delay(&text, XDOTOOL_TYPE_DELAY_MS);
assert_eq!(
&args[..5],
["type", "--clearmodifiers", "--delay", "12", "--"]
);
assert_eq!(args[3], XDOTOOL_TYPE_DELAY_MS.to_string());
assert_eq!(args[5], text);
}
#[test]
fn xdotool_type_uses_return_for_lf_and_crlf() {
let args = xdotool_type_args_with_delay("L1\nL2\r\nL3\r", 12);
assert_eq!(args[5], "L1\rL2\rL3\r");
assert_eq!(
xdotool_type_args_with_delay("quotes '\" and literal \\n: שלום", 12)[5],
"quotes '\" and literal \\n: שלום"
);
}
#[test]
fn xdotool_type_timeout_grows_with_text_length() {
assert_eq!(xdotool_type_timeout("", 12), INPUT_COMMAND_TIMEOUT);
assert_eq!(
xdotool_type_timeout(&"x".repeat(1_000), 12),
INPUT_COMMAND_TIMEOUT + Duration::from_secs(12)
);
assert_eq!(
xdotool_type_timeout(&"x".repeat(1_000), 0),
INPUT_COMMAND_TIMEOUT
);
}
#[tokio::test]
async fn launched_xdotool_failure_does_not_replay_through_ydotool() {
let dir = std::env::temp_dir().join(format!(
"computer-use-linux-xdotool-fallback-{}-{:?}",
std::process::id(),
std::time::SystemTime::now()
));
std::fs::create_dir_all(&dir).expect("create command test directory");
let ydotool = dir.join("ydotool");
let xdotool_marker = dir.join("xdotool-ran");
let ydotool_marker = dir.join("ydotool-ran");
std::fs::write(
&ydotool,
format!("#!/bin/sh\ntouch '{}'\n", ydotool_marker.display()),
)
.expect("write fake ydotool");
std::fs::set_permissions(&ydotool, std::fs::Permissions::from_mode(0o700))
.expect("make fake ydotool executable");
let xdotool_args = vec![
"-c".to_string(),
format!("touch '{}'; exit 9", xdotool_marker.display()),
];
let result = run_xdotool_or_fallback(Path::new("/bin/sh"), &xdotool_args, || async {
TokioCommand::new(&ydotool)
.output()
.await
.map_err(|error| error.to_string())
})
.await;
assert!(result.is_err());
assert!(xdotool_marker.exists(), "fake xdotool did not execute");
assert!(
!ydotool_marker.exists(),
"ydotool replayed input after xdotool started"
);
let _ = std::fs::remove_dir_all(dir);
}
#[tokio::test]
async fn unavailable_xdotool_uses_ydotool_fallback() {
let result = run_xdotool_or_fallback(
Path::new("/definitely/missing/xdotool"),
&xdotool_type_args_with_delay("text", XDOTOOL_TYPE_DELAY_MS),
|| async {
TokioCommand::new("sh")
.args(["-c", "exit 0"])
.output()
.await
.map_err(|error| error.to_string())
},
)
.await
.expect("spawn failure should use fallback");
assert_eq!(result.backend, KeyboardCommandBackend::Ydotool);
assert!(result.output.status.success());
}
#[tokio::test]
async fn wtype_passes_long_unicode_as_one_literal_argument() {
let dir = std::env::temp_dir().join(format!(
"computer-use-linux-wtype-unicode-{}-{:?}",
std::process::id(),
std::time::SystemTime::now()
));
std::fs::create_dir_all(&dir).expect("create command test directory");
let wtype = dir.join("wtype");
let captured = dir.join("captured");
publish_fake_executable(
&wtype,
&format!(
"#!/bin/sh\n[ \"$#\" -eq 2 ] && [ \"$1\" = -- ] || exit 42\nprintf '%s' \"$2\" > '{}'\n",
captured.display()
),
);
let text =
"--Türkçe PostgreSQL (W, g, m, v, f, ş, ü) / 0123456789 - ; 'İstanbul'\n".repeat(50);
let result = run_wtype_type_text_or_fallback(
&wtype,
&text,
WtypeTextMode::Argument,
false,
|| async { panic!("available wtype must not fall back") },
)
.await
.expect("wtype should succeed");
assert_eq!(result.backend, KeyboardCommandBackend::Wtype);
assert_eq!(std::fs::read_to_string(&captured).unwrap(), text);
for text in ["--", "-", ""] {
run_wtype_type_text_or_fallback(
&wtype,
text,
WtypeTextMode::Argument,
false,
|| async { panic!("available wtype must not fall back") },
)
.await
.unwrap();
assert_eq!(std::fs::read_to_string(&captured).unwrap(), text);
}
let _ = std::fs::remove_dir_all(dir);
}
#[tokio::test]
async fn review_argument_mode_never_starts_wtype_for_protected_text() {
// Even /bin/false and the fallback must never run. An exit-status
// error would prove that validation happened too late.
let error = run_wtype_type_text_or_fallback(
Path::new("/bin/false"),
"synthetic-password",
WtypeTextMode::Argument,
true,
|| async { panic!("secret input must not fall back") },
)
.await
.err()
.expect("secret text must be rejected");
assert!(error.contains("password fields"), "{error}");
let error = run_wtype_type_text_or_fallback(
Path::new("/definitely/missing/wtype"),
"synthetic-password",
WtypeTextMode::Argument,
true,
|| async { panic!("secret input must not fall back even if wtype disappears") },
)
.await
.err()
.expect("secret text must be rejected");
assert!(error.contains("password fields"), "{error}");
}
#[tokio::test]
async fn review_slow_focus_probe_has_its_own_input_budget() {
let mode = hyprland_wtype_delivery("long text".repeat(100).as_str(), async {
tokio::time::sleep(Duration::from_millis(300)).await;
Some(FocusProbe::Found(FocusedElementSummary {
role: "entry".to_string(),
name: None,
editable: false,
states: vec!["focused".to_string(), "editable".to_string()],
is_terminal: false,
}))
})
.await
.unwrap();
assert_eq!(
mode,
WtypeTextMode::Argument,
"the indicator's 250 ms budget must not gate text dispatch"
);
}
#[tokio::test]
async fn review_inaccessible_fields_keep_short_text_on_private_stdin() {
let text = "ş".repeat(WTYPE_STDIN_MAX_CHARS);
for probe in [
None,
Some(FocusProbe::NoAccessibleApp),
Some(FocusProbe::Incomplete),
Some(FocusProbe::NoneFocused),
Some(FocusProbe::ShellOnly),
] {
let mode = hyprland_wtype_delivery(&text, async { probe })
.await
.unwrap();
assert_eq!(mode, WtypeTextMode::Stdin);
// cat accepts only '-' here; an argv text argument would instead
// be opened as a filename and fail. No desktop input is generated.
let result = run_wtype_type_text_or_fallback(
Path::new("/bin/cat"),
&text,
mode,
true,
|| async { panic!("available stdin helper must not fall back") },
)
.await
.unwrap();
assert_eq!(result.output.stdout, text.as_bytes());
}
}
#[tokio::test]
async fn review_ambiguous_focused_roles_use_stdin_instead_of_password_rejection() {
let short = "ş".repeat(WTYPE_STDIN_MAX_CHARS);
let long = "ş".repeat(WTYPE_STDIN_MAX_CHARS + 1);
for (role, terminal) in [
("terminal", true),
("Terminal", true),
("entry", true),
("Entry", false),
("Text", false),
("text box", false),
("TextBox", false),
("textbox", false),
("Unknown", false),
("invalid", false),
("", false),
] {
// Editable state/interface must not make an ambiguous role public.
for editable in [false, true] {
let element = FocusedElementSummary {
role: role.to_string(),
name: None,
editable,
states: vec!["focused".to_string(), "editable".to_string()],
is_terminal: terminal,
};
let mode = hyprland_wtype_delivery(&short, async {
Some(FocusProbe::Found(element.clone()))
})
.await;
assert!(
matches!(mode, Ok(WtypeTextMode::Stdin)),
"{role:?}, editable={editable}: {mode:?}"
);
let delivered = run_wtype_type_text_or_fallback(
Path::new("/bin/cat"),
&short,
mode.unwrap(),
true,
|| async { panic!("stdin helper must not fall back") },
)
.await
.unwrap();
assert_eq!(delivered.output.stdout, short.as_bytes());
let error =
hyprland_wtype_delivery(&long, async { Some(FocusProbe::Found(element)) })
.await
.unwrap_err();
assert!(
error.contains("100 Unicode characters"),
"{role:?}: {error}"
);
assert!(!error.contains("password fields"), "{role:?}: {error}");
}
}
}
#[tokio::test]
async fn only_confirmed_password_roles_are_hard_rejected() {
for role in [
"password text",
"PasswordText",
"PASSWORD TEXT",
"password entry",
] {
let error = hyprland_wtype_delivery("short", async {
Some(FocusProbe::Found(FocusedElementSummary {
role: role.to_string(),
name: None,
editable: true,
states: vec!["focused".to_string(), "editable".to_string()],
is_terminal: false,
}))
})
.await
.unwrap_err();
assert!(error.contains("password fields"), "{role}: {error}");
}
for role in ["entry", "text"] {
let mode = hyprland_wtype_delivery(&"x".repeat(3000), async {
Some(FocusProbe::Found(FocusedElementSummary {
role: role.to_string(),
name: None,
editable: false,
states: vec!["focused".to_string(), "editable".to_string()],
is_terminal: false,
}))
})
.await
.unwrap();
assert_eq!(mode, WtypeTextMode::Argument, "{role}");
}
}
#[tokio::test]
async fn unknown_stdin_limit_and_password_rejection_remain_explicit() {
let error = hyprland_wtype_delivery(&"ş".repeat(WTYPE_STDIN_MAX_CHARS + 1), async {
Some(FocusProbe::NoAccessibleApp)
})
.await
.unwrap_err();
assert!(error.contains("100 Unicode characters"), "{error}");
let error = hyprland_wtype_delivery("short", async {
Some(FocusProbe::Found(FocusedElementSummary {
role: "password text".to_string(),
name: None,
editable: true,
states: vec!["focused".to_string(), "editable".to_string()],
is_terminal: false,
}))
})
.await
.unwrap_err();
assert!(error.contains("password fields"), "{error}");
let mode = hyprland_wtype_delivery("short", std::future::pending())
.await
.unwrap();
assert_eq!(
mode,
WtypeTextMode::Stdin,
"timeout must not reject all input"
);
}
#[test]
fn electron_editable_state_does_not_require_the_editable_text_interface() {
let mut element = FocusedElementSummary {
role: "entry".to_string(),
name: None,
editable: false,
states: vec!["focused".to_string(), "editable".to_string()],
is_terminal: false,
};
assert!(!focus_requires_secret_protection(&FocusProbe::Found(
element.clone()
)));
assert!(!describe_focused_element(&element, true).starts_with("WARNING:"));
element.role = "password text".to_string();
assert!(focus_requires_secret_protection(&FocusProbe::Found(
element.clone()
)));
element.role = "frame".to_string();
element.states.clear();
assert!(focus_requires_secret_protection(&FocusProbe::Found(
element
)));
for probe in [
FocusProbe::ShellOnly,
FocusProbe::NoneFocused,
FocusProbe::Incomplete,
FocusProbe::NoAccessibleApp,
] {
assert!(focus_requires_secret_protection(&probe));
}
}
#[tokio::test]
async fn other_compositors_keep_wtype_text_on_stdin() {
for desktop in [None, Some("sway"), Some("wlroots"), Some("river")] {
let mode = wtype_text_mode_for_desktop(desktop);
assert_eq!(mode, WtypeTextMode::Stdin);
let text = "synthetic secret: şü / -- -";
let result = run_wtype_type_text_or_fallback(
Path::new("/bin/cat"),
text,
mode,
true,
|| async { panic!("available stdin helper must not fall back") },
)
.await
.unwrap();
assert_eq!(result.output.stdout, text.as_bytes());
}
assert_eq!(
wtype_text_mode_for_desktop(Some("Hyprland")),
WtypeTextMode::Argument
);
}
#[test]
fn review_invalid_text_is_rejected_before_window_targeting() {
let dir = std::env::temp_dir().join(format!("cul-text-preflight-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let helper = dir.join("wtype");
std::fs::write(&helper, "#!/bin/sh\nexit 99\n").unwrap();
std::fs::set_permissions(&helper, std::fs::Permissions::from_mode(0o700)).unwrap();
let mut command = Command::new(std::env::current_exe().unwrap());
command
.args([
"--exact",
"server::tests::invalid_text_preflight_child",
"--ignored",
"--nocapture",
])
.env("CUL_PREFLIGHT_CHILD", "1")
.env("PATH", &dir)
.env("XDG_SESSION_TYPE", "wayland")
.env("XDG_CURRENT_DESKTOP", "Hyprland")
// Set all related environment variables to prevent inherited values.
.env("DESKTOP_SESSION", "hyprland")
.env("XDG_SESSION_DESKTOP", "Hyprland")
.env_remove("DISPLAY");
for key in [
"COMPUTER_USE_LINUX_FORCE_YDOTOOL_KEYBOARD",
"COMPUTER_USE_LINUX_FORCE_XDOTOOL_KEYBOARD",
"COMPUTER_USE_LINUX_FORCE_PORTAL_KEYBOARD",
] {
command.env_remove(key);
}
let result = crate::command_runner::output_blocking_with_timeout(
&mut command,
"text preflight child",
Duration::from_secs(10),
)
.unwrap();
let _ = std::fs::remove_dir_all(dir);
assert!(
result.status.success(),
"{}{}",
String::from_utf8_lossy(&result.stdout),
String::from_utf8_lossy(&result.stderr)
);
}
#[tokio::test]
#[ignore = "isolated environment subprocess; invoked by review_invalid_text_is_rejected_before_window_targeting"]
async fn invalid_text_preflight_child() {
assert_eq!(env::var("CUL_PREFLIGHT_CHILD").as_deref(), Ok("1"));
let server = ComputerUseLinux::default();
assert_eq!(server.wtype_text_mode(), Some(WtypeTextMode::Argument));
for text in [
"prefix\0suffix".to_string(),
"ş".repeat(WTYPE_MAX_TEXT_BYTES / 2 + 1),
] {
let params: TypeTextParams =
serde_json::from_value(serde_json::json!({"text": text, "window_id": crate::windowing::types::MAX_WINDOW_ID}))
.unwrap();
let result = server.type_text(Parameters(params)).await.unwrap();
let structured = result
.structured_content
.expect("structured type_text output");
assert_eq!(structured["ok"], serde_json::json!(false));
let message = structured["message"].as_str().unwrap();
assert!(
message.contains("Literal text was not sent: wtype"),
"{message}"
);
}
}
#[tokio::test]
async fn unavailable_wtype_uses_ydotool_fallback() {
let result = run_wtype_type_text_or_fallback(
Path::new("/definitely/missing/wtype"),
"text",
WtypeTextMode::Argument,
false,
|| async {
TokioCommand::new("sh")
.args(["-c", "exit 0"])
.output()
.await
.map_err(|error| error.to_string())
},
)
.await
.expect("missing wtype should use fallback");
assert_eq!(result.backend, KeyboardCommandBackend::Ydotool);
}
#[test]
fn wtype_text_validation_counts_bytes_and_rejects_nul() {
assert!(validate_wtype_text(&"x".repeat(WTYPE_MAX_TEXT_BYTES)).is_ok());
assert!(validate_wtype_text(&"x".repeat(WTYPE_MAX_TEXT_BYTES + 1))
.unwrap_err()
.contains("not sent"));
// The argument limit counts UTF-8 bytes, not Unicode scalar values.
assert!(validate_wtype_text(&"ş".repeat(WTYPE_MAX_TEXT_BYTES / 2 + 1)).is_err());
assert!(validate_wtype_text("valid prefix\0suffix")
.unwrap_err()
.contains("not sent"));
assert!(validate_wtype_text("").is_ok());
}
/// Write an executable script to `path` through a sibling file and a
/// rename, so no writable descriptor to `path` exists when it runs. A
/// child forked by another test thread while the script was open for
/// writing would otherwise make its exec fail with ETXTBSY.
fn publish_fake_executable(path: &Path, script: &str) {
let source = path.with_extension("source");
std::fs::write(&source, script).expect("write fake executable");
std::fs::set_permissions(&source, std::fs::Permissions::from_mode(0o700))
.expect("make fake executable runnable");
std::fs::rename(&source, path).expect("publish fake executable");
}
async fn fallback_ran_instead_of_wtype(wtype: &Path) -> KeyboardCommandBackend {
run_wtype_type_text_or_fallback(wtype, "text", WtypeTextMode::Argument, false, || async {
TokioCommand::new("true")
.output()
.await
.map_err(|error| error.to_string())
})
.await
.expect("a wtype that cannot start should use the fallback")
.backend
}
#[tokio::test]
async fn non_executable_wtype_uses_ydotool_fallback() {
let dir = std::env::temp_dir().join(format!(
"computer-use-linux-wtype-nonexec-{}-{:?}",
std::process::id(),
std::time::SystemTime::now()
));
std::fs::create_dir_all(&dir).expect("create command test directory");
let wtype = dir.join("wtype");
std::fs::write(&wtype, "#!/bin/sh\nexit 0\n").expect("write fake wtype");
assert_eq!(
fallback_ran_instead_of_wtype(&wtype).await,
KeyboardCommandBackend::Ydotool
);
let _ = std::fs::remove_dir_all(dir);
}
#[tokio::test]
async fn missing_wtype_interpreter_uses_ydotool_fallback() {
let dir = std::env::temp_dir().join(format!(
"computer-use-linux-wtype-interpreter-{}-{:?}",
std::process::id(),
std::time::SystemTime::now()
));
std::fs::create_dir_all(&dir).expect("create command test directory");
let wtype = dir.join("wtype");
publish_fake_executable(&wtype, "#!/definitely/missing/interpreter\n");
assert_eq!(
fallback_ran_instead_of_wtype(&wtype).await,
KeyboardCommandBackend::Ydotool
);
let _ = std::fs::remove_dir_all(dir);
}
#[test]
fn a_program_on_path_counts_only_when_it_is_executable() {
// No `:` in the name, so the directory can sit in a PATH value.
let dir = std::env::temp_dir().join(format!(
"computer-use-linux-wtype-path-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir_all(&dir).expect("create command test directory");
let wtype = dir.join("wtype");
std::fs::write(&wtype, "#!/bin/sh\nexit 0\n").expect("write fake wtype");
let path = std::env::join_paths([std::path::Path::new("/nonexistent"), dir.as_path()])
.expect("join PATH");
assert!(!executable_in(&path, "wtype"));
std::fs::set_permissions(&wtype, std::fs::Permissions::from_mode(0o700))
.expect("make fake wtype executable");
assert!(executable_in(&path, "wtype"));
assert!(!executable_in(&path, "xdotool"));
let _ = std::fs::remove_dir_all(dir);
}
#[tokio::test]
async fn launched_wtype_failure_does_not_replay_through_ydotool() {
let dir = std::env::temp_dir().join(format!(
"computer-use-linux-wtype-fallback-{}-{:?}",
std::process::id(),
std::time::SystemTime::now()
));
std::fs::create_dir_all(&dir).expect("create command test directory");
let wtype = dir.join("wtype");
let fallback_marker = dir.join("ydotool-ran");
publish_fake_executable(&wtype, "#!/bin/sh\nexit 9\n");
let result = run_wtype_type_text_or_fallback(
&wtype,
"text",
WtypeTextMode::Argument,
false,
|| async {
std::fs::write(&fallback_marker, "ran").unwrap();
TokioCommand::new("true")
.output()
.await
.map_err(|error| error.to_string())
},
)
.await;
assert!(result.is_err());
assert!(
!fallback_marker.exists(),
"ydotool replayed input after wtype started"
);
let _ = std::fs::remove_dir_all(dir);
}
#[tokio::test]
async fn cancelling_xdotool_wait_kills_the_child() {
let dir = std::env::temp_dir().join(format!(
"computer-use-linux-xdotool-cancel-{}-{:?}",
std::process::id(),
std::time::SystemTime::now()
));
std::fs::create_dir_all(&dir).expect("create command test directory");
let xdotool = dir.join("xdotool");
let pid_path = dir.join("pid");
std::fs::write(
&xdotool,
format!(
"#!/bin/sh\nprintf '%s' $$ > '{}'\nexec sleep 60\n",
pid_path.display()
),
)
.expect("write fake xdotool");
std::fs::set_permissions(&xdotool, std::fs::Permissions::from_mode(0o700))
.expect("make fake xdotool executable");
let task = tokio::spawn(async move { run_xdotool(&xdotool, &[]).await });
let mut pid = None;
for _ in 0..200 {
if let Ok(value) = std::fs::read_to_string(&pid_path) {
pid = value.parse::<u32>().ok();
if pid.is_some() {
break;
}
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
let pid = pid.expect("fake xdotool did not record its pid");
task.abort();
let _ = task.await;
for _ in 0..50 {
if !Path::new(&format!("/proc/{pid}")).exists() {
let _ = std::fs::remove_dir_all(dir);
return;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
unsafe {
libc::kill(pid as i32, libc::SIGKILL);
}
let _ = std::fs::remove_dir_all(dir);
panic!("cancelled xdotool child {pid} was not killed");
}
#[tokio::test]
async fn cancelling_between_press_and_release_keeps_input_locked() {
let lock = std::sync::Arc::new(tokio::sync::Mutex::new(()));
let guard = std::sync::Arc::clone(&lock).lock_owned().await;
let (pressed_tx, pressed_rx) = tokio::sync::oneshot::channel();
let (allow_release_tx, allow_release_rx) = tokio::sync::oneshot::channel();
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
let waiter = tokio::spawn(async move {
run_cancellation_safe_input(guard, async move {
let _ = pressed_tx.send(());
let _ = allow_release_rx.await;
let _ = released_tx.send(());
Ok::<(), String>(())
})
.await
});
pressed_rx.await.expect("press command did not finish");
waiter.abort();
let _ = waiter.await;
assert!(
lock.try_lock().is_err(),
"input lock was released while the paired operation was incomplete"
);
allow_release_tx
.send(())
.expect("release command stopped on caller cancellation");
timeout(Duration::from_secs(1), released_rx)
.await
.expect("release command did not finish")
.expect("release command dropped its completion marker");
timeout(
Duration::from_secs(1),
std::sync::Arc::clone(&lock).lock_owned(),
)
.await
.expect("input lock remained held after the operation finished");
}
#[tokio::test]
async fn cancelled_clipboard_restore_keeps_both_operation_locks() {
let input_lock = std::sync::Arc::new(tokio::sync::Mutex::new(()));
let clipboard_lock = std::sync::Arc::new(tokio::sync::Mutex::new(()));
let input_guard = std::sync::Arc::clone(&input_lock).lock_owned().await;
let clipboard_guard = std::sync::Arc::clone(&clipboard_lock).lock_owned().await;
let (pasted_tx, pasted_rx) = tokio::sync::oneshot::channel();
let (allow_restore_tx, allow_restore_rx) = tokio::sync::oneshot::channel();
let (restored_tx, restored_rx) = tokio::sync::oneshot::channel();
let waiter = tokio::spawn(async move {
run_cancellation_safe_guarded((input_guard, clipboard_guard), async move {
let _ = pasted_tx.send(());
let _ = allow_restore_rx.await;
let _ = restored_tx.send(());
})
.await
});
// Cancel the caller after the paste, before the clipboard restore.
pasted_rx.await.expect("paste did not finish");
waiter.abort();
let _ = waiter.await;
assert!(input_lock.try_lock().is_err());
assert!(clipboard_lock.try_lock().is_err());
allow_restore_tx
.send(())
.expect("clipboard restore stopped on caller cancellation");
timeout(Duration::from_secs(1), restored_rx)
.await
.expect("clipboard restore did not finish")
.expect("clipboard restore dropped its completion marker");
tokio::task::yield_now().await;
assert!(input_lock.try_lock().is_ok());
assert!(clipboard_lock.try_lock().is_ok());
}
/// The xdotool path must accept exactly the keys the evdev grammar accepts,
/// so switching backends can never silently widen or narrow the surface.
#[test]
fn xdotool_key_spec_rejects_everything_key_chord_rejects() {
for key in ["NotAKey", "", "ctrl+", "ctrl+NotAKey", "f13", "hyper+a"] {
assert_eq!(
xdotool_key_spec(key).is_some(),
key_chord(key).is_some(),
"backend grammars diverged for {key:?}"
);
}
}
#[test]
fn key_sequence_keeps_shortcuts_and_navigation_on_raw_events() {
assert_eq!(
key_sequence("Ctrl+L"),
Some(vec![
"29:1".to_string(),
"38:1".to_string(),
"38:0".to_string(),
"29:0".to_string(),
])
);
assert_eq!(
key_sequence("ArrowLeft"),
Some(vec!["105:1".to_string(), "105:0".to_string()])
);
assert_eq!(
key_sequence("Escape"),
Some(vec!["1:1".to_string(), "1:0".to_string()])
);
assert_eq!(
key_sequence("Enter"),
Some(vec!["28:1".to_string(), "28:0".to_string()])
);
}
#[test]
fn ydotool_type_timeout_scales_with_text_length() {
assert_eq!(ydotool_type_timeout("").as_secs(), 10);
assert_eq!(ydotool_type_timeout("x").as_secs(), 11);
assert_eq!(ydotool_type_timeout(&"x".repeat(200)).as_secs(), 20);
assert_eq!(ydotool_type_timeout(&"x".repeat(500)).as_secs(), 35);
}
#[tokio::test]
async fn command_wait_drains_output_before_exit() {
let mut command = tokio::process::Command::new("sh");
command.args(["-c", "yes noisy | head -c 200000 >&2; exit 7"]);
command.stdout(Stdio::piped());
command.stderr(Stdio::piped());
let output = crate::command_runner::output_with_timeout(
command,
"run test command",
Duration::from_secs(5),
)
.await
.expect("child should exit before timeout");
assert_eq!(output.status.code(), Some(7));
assert!(output.stderr.len() >= 100_000);
}
#[test]
fn perform_action_defaults_to_primary_action_index() {
assert_eq!(requested_or_primary_action(None), "0");
assert_eq!(requested_or_primary_action(Some(" ")), "0");
assert_eq!(
requested_or_primary_action(Some(" show-menu ")),
"show-menu"
);
}
#[test]
fn element_identifier_overrides_cached_object_ref() {
let backend = ComputerUseLinux::default();
backend.cache_nodes(&[node(7, None)]);
let object_ref = backend
.resolve_object_ref(
Some(7),
Some(":1.99/org/a11y/atspi/accessible/3"),
&ElementSelector::default(),
ElementResolvePurpose::Action,
)
.unwrap();
assert_eq!(object_ref, ":1.99/org/a11y/atspi/accessible/3");
}
#[test]
fn element_index_resolves_to_cached_object_ref() {
let backend = ComputerUseLinux::default();
backend.cache_nodes(&[node(7, None)]);
let object_ref = backend
.resolve_object_ref(
Some(7),
None,
&ElementSelector::default(),
ElementResolvePurpose::Action,
)
.unwrap();
assert_eq!(object_ref, ":1.7/org/a11y/atspi/accessible/7");
}
#[test]
fn semantic_selector_resolves_unique_cached_node_by_role_and_name() {
let backend = ComputerUseLinux::default();
let mut search_entry = node(7, None);
search_entry.role = "entry".to_string();
search_entry.name = Some("Search files".to_string());
search_entry.supports_editable_text = true;
backend.cache_nodes(&[search_entry]);
let object_ref = backend
.resolve_object_ref(
None,
None,
&ElementSelector {
role: Some("entry"),
name: Some("search"),
..Default::default()
},
ElementResolvePurpose::SetValue,
)
.unwrap();
assert_eq!(object_ref, ":1.7/org/a11y/atspi/accessible/7");
}
#[test]
fn semantic_selector_prefers_actionable_match() {
let backend = ComputerUseLinux::default();
let mut label = node(4, None);
label.role = "label".to_string();
label.name = Some("Close".to_string());
let mut button = node_with_actions(7, None, vec![click_action()]);
button.role = "push button".to_string();
button.name = Some("Close".to_string());
backend.cache_nodes(&[label, button]);
let object_ref = backend
.resolve_object_ref(
None,
None,
&ElementSelector {
name: Some("close"),
..Default::default()
},
ElementResolvePurpose::Action,
)
.unwrap();
assert_eq!(object_ref, ":1.7/org/a11y/atspi/accessible/7");
}
#[test]
fn semantic_selector_prefers_editable_match() {
let backend = ComputerUseLinux::default();
let mut label = node(4, None);
label.role = "label".to_string();
label.name = Some("Search".to_string());
let mut entry = node(7, None);
entry.role = "entry".to_string();
entry.name = Some("Search".to_string());
entry.supports_editable_text = true;
backend.cache_nodes(&[label, entry]);
let object_ref = backend
.resolve_object_ref(
None,
None,
&ElementSelector {
name: Some("search"),
..Default::default()
},
ElementResolvePurpose::SetValue,
)
.unwrap();
assert_eq!(object_ref, ":1.7/org/a11y/atspi/accessible/7");
}
#[test]
fn semantic_selector_reports_ambiguous_matches() {
let backend = ComputerUseLinux::default();
let mut first = node_with_actions(7, None, vec![click_action()]);
first.name = Some("Close".to_string());
let mut second = node_with_actions(9, None, vec![click_action()]);
second.name = Some("Close".to_string());
backend.cache_nodes(&[first, second]);
let error = backend
.resolve_object_ref(
None,
None,
&ElementSelector {
name: Some("close"),
..Default::default()
},
ElementResolvePurpose::Action,
)
.unwrap_err();
assert!(error.contains("matched multiple cached nodes"));
assert!(error.contains("element_index 7"));
assert!(error.contains("element_index 9"));
}
#[test]
fn semantic_click_selector_prefers_native_activation() {
let backend = ComputerUseLinux::default();
let mut button = node_with_actions(
7,
Some(Bounds {
x: 10,
y: 20,
width: 100,
height: 40,
}),
vec![click_action()],
);
button.name = Some("Run".to_string());
backend.cache_nodes(&[button]);
let target = backend
.resolve_click_target(&ClickParams {
role: Some("button".to_string()),
name: Some("run".to_string()),
..Default::default()
})
.unwrap();
assert!(matches!(
target,
ClickTarget::PrimaryAction {
action_index: 0,
..
}
));
}
#[test]
fn describe_focused_element_editable() {
let element = FocusedElementSummary {
role: "text".to_string(),
name: Some("Message".to_string()),
editable: true,
states: vec!["focused".to_string()],
is_terminal: false,
};
let described = describe_focused_element(&element, true);
assert!(described.contains("editable"));
assert!(!described.contains("WARNING"));
}
#[test]
fn describe_focused_terminal_accepts_keyboard_input_without_editable_text() {
let mut element = focused("terminal", true);
element.name = Some("Terminal".to_string());
let text = describe_focused_element(&element, true);
assert!(!text.contains("WARNING"), "{text}");
assert!(text.contains("terminal keyboard input"), "{text}");
}
#[test]
fn describe_focused_element_warns_on_non_editable_when_typing() {
let element = FocusedElementSummary {
role: "push button".to_string(),
name: Some("OK".to_string()),
editable: false,
states: vec!["focused".to_string()],
is_terminal: false,
};
let described = describe_focused_element(&element, true);
assert!(described.contains("WARNING"));
assert!(described.contains("not editable"));
}
#[test]
fn describe_focused_element_no_warning_for_press_key() {
let element = FocusedElementSummary {
role: "push button".to_string(),
name: None,
editable: false,
states: vec![],
is_terminal: false,
};
let described = describe_focused_element(&element, false);
assert!(!described.contains("WARNING"));
}
#[test]
fn relative_scroll_translates_coordinates() {
let mut params = ScrollParams {
element_index: None,
x: Some(10),
y: Some(20),
direction: "down".to_string(),
pages: None,
window_id: Some(1),
pid: None,
app_id: None,
wm_class: None,
window_title: None,
relative: Some(true),
..Default::default()
};
apply_window_relative_scroll_coordinates(&mut params, (100, 200, 800, 600)).unwrap();
assert_eq!(params.x, Some(110));
assert_eq!(params.y, Some(220));
}
#[test]
fn window_targeted_scroll_defaults_to_window_center() {
let mut params = ScrollParams {
element_index: None,
x: None,
y: None,
direction: "down".to_string(),
pages: None,
window_id: Some(1),
pid: None,
app_id: None,
wm_class: None,
window_title: None,
relative: None,
..Default::default()
};
apply_window_center_scroll_point(&mut params, (100, 200, 800, 600)).unwrap();
assert_eq!(params.x, Some(500));
assert_eq!(params.y, Some(500));
}
#[test]
fn window_targeted_scroll_with_empty_capture_rect_errors() {
let mut params = ScrollParams {
element_index: None,
x: None,
y: None,
direction: "down".to_string(),
pages: None,
window_id: Some(1),
pid: None,
app_id: None,
wm_class: None,
window_title: None,
relative: None,
..Default::default()
};
let error = apply_window_center_scroll_point(&mut params, (0, 0, 0, 0)).unwrap_err();
assert!(error.contains("pass x/y explicitly"));
assert_eq!(params.x, None);
assert_eq!(params.y, None);
}
#[test]
fn relative_scroll_rejects_out_of_bounds() {
let mut params = ScrollParams {
element_index: None,
x: Some(801),
y: Some(20),
direction: "down".to_string(),
pages: None,
window_id: Some(1),
pid: None,
app_id: None,
wm_class: None,
window_title: None,
relative: Some(true),
..Default::default()
};
assert!(
apply_window_relative_scroll_coordinates(&mut params, (100, 200, 800, 600)).is_err()
);
}
#[test]
fn shell_execution_requires_exact_operator_opt_in() {
assert!(shell_execution_enabled_value(Some("1")));
for value in [None, Some(""), Some("0"), Some("true"), Some("yes")] {
assert!(!shell_execution_enabled_value(value));
}
}
#[tokio::test]
async fn shell_execution_clears_ambient_credentials_and_accepts_explicit_env() {
let _serial = SHELL_ENV_TEST_LOCK.lock().await;
let _enabled = EnvVarGuard::set(SHELL_ENABLE_ENV, "1");
let _credential = EnvVarGuard::set("OPENAI_API_KEY", "test-ambient-credential");
let output = execute_shell(RunShellParams {
command: "printf '%s|%s' \"${OPENAI_API_KEY-unset}\" \"$EXPLICIT_VALUE\"".into(),
cwd: None,
env: BTreeMap::from([("EXPLICIT_VALUE".into(), "test-explicit-value".into())]),
timeout_seconds: Some(2),
})
.await;
assert!(output.ok, "{:?}", output.error);
assert_eq!(output.stdout, "unset|test-explicit-value");
assert_eq!(output.exit_code, Some(0));
}
#[tokio::test]
async fn shell_execution_disabled_never_runs_requested_command() {
async fn assert_refused() {
let output = execute_shell(RunShellParams {
command: "printf must-not-run".into(),
cwd: None,
env: BTreeMap::new(),
timeout_seconds: None,
})
.await;
assert!(!output.ok);
assert!(output.stdout.is_empty());
assert!(output
.error
.unwrap()
.contains("shell execution is disabled"));
}
let _serial = SHELL_ENV_TEST_LOCK.lock().await;
{
let _disabled = EnvVarGuard::set(SHELL_ENABLE_ENV, "0");
assert_refused().await;
}
{
let _unset = EnvVarGuard::unset(SHELL_ENABLE_ENV);
assert_refused().await;
}
}
#[test]
fn shell_environment_policy_excludes_ambient_credentials() {
for allowed in [
"PATH",
"HOME",
"LANG",
"LC_ALL",
"XDG_RUNTIME_DIR",
"DISPLAY",
"WAYLAND_DISPLAY",
"DBUS_SESSION_BUS_ADDRESS",
] {
assert!(inherited_shell_environment_allowed(allowed));
}
for secret in [
"AWS_SECRET_ACCESS_KEY",
"GITHUB_TOKEN",
"OPENAI_API_KEY",
"SSH_AUTH_SOCK",
"LD_PRELOAD",
] {
assert!(!inherited_shell_environment_allowed(secret));
}
}
#[test]
fn shell_environment_skips_non_utf8_entries() {
use std::os::unix::ffi::OsStringExt;
let variables = [
(OsString::from("PATH"), OsString::from("/usr/bin")),
(OsString::from_vec(vec![0xff]), OsString::from("ignored")),
(OsString::from("LANG"), OsString::from_vec(vec![0xff])),
(OsString::from("GITHUB_TOKEN"), OsString::from("secret")),
];
assert_eq!(
inherited_shell_environment(variables),
vec![("PATH".to_string(), "/usr/bin".to_string())]
);
}
#[test]
fn shell_environment_names_follow_exec_rules() {
for valid in ["A", "_PRIVATE", "NAME_2"] {
assert!(valid_environment_name(valid));
}
for invalid in ["", "2FAST", "BAD-NAME", "A=B", "naïve"] {
assert!(!valid_environment_name(invalid));
}
}
#[test]
fn shell_output_is_bounded_before_mcp_serialization() {
let oversized = vec![b'x'; SHELL_RESPONSE_STREAM_BYTES + 17];
let (visible, truncated) = bounded_shell_stream(&oversized);
assert!(truncated);
assert_eq!(visible.len(), SHELL_RESPONSE_STREAM_BYTES);
let (visible, truncated) = bounded_shell_stream(b"ok");
assert!(!truncated);
assert_eq!(visible, "ok");
}
#[test]
fn shell_audit_digest_is_stable_and_does_not_echo_command_text() {
let digest = shell_command_sha256("printf secret");
assert_eq!(digest.len(), 64);
assert!(digest
.chars()
.all(|character| character.is_ascii_hexdigit()));
assert!(!digest.contains("secret"));
assert_eq!(digest, shell_command_sha256("printf secret"));
}
}
#[cfg(test)]
mod app_state_pid_scope_tests {
use super::*;
use crate::test_sockets::PrivateBus;
use std::os::unix::fs::PermissionsExt;
use zbus::zvariant::OwnedObjectPath;
const ROOT_PATH: &str = "/org/a11y/atspi/accessible/root";
struct FixtureAccessible {
name: String,
role: &'static str,
children: Vec<(String, OwnedObjectPath)>,
has_action: bool,
application: Option<(String, OwnedObjectPath)>,
}
#[zbus::interface(name = "org.a11y.atspi.Accessible")]
impl FixtureAccessible {
#[zbus(property)]
fn child_count(&self) -> i32 {
self.children.len() as i32
}
#[zbus(property)]
fn name(&self) -> &str {
&self.name
}
fn get_role_name(&self) -> &str {
self.role
}
fn get_application(&self) -> ((String, OwnedObjectPath),) {
(self
.application
.clone()
.unwrap_or_else(|| (String::new(), "/org/a11y/atspi/null".try_into().unwrap())),)
}
fn get_interfaces(&self) -> Vec<&str> {
let mut interfaces = vec!["org.a11y.atspi.Accessible"];
if self.has_action {
interfaces.push("org.a11y.atspi.Action");
}
interfaces
}
fn get_child_at_index(
&self,
index: i32,
) -> zbus::fdo::Result<((String, OwnedObjectPath),)> {
self.children
.get(index as usize)
.cloned()
.map(|child| (child,))
.ok_or_else(|| zbus::fdo::Error::InvalidArgs("invalid child index".into()))
}
}
struct FixtureAction(Box<dyn Fn() + Send + Sync>);
#[zbus::interface(name = "org.a11y.atspi.Action")]
impl FixtureAction {
#[zbus(property, name = "NActions")]
fn nactions(&self) -> i32 {
1
}
fn get_name(&self, _index: i32) -> &str {
"press"
}
fn get_description(&self, _index: i32) -> &str {
""
}
fn get_key_binding(&self, _index: i32) -> &str {
""
}
fn do_action(&self, index: i32) -> bool {
if index != 0 {
return false;
}
(self.0)();
true
}
}
struct FixtureBus(String);
#[zbus::interface(name = "org.a11y.Bus")]
impl FixtureBus {
fn get_address(&self) -> &str {
&self.0
}
}
#[tokio::test]
async fn app_state_keeps_pid_scope_when_window_lookup_fails() {
let bus = PrivateBus::start("cul-app-state-pid");
let mut peers = Vec::new();
for child_count in [0, 1] {
let peer = zbus::connection::Builder::address(bus.address())
.unwrap()
.build()
.await
.unwrap();
let name = peer.unique_name().unwrap().to_string();
peer.object_server()
.at(
ROOT_PATH,
FixtureAccessible {
name: format!("Target root {child_count}"),
role: "application",
children: (0..child_count)
.map(|_| (name.clone(), "/button".try_into().unwrap()))
.collect(),
has_action: false,
application: None,
},
)
.await
.unwrap();
peer.object_server()
.at(
"/button",
FixtureAccessible {
name: "Target button".to_string(),
role: "push button",
children: Vec::new(),
has_action: false,
application: None,
},
)
.await
.unwrap();
peers.push(peer);
}
let dir = bus.dir();
for (program, script) in [
("systemctl", "#!/bin/sh\nexit 1\n"),
("xprop", "#!/bin/sh\nexit 1\n"),
(
"busctl",
r#"#!/bin/sh
case "$*" in
*"org.a11y.Bus /org/a11y/bus org.a11y.Bus GetAddress") printf 's "%s"\n' "$DBUS_SESSION_BUS_ADDRESS" ;;
*"org.a11y.Bus /org/a11y/bus org.a11y.Status IsEnabled") printf 'b true\n' ;;
*) exit 1 ;;
esac
"#,
),
(
"wmctrl",
r#"#!/bin/sh
[ -f "$CUL_PID_FIXTURE_WINDOWS" ] || exit 1
read -r mode fixture_pid < "$CUL_PID_FIXTURE_WINDOWS"
case "$mode" in
action)
printf '0x00000001 0 %s 0 0 100 100 other.Other host Window 1\n0x00000003 0 %s 0 0 100 100 other.Other host Window 3\n' "$fixture_pid" "$fixture_pid"
exit 0
;;
pidless) pid=0 ;;
proxied)
printf '0x00000009 0 999999 0 0 100 100 other.Other host Other app\n'
exit 0
;;
*) pid="$CUL_PID_FIXTURE_PID" ;;
esac
printf '0x00000007 0 %s 0 0 100 100 first.First host Target 7\n0x00000008 0 %s 0 0 100 100 second.Second host Target 8\n' "$pid" "$pid"
"#,
),
] {
let path = dir.join(program);
std::fs::write(&path, script).unwrap();
std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700)).unwrap();
}
let mut command = Command::new(std::env::current_exe().unwrap());
command
.args([
"--exact",
"server::app_state_pid_scope_tests::app_state_pid_scope_child",
"--ignored",
"--nocapture",
])
.env_clear()
.env("PATH", dir)
.env("HOME", dir)
.env("XDG_RUNTIME_DIR", dir)
.env("DBUS_SESSION_BUS_ADDRESS", bus.address())
.env("DISPLAY", ":9876")
.env("WAYLAND_DISPLAY", "")
.env("XDG_SESSION_TYPE", "x11")
.env("CUL_PID_FIXTURE_PID", std::process::id().to_string())
.env("CUL_PID_FIXTURE_WINDOWS", dir.join("windows"))
.env(
"CUL_PID_FIXTURE_ROOTS",
peers
.iter()
.map(|peer| peer.unique_name().unwrap().as_str())
.collect::<Vec<_>>()
.join(","),
);
for key in [
"DESKTOP_SESSION",
"HYPRLAND_INSTANCE_SIGNATURE",
"NIRI_SOCKET",
"XAUTHORITY",
"YDOTOOL_SOCKET",
"XDG_SESSION_DESKTOP",
"XDG_CURRENT_DESKTOP",
] {
command.env(key, "app-state-pid-fixture");
}
// Keep the parent's D-Bus peers serving while the isolated process
// runs app_state against roots owned by both processes.
let output = tokio::task::spawn_blocking(move || {
crate::command_runner::output_blocking_with_timeout(
&mut command,
"app_state PID scope child",
Duration::from_secs(30),
)
.unwrap()
})
.await
.unwrap();
assert!(String::from_utf8_lossy(&output.stdout).contains("running 1 test"));
assert!(
output.status.success(),
"{}{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
}
#[tokio::test]
#[ignore = "isolated environment subprocess; invoked by app_state_keeps_pid_scope_when_window_lookup_fails"]
async fn app_state_pid_scope_child() {
let target_pid: u32 = env::var("CUL_PID_FIXTURE_PID").unwrap().parse().unwrap();
let address = env::var("DBUS_SESSION_BUS_ADDRESS").unwrap();
let other = zbus::connection::Builder::address(address.as_str())
.unwrap()
.serve_at(
ROOT_PATH,
FixtureAccessible {
name: "Other app".to_string(),
role: "application",
children: Vec::new(),
has_action: false,
application: None,
},
)
.unwrap()
.build()
.await
.unwrap();
let roots = env::var("CUL_PID_FIXTURE_ROOTS").unwrap();
let children = roots
.split(',')
.chain(std::iter::once(other.unique_name().unwrap().as_str()))
.map(|name| (name.to_string(), ROOT_PATH.try_into().unwrap()))
.collect();
let _registry = zbus::connection::Builder::address(address.as_str())
.unwrap()
.name("org.a11y.Bus")
.unwrap()
.name("org.a11y.atspi.Registry")
.unwrap()
.serve_at("/org/a11y/bus", FixtureBus(address.clone()))
.unwrap()
.serve_at(
ROOT_PATH,
FixtureAccessible {
name: "Desktop".to_string(),
role: "desktop frame",
children,
has_action: false,
application: None,
},
)
.unwrap()
.build()
.await
.unwrap();
let (server, diagnostics_probes) =
super::tests::server_with_labelled_diagnostics(|| true, || false);
let server = Arc::new(server);
for windows_available in [false, true] {
if windows_available {
std::fs::write(env::var("CUL_PID_FIXTURE_WINDOWS").unwrap(), "").unwrap();
}
for pid in [target_pid, u32::MAX] {
let params = serde_json::json!({"pid": pid, "include_screenshot": false});
let parsed = serde_json::from_value(params.clone()).unwrap();
let (window, error, _, enumerated, _) =
server.resolve_window_context(&parsed).await;
assert!(window.is_none());
assert!(error.is_some());
assert_eq!(enumerated, windows_available);
let state = super::tests::app_state_value(&server, params).await;
assert!(state["accessibility_error"].is_null(), "{state}");
assert_eq!(state["tree_scoped"], true, "{state}");
let nodes = state["accessibility_tree"].as_array().unwrap();
let message = state["message"].as_str().unwrap();
if pid == target_pid {
let names = nodes
.iter()
.map(|node| node["name"].as_str().unwrap())
.collect::<Vec<_>>();
assert_eq!(names, ["Target root 0", "Target root 1", "Target button"]);
assert_eq!(server.last_snapshot.lock().unwrap().pid, Some(target_pid));
assert!(!message.contains("No AT-SPI application root matched"));
} else {
assert!(nodes.is_empty(), "a missing PID widened the tree: {state}");
assert_eq!(server.last_snapshot.lock().unwrap().pid, None);
assert!(message.contains(&pid.to_string()), "{message}");
assert!(message.contains("list_apps"), "{message}");
assert!(
message.contains("app_name_or_bundle_identifier"),
"{message}"
);
assert!(
!message.contains("tree extraction are working"),
"{message}"
);
}
}
}
let capture = PostActionCaptureParams {
include_tree: Some(true),
..Default::default()
};
let missing_target = WindowTarget {
pid: Some(u32::MAX),
..Default::default()
};
let (observation, _, notes) = server
.capture_post_action_observation(
&capture,
Some(&missing_target),
None,
TargetRaised::No,
PostActionScope::default(),
)
.await;
assert!(
observation.accessibility_error.is_none(),
"{observation:?}; {notes:?}"
);
assert_eq!(observation.accessibility_tree_raw_count, Some(0));
assert_eq!(observation.tree_scoped, Some(true));
assert!(notes.iter().any(|note| note.contains(&u32::MAX.to_string())
&& note.contains("list_apps")
&& note.contains("app_name_or_bundle_identifier")));
// A listed window whose pid owns no AT-SPI root, as for a Flatpak app
// behind xdg-dbus-proxy, finds its root by the window title. The tree
// records no pid, so element calls take the per-node owner check.
let windows_path = env::var("CUL_PID_FIXTURE_WINDOWS").unwrap();
std::fs::write(&windows_path, "proxied").unwrap();
let proxied = super::tests::app_state_value(
&server,
serde_json::json!({"window_id": 9, "include_screenshot": false}),
)
.await;
assert!(proxied["accessibility_error"].is_null(), "{proxied}");
let names = proxied["accessibility_tree"]
.as_array()
.unwrap()
.iter()
.map(|node| node["name"].as_str().unwrap())
.collect::<Vec<_>>();
assert_eq!(names, ["Other app"], "{proxied}");
assert_eq!(server.last_snapshot.lock().unwrap().pid, None);
assert!(!proxied["message"]
.as_str()
.unwrap()
.contains("No AT-SPI application root matched"));
let proxied_window = WindowTarget {
window_id: Some(9),
..Default::default()
};
let (observation, _, notes) = server
.capture_post_action_observation(
&capture,
Some(&proxied_window),
None,
TargetRaised::No,
PostActionScope::default(),
)
.await;
assert_eq!(
observation.accessibility_tree_raw_count,
Some(1),
"{observation:?}; {notes:?}"
);
assert_eq!(server.last_snapshot.lock().unwrap().pid, None);
// An element acted on in that window is bound to its bus owner, the
// proxy's pid here, not the pid the compositor lists for window 9.
let proxied_element = ElementObservationTarget {
owner: WindowTarget {
pid: Some(std::process::id()),
window_id: Some(9),
..Default::default()
},
tree_window: None,
};
let (observation, _, notes) = server
.capture_post_action_observation(
&capture,
Some(&proxied_element.owner),
None,
TargetRaised::No,
PostActionScope::bound_to(&proxied_element),
)
.await;
assert_eq!(
observation.accessibility_tree_raw_count,
Some(1),
"{observation:?}; {notes:?}"
);
// An owner pid with no root keeps the strict pid scope: no title fallback.
let bound_owner = ElementObservationTarget {
owner: WindowTarget {
pid: Some(999_999),
window_id: Some(9),
..Default::default()
},
tree_window: None,
};
let (observation, _, notes) = server
.capture_post_action_observation(
&capture,
Some(&bound_owner.owner),
None,
TargetRaised::No,
PostActionScope::bound_to(&bound_owner),
)
.await;
assert_eq!(
observation.accessibility_tree_raw_count,
Some(0),
"{observation:?}; {notes:?}"
);
assert!(notes.iter().any(|note| note.contains("999999")));
std::fs::write(&windows_path, "").unwrap();
let missing_window = super::tests::app_state_value(
&server,
serde_json::json!({
"window_id": 999,
"pid": std::process::id(),
"include_screenshot": false,
}),
)
.await;
assert!(
missing_window["window_error"].is_string(),
"{missing_window}"
);
assert!(missing_window["accessibility_tree"]
.as_array()
.unwrap()
.is_empty());
assert_ne!(target_pid, std::process::id());
other
.object_server()
.at(
"/embedded_button",
FixtureAccessible {
name: "Embedded WebKit button".to_string(),
role: "push button",
children: Vec::new(),
has_action: true,
application: Some((
roots.split(',').nth(1).unwrap().to_string(),
ROOT_PATH.try_into().unwrap(),
)),
},
)
.await
.unwrap();
other
.object_server()
.at("/embedded_button", FixtureAction(Box::new(|| {})))
.await
.unwrap();
server.clear_snapshot();
let params: ActionParams = serde_json::from_value(serde_json::json!({
"element_identifier": format!("{}/embedded_button", other.unique_name().unwrap()),
"include_tree": true,
}))
.unwrap();
server.diagnostics.report(false).await;
let probes = diagnostics_probes.load(std::sync::atomic::Ordering::SeqCst);
let observed = server
.perform_action(Parameters(params))
.await
.unwrap()
.structured_content
.unwrap();
assert_eq!(observed["ok"], true, "{observed}");
assert!(observed["accessibility_error"].is_null(), "{observed}");
let names = observed["accessibility_tree"]
.as_array()
.unwrap()
.iter()
.map(|node| node["name"].as_str().unwrap())
.collect::<Vec<_>>();
assert_eq!(names, ["Target root 0", "Target root 1", "Target button"]);
assert_eq!(server.last_snapshot.lock().unwrap().pid, Some(target_pid));
server.diagnostics.report(false).await;
assert_eq!(
diagnostics_probes.load(std::sync::atomic::Ordering::SeqCst),
probes,
"an app-wide element observation needs no unique compositor window"
);
let target_button = format!("{}/button", roots.split(',').nth(1).unwrap());
for fields in [
serde_json::json!({"window_title": "Target 8"}),
serde_json::json!({"app_id": "second"}),
serde_json::json!({"wm_class": "Second"}),
] {
let requested: ElementActionTarget = serde_json::from_value(fields).unwrap();
let bound = server
.element_observation_target(&requested, &target_button, None)
.await
.unwrap();
assert_eq!(bound.owner.pid, Some(target_pid));
assert_eq!(bound.owner.window_id, Some(8));
assert_eq!(
resolve_window_target(&list_windows().await.unwrap(), &bound.owner)
.unwrap()
.window_id,
8
);
assert!(server
.element_observation_target(
&requested,
&format!("{}{ROOT_PATH}", other.unique_name().unwrap()),
None,
)
.await
.is_err());
}
let windows_path = env::var("CUL_PID_FIXTURE_WINDOWS").unwrap();
std::fs::write(&windows_path, "pidless").unwrap();
let observed = super::tests::app_state_value(
&server,
serde_json::json!({
"window_id": 8,
"pid": std::process::id(),
"app_name_or_bundle_identifier": format!("{}{ROOT_PATH}", roots.split(',').nth(1).unwrap()),
"include_screenshot": false,
}),
)
.await;
assert!(observed["accessibility_error"].is_null(), "{observed}");
assert_eq!(
server.snapshot_window(),
Some((8, registry::X11_BACKEND.to_string()))
);
assert!(server.last_snapshot.lock().unwrap().window_scope.is_none());
let requested: ElementActionTarget =
serde_json::from_value(serde_json::json!({"window_title": "Target 8"})).unwrap();
let index = observed["accessibility_tree"]
.as_array()
.unwrap()
.iter()
.find(|node| node["name"] == "Target button")
.unwrap()["index"]
.as_u64()
.unwrap() as u32;
server
.check_element_action_target(
&requested,
Some(index),
None,
&ElementSelector::default(),
ElementResolvePurpose::Action,
)
.await
.unwrap();
let bound = server
.element_observation_target(&requested, &target_button, None)
.await
.unwrap();
assert_eq!(bound.owner.pid, Some(target_pid));
assert_eq!(bound.owner.window_id, Some(8));
// A raw identifier may pass the pidless target check without an
// observed window; sharing the process is not proof of that window.
let uncached_ref = format!("{}{ROOT_PATH}", roots.split(',').next().unwrap());
server
.check_element_action_target(
&requested,
None,
Some(&uncached_ref),
&ElementSelector::default(),
ElementResolvePurpose::Action,
)
.await
.unwrap();
let bound = server
.element_observation_target(&requested, &uncached_ref, None)
.await
.unwrap();
assert_eq!(bound.owner.pid, Some(target_pid));
assert_eq!(bound.owner.window_id, None);
std::fs::write(&windows_path, "").unwrap();
// Replace the snapshot during real dispatch, before its observation.
let name = other.unique_name().unwrap().to_string();
for path in ["/window_1", "/window_3"] {
other
.object_server()
.at(
path,
FixtureAccessible {
name: if path == "/window_1" {
"Window 1"
} else {
"Window 3"
}
.to_string(),
role: "frame",
children: Vec::new(),
has_action: path == "/window_1",
application: None,
},
)
.await
.unwrap();
}
other
.object_server()
.at(
"/popup",
FixtureAccessible {
name: "Popup action".to_string(),
role: "menu item",
children: Vec::new(),
has_action: true,
application: None,
},
)
.await
.unwrap();
other
.object_server()
.interface::<_, FixtureAccessible>(ROOT_PATH)
.await
.unwrap()
.get_mut()
.await
.children = ["/window_1", "/window_3", "/popup"]
.into_iter()
.map(|path| (name.clone(), path.try_into().unwrap()))
.collect();
let replaced = super::tests::app_state_value(
&server,
serde_json::json!({
"pid": std::process::id(),
"title": "Window 3",
"include_screenshot": false,
}),
)
.await;
assert!(replaced["accessibility_error"].is_null(), "{replaced}");
assert_eq!(
server.last_snapshot.lock().unwrap().window_scope.as_deref(),
Some("Window 3")
);
assert!(!replaced["accessibility_tree"]
.as_array()
.unwrap()
.iter()
.any(|node| node["name"] == "Window 1"));
let narrowed_nodes = server.last_snapshot.lock().unwrap().nodes.clone();
let action_server = Arc::clone(&server);
let action_windows_path = windows_path.clone();
let replace_snapshot = move || {
action_server.store_snapshot(
&narrowed_nodes,
Some(std::process::id()),
None,
None,
TreeShape {
detail: NodeDetail::Compact,
window_scope: Some("Window 3".to_string()),
},
);
std::fs::write(&action_windows_path, "").unwrap();
};
for path in ["/window_1", "/popup"] {
other
.object_server()
.at(path, FixtureAction(Box::new(replace_snapshot.clone())))
.await
.unwrap();
}
// A title target must be bound before DoAction removes its window
// from enumeration; neither call may inherit the replacement's scope.
for window_targeted in [false, true] {
std::fs::write(&windows_path, format!("action {}", std::process::id())).unwrap();
let observed = super::tests::app_state_value(
&server,
serde_json::json!({
"pid": std::process::id(),
"include_screenshot": false,
}),
)
.await;
assert!(observed["accessibility_error"].is_null(), "{observed}");
let params: ActionParams = serde_json::from_value(serde_json::json!({
"element_identifier": format!("{name}/window_1"),
"window_title": window_targeted.then_some("Window 1"),
"include_tree": true,
}))
.unwrap();
let result = server.perform_action(Parameters(params)).await.unwrap();
let result = result.structured_content.unwrap();
assert_eq!(result["ok"], true, "{result}");
assert!(result["accessibility_error"].is_null(), "{result}");
assert_eq!(result["tree_scoped"], true, "{result}");
let names = result["accessibility_tree"]
.as_array()
.unwrap()
.iter()
.map(|node| node["name"].as_str().unwrap())
.collect::<Vec<_>>();
let expected = if window_targeted {
vec!["Other app", "Window 1", "Popup action"]
} else {
vec!["Other app", "Window 1", "Window 3", "Popup action"]
};
assert_eq!(names, expected);
assert_eq!(std::fs::read_to_string(&windows_path).unwrap(), "");
}
// A sibling popup keeps the observed tree's scope without claiming
// that its own window is the observed main frame.
std::fs::write(&windows_path, format!("action {}", std::process::id())).unwrap();
let observed = super::tests::app_state_value(
&server,
serde_json::json!({"window_id": 1, "include_screenshot": false}),
)
.await;
assert!(observed["accessibility_error"].is_null(), "{observed}");
let popup = observed["accessibility_tree"]
.as_array()
.unwrap()
.iter()
.find(|node| node["name"] == "Popup action")
.unwrap();
let bound = server
.element_observation_target(
&ElementActionTarget::default(),
&format!("{name}/popup"),
None,
)
.await
.unwrap();
assert_eq!(bound.owner.window_id, None);
assert_eq!(
bound
.tree_window
.as_ref()
.map(|window| window.scope.as_str()),
Some("Window 1")
);
let params: ActionParams = serde_json::from_value(serde_json::json!({
"element_index": popup["index"],
"snapshot_id": observed["snapshot_id"],
"include_tree": true,
}))
.unwrap();
let result = server
.perform_action(Parameters(params))
.await
.unwrap()
.structured_content
.unwrap();
assert_eq!(result["ok"], true, "{result}");
assert!(result["accessibility_error"].is_null(), "{result}");
let names = result["accessibility_tree"]
.as_array()
.unwrap()
.iter()
.map(|node| node["name"].as_str().unwrap())
.collect::<Vec<_>>();
assert_eq!(names, ["Other app", "Window 1", "Popup action"]);
assert_eq!(
server.last_snapshot.lock().unwrap().window_scope.as_deref(),
Some("Window 1")
);
assert_eq!(
server.snapshot_window(),
Some((1, registry::X11_BACKEND.to_string()))
);
}
}
#[cfg(test)]
mod accessibility_tree_note_tests {
use super::{truncated_accessibility_tree_note, unscoped_accessibility_tree_warning};
#[test]
fn unscoped_warning_only_when_tree_succeeded_without_a_scope() {
let warning = unscoped_accessibility_tree_warning(false, true, false).unwrap();
assert!(warning.starts_with("WARNING:"));
assert!(warning.contains("no app target scoped"));
assert!(warning.contains("app_name_or_bundle_identifier"));
assert!(warning.contains("window_id, pid, app_id, wm_class, title"));
for target_requested in [false, true] {
assert!(unscoped_accessibility_tree_warning(true, true, target_requested).is_none());
assert!(unscoped_accessibility_tree_warning(false, false, target_requested).is_none());
assert!(unscoped_accessibility_tree_warning(true, false, target_requested).is_none());
}
}
/// A caller that already passed a target must not be told to pass a target;
/// that advice would replay the same desktop-wide snapshot.
#[test]
fn unscoped_warning_with_a_target_points_at_accessibility_support() {
let warning = unscoped_accessibility_tree_warning(false, true, true).unwrap();
assert!(warning.starts_with("WARNING:"));
assert!(warning.contains("matched no AT-SPI application root"));
assert!(warning.contains("force-renderer-accessibility"));
assert!(warning.contains("list_apps"));
assert!(!warning.contains("Pass app_name_or_bundle_identifier or a window target"));
}
#[test]
fn truncated_note_only_when_the_node_cap_was_hit() {
let note = truncated_accessibility_tree_note(true).unwrap();
assert!(note.contains("raise max_nodes or max_depth"));
assert!(
!note.contains("smaller max_nodes"),
"shrinking the cap belongs to the unscoped warning, not the truncation note"
);
assert!(truncated_accessibility_tree_note(false).is_none());
}
}
#[cfg(test)]
mod focus_probe_feedback_tests {
use super::{focus_probe_feedback, FocusProbe, FocusedElementSummary};
#[test]
fn only_a_complete_empty_search_warns() {
let none = focus_probe_feedback(&FocusProbe::NoneFocused, true, false);
assert!(none.starts_with("WARNING:"), "{none}");
for probe in [FocusProbe::Incomplete, FocusProbe::NoAccessibleApp] {
let text = focus_probe_feedback(&probe, true, false);
assert!(
!text.contains("WARNING"),
"{probe:?} must not claim lost input: {text}"
);
assert!(text.contains("could not be verified"), "{text}");
}
assert!(
focus_probe_feedback(&FocusProbe::Incomplete, true, false).contains("search limits")
);
assert!(
focus_probe_feedback(&FocusProbe::NoAccessibleApp, true, false)
.contains("--force-renderer-accessibility")
);
}
#[test]
fn review_exact_focus_only_softens_shell_only_tree() {
for expects_editable in [false, true] {
let text = focus_probe_feedback(&FocusProbe::NoneFocused, expects_editable, true);
assert!(text.starts_with("WARNING:"), "{text}");
let text = focus_probe_feedback(&FocusProbe::ShellOnly, expects_editable, false);
assert!(text.starts_with("WARNING:"), "{text}");
let text = focus_probe_feedback(&FocusProbe::ShellOnly, expects_editable, true);
assert!(!text.contains("WARNING"), "{text}");
assert!(!text.contains("landed nowhere"), "{text}");
assert!(text.contains("before input"), "{text}");
assert!(text.contains("Field contents were not verified"), "{text}");
}
}
#[test]
fn found_element_is_described() {
let element = FocusedElementSummary {
role: "text".to_string(),
name: None,
editable: true,
states: vec!["focused".to_string()],
is_terminal: false,
};
let text = focus_probe_feedback(&FocusProbe::Found(element), true, false);
assert!(text.contains("text"), "{text}");
assert!(!text.starts_with("WARNING"), "{text}");
}
}
#[cfg(test)]
mod action_recovery_tests {
use super::*;
use std::sync::atomic::{AtomicUsize, Ordering};
const OLD_REF: &str = ":1.10/org/a11y/atspi/accessible/7";
const NEW_REF: &str = ":1.20/org/a11y/atspi/accessible/7";
const OTHER_REF: &str = ":1.30/org/a11y/atspi/accessible/7";
const EXPLICIT_REF: &str = ":1.99/org/a11y/atspi/accessible/9";
fn action_node(index: u32, object_ref: &str) -> AccessibilityNode {
AccessibilityNode {
index,
parent_index: None,
depth: 1,
object_ref: object_ref.to_string(),
role: "push button".to_string(),
name: Some("Save document".to_string()),
description: None,
child_count: 0,
bounds: None,
states: vec!["enabled".to_string(), "showing".to_string()],
actions: vec![AccessibilityAction {
index: 0,
name: "click".to_string(),
description: String::new(),
keybinding: String::new(),
}],
value: None,
text: None,
supports_editable_text: false,
}
}
fn semantic_params() -> ActionParams {
ActionParams {
role: Some("push button".to_string()),
name: Some("save".to_string()),
..Default::default()
}
}
fn recovery_scope(params: serde_json::Value) -> AccessibilityRecoveryScope {
AccessibilityRecoveryScope {
params: serde_json::from_value(params).unwrap(),
target_pid: None,
}
}
fn named_scope() -> AccessibilityRecoveryScope {
recovery_scope(serde_json::json!({
"app_name_or_bundle_identifier": "org.example.Editor"
}))
}
fn recovery_window(window_id: u64, pid: u32) -> WindowInfo {
WindowInfo {
window_id,
title: Some("Editor".to_string()),
app_id: Some("editor".to_string()),
wm_class: None,
pid: Some(pid),
bounds: None,
workspace: None,
focused: false,
hidden: false,
client_type: None,
backend: KWIN_BACKEND.to_string(),
terminal: None,
}
}
fn refreshed_snapshot(
nodes: Vec<AccessibilityNode>,
scoped: bool,
root_pid: Option<u32>,
target_pid: Option<u32>,
) -> RefreshedAccessibilitySnapshot {
RefreshedAccessibilitySnapshot {
snapshot: crate::atspi_tree_impl::AccessibilitySnapshot {
nodes,
scoped,
root_pid,
app_filter_matched: true,
truncated: false,
window_scope: None,
},
target_pid,
window: None,
}
}
fn successful_invocation() -> crate::atspi_tree_impl::ActionInvocation {
crate::atspi_tree_impl::ActionInvocation {
action_index: 0,
action_name: Some("click".to_string()),
ok: true,
}
}
fn dead_name_error() -> anyhow::Error {
anyhow::Error::new(zbus::fdo::Error::ServiceUnknown(
"the cached unique name has no owner".to_string(),
))
.context("failed to open the cached AT-SPI object")
}
#[test]
fn lossy_atspi_dead_name_errors_are_query_phase_recoverable() {
let error = anyhow::Error::new(atspi::AtspiError::from(zbus::fdo::Error::ServiceUnknown(
"dead name".to_string(),
)));
assert!(is_recoverable_action_query_error(&error));
assert!(!is_recoverable_action_query_error(&anyhow::anyhow!(
"org.freedesktop.DBus.Error.UnknownObject"
)));
assert!(!is_recoverable_action_query_error(
&error.context("failed to invoke AT-SPI action 0")
));
}
#[test]
fn index_and_semantic_selector_resolve_the_same_object_ref() {
let backend = ComputerUseLinux::default();
backend.cache_snapshot_with_scope(&[action_node(7, OLD_REF)], None, Some(named_scope()));
let params = semantic_params();
let selector = params.selector();
let by_index = backend
.resolve_object_ref(Some(7), None, &selector, ElementResolvePurpose::Action)
.unwrap();
let by_selector = backend
.resolve_object_ref(None, None, &selector, ElementResolvePurpose::Action)
.unwrap();
assert_eq!(by_index, OLD_REF);
assert_eq!(by_selector, by_index);
}
#[tokio::test]
async fn app_state_snapshots_retain_scope_only_when_scoped() {
let backend = ComputerUseLinux::default();
let params: GetAppStateParams = serde_json::from_value(serde_json::json!({
"app_name_or_bundle_identifier": "org.example.Editor",
"app_id": "org.example.Editor.desktop",
"window_id": 42,
"pid": 1234
}))
.unwrap();
backend
.commit_app_state_snapshot(
&[action_node(0, OLD_REF)],
Some(1234),
None,
¶ms,
true,
TreeShape::default(),
)
.await;
let scope = backend
.last_snapshot
.lock()
.unwrap()
.recovery_scope
.clone()
.expect("scoped app-state snapshots need recovery scope");
assert_eq!(
scope.params.app_name_or_bundle_identifier.as_deref(),
Some("org.example.Editor")
);
assert_eq!(
scope.params.app_id.as_deref(),
Some("org.example.Editor.desktop")
);
assert_eq!(scope.params.window_id, Some(42));
assert_eq!(scope.params.pid, Some(1234));
assert_eq!(scope.target_pid, Some(1234));
backend
.commit_app_state_snapshot(
&[action_node(0, OLD_REF)],
None,
None,
¶ms,
false,
TreeShape::default(),
)
.await;
assert!(backend
.last_snapshot
.lock()
.unwrap()
.recovery_scope
.is_none());
}
#[tokio::test]
async fn semantic_recovery_regression_old_index_cannot_dispatch_a_reordered_element() {
let backend = ComputerUseLinux::default();
backend.cache_snapshot_with_scope(
&[action_node(0, OLD_REF)],
Some(1234),
Some(recovery_scope(serde_json::json!({"pid": 1234}))),
);
let (Json(recovered), _) = backend
.perform_element_action_with(
&semantic_params(),
Some("click"),
|object_ref, _| async move {
if object_ref == OLD_REF {
Err(dead_name_error())
} else {
assert_eq!(object_ref, NEW_REF);
Ok(successful_invocation())
}
},
|_| async {
let mut other = action_node(0, OTHER_REF);
other.name = Some("Delete document".to_string());
Ok(refreshed_snapshot(
vec![other, action_node(1, NEW_REF)],
true,
Some(1234),
Some(1234),
))
},
)
.await;
assert!(recovered.ok, "{}", recovered.message);
let stale_dispatches = Arc::new(Mutex::new(Vec::new()));
let (Json(indexed), _) = backend
.perform_element_action_with(
&ActionParams {
element_index: Some(0),
..Default::default()
},
Some("click"),
{
let stale_dispatches = Arc::clone(&stale_dispatches);
move |object_ref, _| {
stale_dispatches.lock().unwrap().push(object_ref);
async { Ok(successful_invocation()) }
}
},
|_| async { panic!("indexed actions must never refresh") },
)
.await;
assert!(
stale_dispatches.lock().unwrap().is_empty(),
"stale index dispatched to {:?}",
stale_dispatches.lock().unwrap()
);
assert!(!indexed.ok);
assert!(recovered.message.contains("get_app_state"));
assert!(indexed.message.contains("get_app_state"));
}
#[tokio::test]
async fn semantic_recovery_regression_unstable_scope_cannot_rebind_another_app() {
for params in [
serde_json::json!({"app_name_or_bundle_identifier": "Editor"}),
serde_json::json!({"app_id": "org.example.Editor"}),
serde_json::json!({"title": "Editor window"}),
] {
let backend = ComputerUseLinux::default();
backend.cache_snapshot_with_scope(
&[action_node(0, OLD_REF)],
None,
Some(recovery_scope(params)),
);
let refreshes = Arc::new(AtomicUsize::new(0));
let dispatches = Arc::new(AtomicUsize::new(0));
let (Json(output), _) = backend
.perform_element_action_with(
&semantic_params(),
Some("click"),
{
let dispatches = Arc::clone(&dispatches);
move |object_ref, _| {
let dispatches = Arc::clone(&dispatches);
async move {
if object_ref == OLD_REF {
Err(dead_name_error())
} else {
dispatches.fetch_add(1, Ordering::SeqCst);
Ok(successful_invocation())
}
}
}
},
{
let refreshes = Arc::clone(&refreshes);
move |_| async move {
refreshes.fetch_add(1, Ordering::SeqCst);
// A different application's matching button can satisfy
// a name substring without proving its old identity.
Ok(refreshed_snapshot(
vec![action_node(0, OTHER_REF)],
true,
None,
None,
))
}
},
)
.await;
assert_eq!(refreshes.load(Ordering::SeqCst), 0);
assert_eq!(dispatches.load(Ordering::SeqCst), 0);
assert!(!output.ok);
assert!(output.message.contains("get_app_state"));
}
}
#[tokio::test]
async fn semantic_recovery_identity_regression_observed_pid_cannot_roll_over() {
for params in [
serde_json::json!({"title": "Editor"}),
serde_json::json!({"app_id": "org.example.Editor"}),
] {
let backend = ComputerUseLinux::default();
backend
.commit_app_state_snapshot(
&[action_node(0, OLD_REF)],
Some(1234),
Some(&recovery_window(42, 1234)),
&serde_json::from_value(params).unwrap(),
true,
TreeShape::default(),
)
.await;
let dispatches = Arc::new(AtomicUsize::new(0));
let (Json(output), _) = backend
.perform_element_action_with(
&semantic_params(),
Some("click"),
{
let dispatches = Arc::clone(&dispatches);
move |object_ref, _| {
let dispatches = Arc::clone(&dispatches);
async move {
if object_ref == OLD_REF {
Err(dead_name_error())
} else {
dispatches.fetch_add(1, Ordering::SeqCst);
Ok(successful_invocation())
}
}
}
},
|_scope| async move {
let mut refreshed = refreshed_snapshot(
vec![action_node(0, OTHER_REF)],
true,
Some(5678),
Some(5678),
);
refreshed.window = Some(recovery_window(43, 5678));
Ok(refreshed)
},
)
.await;
assert_eq!(dispatches.load(Ordering::SeqCst), 0);
assert!(!output.ok);
assert!(output.message.contains("get_app_state"));
assert_eq!(backend.last_snapshot.lock().unwrap().pid, Some(1234));
}
}
#[tokio::test]
async fn semantic_recovery_identity_regression_unobserved_window_cannot_admit_refresh() {
let backend = ComputerUseLinux::default();
backend
.commit_app_state_snapshot(
&[action_node(0, OLD_REF)],
None,
None,
&serde_json::from_value(serde_json::json!({"window_id": 42})).unwrap(),
true,
TreeShape::default(),
)
.await;
let refreshes = Arc::new(AtomicUsize::new(0));
let (Json(output), _) = backend
.perform_element_action_with(
&semantic_params(),
Some("click"),
|object_ref, _| async move {
if object_ref == OLD_REF {
Err(dead_name_error())
} else {
Ok(successful_invocation())
}
},
{
let refreshes = Arc::clone(&refreshes);
move |_| async move {
refreshes.fetch_add(1, Ordering::SeqCst);
let mut refreshed = refreshed_snapshot(
vec![action_node(0, OTHER_REF)],
true,
Some(5678),
Some(5678),
);
refreshed.window = Some(recovery_window(42, 5678));
Ok(refreshed)
}
},
)
.await;
assert_eq!(refreshes.load(Ordering::SeqCst), 0);
assert!(!output.ok);
assert!(output.message.contains("get_app_state"));
}
#[tokio::test]
async fn window_only_recovery_requires_exact_window_and_matching_root_pid() {
for (fresh_window_id, fresh_backend, fresh_pid, expected_ok) in [
(42, KWIN_BACKEND, Some(5678), true),
(43, KWIN_BACKEND, Some(5678), false),
(42, "x11", Some(5678), false),
(42, KWIN_BACKEND, None, false),
] {
let backend = ComputerUseLinux::default();
let mut original_window = recovery_window(42, 1234);
original_window.pid = None;
backend
.commit_app_state_snapshot(
&[action_node(0, OLD_REF)],
None,
Some(&original_window),
&serde_json::from_value(serde_json::json!({"window_id": 42})).unwrap(),
true,
TreeShape::default(),
)
.await;
let dispatches = Arc::new(AtomicUsize::new(0));
let (Json(output), _) = backend
.perform_element_action_with(
&semantic_params(),
Some("click"),
{
let dispatches = Arc::clone(&dispatches);
move |object_ref, _| {
let dispatches = Arc::clone(&dispatches);
async move {
if object_ref == OLD_REF {
Err(dead_name_error())
} else {
dispatches.fetch_add(1, Ordering::SeqCst);
Ok(successful_invocation())
}
}
}
},
|scope| async move {
let pid = action_recovery_target_pid(&scope, fresh_pid);
assert_eq!(pid, fresh_pid);
// An unrelated name-matching accessibility root still
// looks scoped when neither owner supplies a PID.
let object_ref = if fresh_pid.is_none() {
OTHER_REF
} else {
NEW_REF
};
let mut refreshed = refreshed_snapshot(
vec![action_node(0, object_ref)],
true,
fresh_pid,
pid,
);
refreshed.window = Some(WindowInfo {
backend: fresh_backend.to_string(),
pid: fresh_pid,
..recovery_window(fresh_window_id, 5678)
});
Ok(refreshed)
},
)
.await;
assert_eq!(dispatches.load(Ordering::SeqCst), usize::from(expected_ok));
assert_eq!(output.ok, expected_ok, "{}", output.message);
if fresh_pid.is_none() {
assert!(output.message.contains("could not prove"));
}
}
}
#[tokio::test]
async fn dead_owner_recovers_new_identity_dispatches_once_and_invalidates_snapshot() {
let backend = ComputerUseLinux::default();
let scope = recovery_scope(serde_json::json!({
"app_name_or_bundle_identifier": "org.example.Editor",
"app_id": "org.example.Editor.desktop",
"window_id": 42,
"pid": 1234,
"max_nodes": 50,
"max_depth": 8
}));
backend.cache_snapshot_with_window(
&[action_node(0, OLD_REF)],
Some(1234),
Some(scope),
Some(&recovery_window(42, 1234)),
);
let queries = Arc::new(AtomicUsize::new(0));
let dispatches = Arc::new(AtomicUsize::new(0));
let refreshes = Arc::new(AtomicUsize::new(0));
let invoked_refs = Arc::new(Mutex::new(Vec::new()));
let params = semantic_params();
let (Json(output), _) = backend
.perform_element_action_with(
¶ms,
Some("0"),
{
let queries = Arc::clone(&queries);
let dispatches = Arc::clone(&dispatches);
let invoked_refs = Arc::clone(&invoked_refs);
move |object_ref, _action| {
let queries = Arc::clone(&queries);
let dispatches = Arc::clone(&dispatches);
let invoked_refs = Arc::clone(&invoked_refs);
async move {
invoked_refs.lock().unwrap().push(object_ref);
if queries.fetch_add(1, Ordering::SeqCst) == 0 {
Err(dead_name_error())
} else {
dispatches.fetch_add(1, Ordering::SeqCst);
Ok(successful_invocation())
}
}
}
},
{
let refreshes = Arc::clone(&refreshes);
move |scope| async move {
refreshes.fetch_add(1, Ordering::SeqCst);
assert_eq!(
scope.params.app_name_or_bundle_identifier.as_deref(),
Some("org.example.Editor")
);
assert_eq!(
scope.params.app_id.as_deref(),
Some("org.example.Editor.desktop")
);
assert_eq!(scope.params.window_id, Some(42));
assert_eq!(scope.params.pid, Some(1234));
assert_eq!(scope.params.max_nodes, Some(50));
assert_eq!(scope.params.max_depth, Some(8));
let mut filler = action_node(0, ":1.20/org/a11y/atspi/accessible/1");
filler.depth = 2;
filler.role = "filler".to_string();
filler.name = None;
filler.actions.clear();
let mut replacement = action_node(1, NEW_REF);
replacement.depth = 2;
replacement.parent_index = Some(0);
let mut refreshed = refreshed_snapshot(
vec![filler, replacement],
true,
Some(1234),
Some(1234),
);
refreshed.window = Some(recovery_window(42, 1234));
Ok(refreshed)
}
},
)
.await;
assert!(output.ok, "{}", output.message);
assert_eq!(queries.load(Ordering::SeqCst), 2);
assert_eq!(dispatches.load(Ordering::SeqCst), 1);
assert_eq!(refreshes.load(Ordering::SeqCst), 1);
assert_eq!(*invoked_refs.lock().unwrap(), vec![OLD_REF, NEW_REF]);
let snapshot = backend.last_snapshot.lock().unwrap();
assert!(snapshot.nodes.is_empty());
assert!(snapshot.pid.is_none());
assert!(snapshot.window_id.is_none());
assert!(snapshot.recovery_scope.is_none());
assert!(output.message.contains("Call get_app_state"));
}
#[tokio::test]
async fn pid_scoped_recovery_invalidates_window_binding_after_local_resolution() {
let old_window = WindowInfo {
window_id: 42,
title: Some("Editor".to_string()),
app_id: Some("editor".to_string()),
wm_class: None,
pid: Some(1234),
bounds: None,
workspace: None,
focused: false,
hidden: false,
client_type: None,
backend: KWIN_BACKEND.to_string(),
terminal: None,
};
for fresh_window_id in [Some(84), None] {
let backend = ComputerUseLinux::default();
backend
.commit_app_state_snapshot(
&[action_node(0, OLD_REF)],
Some(1234),
Some(&old_window),
&serde_json::from_value(serde_json::json!({
"app_name_or_bundle_identifier": "Editor"
}))
.unwrap(),
true,
TreeShape::default(),
)
.await;
let fresh_window = fresh_window_id.map(|window_id| WindowInfo {
window_id,
..old_window.clone()
});
let (Json(output), _) = backend
.perform_element_action_with(
&semantic_params(),
Some("click"),
|object_ref, _action| async move {
if object_ref == OLD_REF {
Err(dead_name_error())
} else {
Ok(successful_invocation())
}
},
|scope| async move {
assert_eq!(scope.params.pid, None);
assert_eq!(scope.target_pid, Some(1234));
let mut refreshed = refreshed_snapshot(
vec![action_node(0, NEW_REF)],
true,
Some(1234),
Some(1234),
);
refreshed.window = fresh_window;
Ok(refreshed)
},
)
.await;
assert!(output.ok, "{}", output.message);
let cached = backend.last_snapshot.lock().unwrap();
assert!(cached.nodes.is_empty());
assert!(cached.pid.is_none());
assert!(cached.window_id.is_none());
assert!(cached.window_backend.is_none());
assert!(cached.recovery_scope.is_none());
assert!(output.message.contains("Call get_app_state"));
}
}
#[tokio::test]
async fn explicit_window_recovery_rejects_missing_or_different_window_without_replacing_cache()
{
for fresh_window_id in [None, Some(43)] {
let backend = ComputerUseLinux::default();
let scope = recovery_scope(serde_json::json!({
"app_name_or_bundle_identifier": "Editor",
"window_id": 42
}));
backend.cache_snapshot_with_window(
&[action_node(0, OLD_REF)],
Some(1234),
Some(scope),
Some(&recovery_window(42, 1234)),
);
let queries = Arc::new(AtomicUsize::new(0));
let (Json(output), _) = backend
.perform_element_action_with(
&semantic_params(),
Some("click"),
{
let queries = Arc::clone(&queries);
move |object_ref, _action| {
let queries = Arc::clone(&queries);
async move {
queries.fetch_add(1, Ordering::SeqCst);
if object_ref == OLD_REF {
Err(dead_name_error())
} else {
Ok(successful_invocation())
}
}
}
},
|_scope| async move {
let mut refreshed = refreshed_snapshot(
vec![action_node(0, NEW_REF)],
true,
Some(1234),
Some(1234),
);
refreshed.window = fresh_window_id.map(|id| recovery_window(id, 1234));
Ok(refreshed)
},
)
.await;
assert!(!output.ok, "{fresh_window_id:?}: {}", output.message);
assert_eq!(queries.load(Ordering::SeqCst), 1);
let cached = backend.last_snapshot.lock().unwrap();
assert_eq!(cached.nodes[0].object_ref, OLD_REF);
assert_eq!(cached.window_id, Some(42));
assert_eq!(cached.window_backend.as_deref(), Some(KWIN_BACKEND));
}
}
#[tokio::test]
async fn window_derived_pid_scope_rejects_name_only_refresh_after_window_loss() {
let backend = ComputerUseLinux::default();
let mut scope = recovery_scope(serde_json::json!({
"app_name_or_bundle_identifier": "Editor",
"window_id": 42
}));
scope.target_pid = Some(1234);
assert_eq!(scope.params.pid, None);
assert_eq!(action_recovery_target_pid(&scope, None), Some(1234));
assert_eq!(action_recovery_target_pid(&scope, Some(5678)), Some(1234));
backend.cache_snapshot_with_scope(&[action_node(0, OLD_REF)], Some(1234), Some(scope));
let queries = Arc::new(AtomicUsize::new(0));
let dispatches = Arc::new(AtomicUsize::new(0));
let params = semantic_params();
let (Json(output), _) = backend
.perform_element_action_with(
¶ms,
Some("0"),
{
let queries = Arc::clone(&queries);
let dispatches = Arc::clone(&dispatches);
move |_object_ref, _action| {
let queries = Arc::clone(&queries);
let dispatches = Arc::clone(&dispatches);
async move {
if queries.fetch_add(1, Ordering::SeqCst) == 0 {
Err(dead_name_error())
} else {
dispatches.fetch_add(1, Ordering::SeqCst);
Ok(successful_invocation())
}
}
}
},
|scope| async move {
let target_pid = action_recovery_target_pid(&scope, None);
Ok(refreshed_snapshot(
vec![action_node(0, OTHER_REF)],
true,
None,
target_pid,
))
},
)
.await;
assert!(!output.ok);
assert!(output.message.contains("original app scope"));
assert_eq!(queries.load(Ordering::SeqCst), 1);
assert_eq!(dispatches.load(Ordering::SeqCst), 0);
let snapshot = backend.last_snapshot.lock().unwrap();
assert_eq!(snapshot.pid, Some(1234));
assert_eq!(snapshot.nodes[0].object_ref, OLD_REF);
}
#[tokio::test]
async fn pid_scope_rejects_identical_selector_from_another_app() {
let backend = ComputerUseLinux::default();
backend.cache_snapshot_with_scope(
&[action_node(0, OLD_REF)],
Some(1234),
Some(recovery_scope(serde_json::json!({
"app_name_or_bundle_identifier": "Editor",
"pid": 1234
}))),
);
let invoked_refs = Arc::new(Mutex::new(Vec::new()));
let params = semantic_params();
let (Json(output), _) = backend
.perform_element_action_with(
¶ms,
Some("0"),
{
let invoked_refs = Arc::clone(&invoked_refs);
move |object_ref, _action| {
let invoked_refs = Arc::clone(&invoked_refs);
async move {
invoked_refs.lock().unwrap().push(object_ref);
Err(dead_name_error())
}
}
},
|_scope| async {
Ok(refreshed_snapshot(
vec![action_node(0, OTHER_REF)],
true,
Some(5678),
Some(1234),
))
},
)
.await;
assert!(!output.ok);
assert!(
output.message.contains("original app scope"),
"{}",
output.message
);
assert!(output.message.contains("Call get_app_state"));
assert_eq!(*invoked_refs.lock().unwrap(), vec![OLD_REF]);
let snapshot = backend.last_snapshot.lock().unwrap();
assert_eq!(snapshot.pid, Some(1234));
assert_eq!(snapshot.nodes[0].object_ref, OLD_REF);
}
#[tokio::test]
async fn recovery_rejects_reused_pid_when_original_filter_does_not_match() {
let backend = ComputerUseLinux::default();
let mut scope = recovery_scope(serde_json::json!({
"app_name_or_bundle_identifier": "Editor",
"pid": 1234
}));
scope.target_pid = Some(1234);
backend.cache_snapshot_with_scope(&[action_node(0, OLD_REF)], Some(1234), Some(scope));
let invoked_refs = Arc::new(Mutex::new(Vec::new()));
let params = semantic_params();
let (Json(output), _) = backend
.perform_element_action_with(
¶ms,
Some("0"),
{
let invoked_refs = Arc::clone(&invoked_refs);
move |object_ref, _action| {
let invoked_refs = Arc::clone(&invoked_refs);
async move {
invoked_refs.lock().unwrap().push(object_ref);
Err(dead_name_error())
}
}
},
|_scope| async {
let mut refreshed = refreshed_snapshot(
vec![action_node(0, OTHER_REF)],
true,
Some(1234),
Some(1234),
);
refreshed.snapshot.app_filter_matched = false;
Ok(refreshed)
},
)
.await;
assert!(!output.ok);
assert!(output.message.contains("original app scope"));
assert_eq!(*invoked_refs.lock().unwrap(), vec![OLD_REF]);
let snapshot = backend.last_snapshot.lock().unwrap();
assert_eq!(snapshot.pid, Some(1234));
assert_eq!(snapshot.nodes[0].object_ref, OLD_REF);
assert_eq!(
snapshot.recovery_scope.as_ref().unwrap().target_pid,
Some(1234)
);
}
#[tokio::test]
async fn window_scoped_recovery_requires_observation_before_a_second_recovery() {
let backend = ComputerUseLinux::default();
let mut scope = recovery_scope(serde_json::json!({
"app_name_or_bundle_identifier": "Editor",
"window_id": 42,
"pid": 8765
}));
scope.target_pid = Some(1234);
backend.cache_snapshot_with_window(
&[action_node(0, OLD_REF)],
Some(1234),
Some(scope),
Some(&recovery_window(42, 1234)),
);
let params = semantic_params();
let first_refs = Arc::new(Mutex::new(Vec::new()));
let (Json(output), _) = backend
.perform_element_action_with(
¶ms,
Some("0"),
{
let first_refs = Arc::clone(&first_refs);
move |object_ref, _action| {
let first_refs = Arc::clone(&first_refs);
async move {
first_refs.lock().unwrap().push(object_ref.clone());
if object_ref == OLD_REF {
Err(dead_name_error())
} else {
Ok(successful_invocation())
}
}
}
},
|scope| async move {
assert_eq!(action_recovery_target_pid(&scope, None), Some(1234));
let mut refreshed = refreshed_snapshot(
vec![action_node(0, NEW_REF)],
true,
Some(1234),
Some(1234),
);
refreshed.window = Some(recovery_window(42, 1234));
Ok(refreshed)
},
)
.await;
assert!(output.ok, "{}", output.message);
assert_eq!(*first_refs.lock().unwrap(), vec![OLD_REF, NEW_REF]);
{
let snapshot = backend.last_snapshot.lock().unwrap();
assert!(snapshot.nodes.is_empty());
assert!(snapshot.pid.is_none());
assert!(snapshot.recovery_scope.is_none());
}
let (Json(without_observation), _) = backend
.perform_element_action_with(
¶ms,
Some("click"),
|_, _| async { panic!("invalidated indices must not dispatch") },
|_| async { panic!("invalidated snapshots must not refresh") },
)
.await;
assert!(!without_observation.ok);
assert!(without_observation.message.contains("get_app_state"));
// Simulate the next explicit get_app_state. Its observed PID and exact
// window binding, rather than the hidden recovery tree, own this cache.
backend
.commit_app_state_snapshot(
&[action_node(0, NEW_REF)],
Some(5678),
Some(&recovery_window(42, 5678)),
&serde_json::from_value(serde_json::json!({"window_id": 42})).unwrap(),
true,
TreeShape::default(),
)
.await;
let second_refs = Arc::new(Mutex::new(Vec::new()));
let (Json(output), _) = backend
.perform_element_action_with(
¶ms,
Some("0"),
{
let second_refs = Arc::clone(&second_refs);
move |object_ref, _action| {
let second_refs = Arc::clone(&second_refs);
async move {
second_refs.lock().unwrap().push(object_ref.clone());
if object_ref == NEW_REF {
Err(dead_name_error())
} else {
Ok(successful_invocation())
}
}
}
},
|scope| async move {
assert_eq!(action_recovery_target_pid(&scope, None), Some(5678));
let mut refreshed = refreshed_snapshot(
vec![action_node(0, OTHER_REF)],
true,
Some(5678),
Some(5678),
);
refreshed.window = Some(recovery_window(42, 5678));
Ok(refreshed)
},
)
.await;
assert!(output.ok, "{}", output.message);
assert_eq!(*second_refs.lock().unwrap(), vec![NEW_REF, OTHER_REF]);
let snapshot = backend.last_snapshot.lock().unwrap();
assert!(snapshot.nodes.is_empty());
assert!(snapshot.pid.is_none());
assert!(snapshot.recovery_scope.is_none());
}
#[tokio::test]
async fn unscoped_refresh_is_not_cached_or_invoked() {
let backend = ComputerUseLinux::default();
backend.cache_snapshot_with_scope(
&[action_node(0, OLD_REF)],
Some(1234),
Some(recovery_scope(serde_json::json!({"pid": 1234}))),
);
let queries = Arc::new(AtomicUsize::new(0));
let params = semantic_params();
let (Json(output), _) = backend
.perform_element_action_with(
¶ms,
Some("0"),
{
let queries = Arc::clone(&queries);
move |_object_ref, _action| {
let queries = Arc::clone(&queries);
async move {
queries.fetch_add(1, Ordering::SeqCst);
Err(dead_name_error())
}
}
},
|_scope| async {
Ok(refreshed_snapshot(
vec![action_node(0, OTHER_REF)],
false,
None,
None,
))
},
)
.await;
assert!(!output.ok);
assert!(output.message.contains("could not prove"));
assert_eq!(queries.load(Ordering::SeqCst), 1);
assert_eq!(
backend.last_snapshot.lock().unwrap().nodes[0].object_ref,
OLD_REF
);
}
#[tokio::test]
async fn numeric_and_explicit_identifiers_never_refresh_or_get_rewritten() {
let numeric = ActionParams {
element_index: Some(7),
..semantic_params()
};
let explicit = ActionParams {
element_identifier: Some(EXPLICIT_REF.to_string()),
..semantic_params()
};
for (params, expected_ref) in [(numeric, OLD_REF), (explicit, EXPLICIT_REF)] {
let backend = ComputerUseLinux::default();
backend.cache_snapshot_with_scope(
&[action_node(7, OLD_REF)],
None,
Some(named_scope()),
);
let refreshes = Arc::new(AtomicUsize::new(0));
let invoked_refs = Arc::new(Mutex::new(Vec::new()));
let (Json(output), _) = backend
.perform_element_action_with(
¶ms,
Some("0"),
{
let invoked_refs = Arc::clone(&invoked_refs);
move |object_ref, _action| {
let invoked_refs = Arc::clone(&invoked_refs);
async move {
invoked_refs.lock().unwrap().push(object_ref);
Err(dead_name_error())
}
}
},
{
let refreshes = Arc::clone(&refreshes);
move |_scope| async move {
refreshes.fetch_add(1, Ordering::SeqCst);
Ok(refreshed_snapshot(
vec![action_node(0, NEW_REF)],
true,
None,
None,
))
}
},
)
.await;
assert!(!output.ok);
assert_eq!(*invoked_refs.lock().unwrap(), vec![expected_ref]);
assert_eq!(refreshes.load(Ordering::SeqCst), 0);
assert_eq!(
backend.last_snapshot.lock().unwrap().nodes[0].object_ref,
OLD_REF
);
}
}
#[tokio::test]
async fn invocation_phase_dead_name_error_is_never_retried() {
let backend = ComputerUseLinux::default();
backend.cache_snapshot_with_scope(&[action_node(0, OLD_REF)], None, Some(named_scope()));
let queries = Arc::new(AtomicUsize::new(0));
let refreshes = Arc::new(AtomicUsize::new(0));
let params = semantic_params();
let (Json(output), _) = backend
.perform_element_action_with(
¶ms,
Some("0"),
{
let queries = Arc::clone(&queries);
move |_object_ref, _action| {
let queries = Arc::clone(&queries);
async move {
queries.fetch_add(1, Ordering::SeqCst);
Err(dead_name_error().context("failed to invoke AT-SPI action 0"))
}
}
},
{
let refreshes = Arc::clone(&refreshes);
move |_scope| async move {
refreshes.fetch_add(1, Ordering::SeqCst);
Ok(refreshed_snapshot(
vec![action_node(0, NEW_REF)],
true,
None,
None,
))
}
},
)
.await;
assert!(!output.ok);
assert!(output.message.contains("failed to invoke AT-SPI action 0"));
assert_eq!(queries.load(Ordering::SeqCst), 1);
assert_eq!(refreshes.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn recovered_dispatch_failure_is_not_retried() {
let backend = ComputerUseLinux::default();
backend.cache_snapshot_with_scope(
&[action_node(0, OLD_REF)],
Some(1234),
Some(recovery_scope(serde_json::json!({"pid": 1234}))),
);
let queries = Arc::new(AtomicUsize::new(0));
let refreshes = Arc::new(AtomicUsize::new(0));
let params = semantic_params();
let (Json(output), _) = backend
.perform_element_action_with(
¶ms,
Some("0"),
{
let queries = Arc::clone(&queries);
move |_object_ref, _action| {
let queries = Arc::clone(&queries);
async move {
if queries.fetch_add(1, Ordering::SeqCst) == 0 {
Err(dead_name_error())
} else {
Err(dead_name_error().context("failed to invoke AT-SPI action 0"))
}
}
}
},
{
let refreshes = Arc::clone(&refreshes);
move |_scope| async move {
refreshes.fetch_add(1, Ordering::SeqCst);
Ok(refreshed_snapshot(
vec![action_node(0, NEW_REF)],
true,
Some(1234),
Some(1234),
))
}
},
)
.await;
assert!(!output.ok);
assert!(output.message.contains("failed to invoke AT-SPI action 0"));
assert_eq!(queries.load(Ordering::SeqCst), 2);
assert_eq!(refreshes.load(Ordering::SeqCst), 1);
assert!(backend.last_snapshot.lock().unwrap().nodes.is_empty());
assert!(output.message.contains("Call get_app_state"));
}
#[tokio::test]
async fn dead_name_without_stable_scope_requires_fresh_app_state() {
let backend = ComputerUseLinux::default();
backend.cache_nodes(&[action_node(0, OLD_REF)]);
let refreshes = Arc::new(AtomicUsize::new(0));
let params = semantic_params();
let (Json(output), _) = backend
.perform_element_action_with(
¶ms,
Some("0"),
|_object_ref, _action| async { Err(dead_name_error()) },
{
let refreshes = Arc::clone(&refreshes);
move |_scope| async move {
refreshes.fetch_add(1, Ordering::SeqCst);
Ok(refreshed_snapshot(Vec::new(), true, None, None))
}
},
)
.await;
assert!(!output.ok);
assert!(output.message.contains("no stable scoped recovery target"));
assert!(output.message.contains("Call get_app_state"));
assert_eq!(refreshes.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn post_action_snapshot_does_not_retain_stale_recovery_scope() {
let backend = ComputerUseLinux::default();
backend.cache_snapshot_with_scope(
&[action_node(0, OLD_REF)],
Some(1234),
Some(named_scope()),
);
backend
.commit_snapshot(&[action_node(0, NEW_REF)], Some(5678))
.await;
let snapshot = backend.last_snapshot.lock().unwrap();
assert_eq!(snapshot.nodes[0].object_ref, NEW_REF);
assert_eq!(snapshot.pid, Some(5678));
assert!(snapshot.recovery_scope.is_none());
}
}
#[cfg(test)]
mod node_target_scope_tests {
use super::*;
fn cached_node(index: u32, object_ref: &str) -> AccessibilityNode {
AccessibilityNode {
index,
parent_index: None,
depth: 1,
object_ref: object_ref.to_string(),
role: "push button".to_string(),
name: Some("Hit me".to_string()),
description: None,
child_count: 0,
bounds: None,
states: vec!["enabled".to_string()],
actions: Vec::new(),
value: None,
text: None,
supports_editable_text: false,
}
}
#[tokio::test]
async fn index_from_another_apps_snapshot_is_rejected() {
let backend = ComputerUseLinux::default();
let node = cached_node(2, ":1.9/org/a11y/atspi/accessible/42");
backend.cache_snapshot(std::slice::from_ref(&node), Some(2_690_687));
let error = backend
.check_node_target(&node, &ElementOwnerTarget::pid(Some(2_690_664)))
.await
.unwrap_err();
assert!(error.contains("pid 2690687"), "{error}");
assert!(error.contains("target pid 2690664"), "{error}");
assert!(
error.contains("Call get_app_state for the target app"),
"{error}"
);
}
#[tokio::test]
async fn same_target_or_no_target_is_allowed() {
let backend = ComputerUseLinux::default();
let node = cached_node(2, ":1.9/org/a11y/atspi/accessible/42");
backend.cache_snapshot(std::slice::from_ref(&node), Some(2_690_664));
assert!(backend
.check_node_target(&node, &ElementOwnerTarget::pid(Some(2_690_664)))
.await
.is_ok());
assert!(backend
.check_node_target(&node, &ElementOwnerTarget::pid(None))
.await
.is_ok());
}
#[tokio::test]
async fn unscoped_snapshot_with_unknown_owner_is_allowed() {
// No snapshot pid, and the owner lookup cannot resolve this ref here,
// so a mismatch cannot be shown and the action proceeds.
let backend = ComputerUseLinux::default();
let node = cached_node(2, ":1.9999/org/a11y/atspi/accessible/42");
backend.cache_snapshot(std::slice::from_ref(&node), None);
assert!(backend
.check_node_target(&node, &ElementOwnerTarget::pid(Some(1)))
.await
.is_ok());
}
#[tokio::test]
async fn concurrent_snapshot_commit_waits_until_checked_action_finishes() {
let backend = ComputerUseLinux::default();
let mut original = cached_node(2, ":1.9/org/a11y/atspi/accessible/42");
original.bounds = Some(Bounds {
x: 10,
y: 20,
width: 100,
height: 40,
});
original.actions = vec![AccessibilityAction {
index: 0,
name: "click".into(),
description: String::new(),
keybinding: String::new(),
}];
backend
.commit_snapshot(std::slice::from_ref(&original), Some(10))
.await;
// Model the click/scroll operation: it holds this same lock while its
// target ownership check awaits and until the action has been resolved.
let input_guard = Arc::clone(&backend.input_operation_lock).lock_owned().await;
let checked_node = backend
.cached_node_for(
Some(2),
&ElementSelector::default(),
ElementResolvePurpose::Click,
)
.unwrap();
let barrier = Arc::new(tokio::sync::Barrier::new(2));
let writer_backend = backend.clone();
let writer_barrier = Arc::clone(&barrier);
let writer = tokio::spawn(async move {
let replacement = cached_node(2, ":1.20/org/a11y/atspi/accessible/99");
writer_barrier.wait().await;
writer_backend
.commit_snapshot(&[replacement], Some(20))
.await;
});
barrier.wait().await;
tokio::task::yield_now().await;
assert!(
!writer.is_finished(),
"snapshot publication must wait for input"
);
assert!(backend
.check_node_target(&checked_node, &ElementOwnerTarget::pid(Some(10)))
.await
.is_ok());
let params = ClickParams {
element_index: Some(2),
..Default::default()
};
assert!(matches!(backend.resolve_click_target(¶ms).unwrap(),
ClickTarget::PrimaryAction { object_ref, .. } if object_ref == original.object_ref));
assert_eq!(
backend
.resolve_optional_target_point(None, None, Some(2))
.unwrap(),
Some((60, 40))
);
// perform_action/set_value resolve the same snapshot while holding the
// input lock; their explicit object identifiers remain independent.
for purpose in [
ElementResolvePurpose::Action,
ElementResolvePurpose::SetValue,
] {
assert_eq!(
backend
.resolve_object_ref(Some(2), None, &ElementSelector::default(), purpose)
.unwrap(),
original.object_ref
);
}
drop(input_guard);
writer.await.unwrap();
let snapshot = backend.last_snapshot.lock().unwrap();
assert_eq!(snapshot.pid, Some(20));
assert_eq!(
snapshot.nodes[0].object_ref,
":1.20/org/a11y/atspi/accessible/99"
);
}
#[test]
fn a_new_snapshot_replaces_the_recorded_pid() {
let backend = ComputerUseLinux::default();
backend.cache_snapshot(&[], Some(10));
backend.cache_snapshot(&[], None);
assert_eq!(backend.last_snapshot.lock().unwrap().pid, None);
backend.cache_snapshot(&[], Some(11));
backend.clear_cached_nodes();
assert_eq!(backend.last_snapshot.lock().unwrap().pid, None);
}
const APP_A: u32 = 2_690_664;
const APP_B: u32 = 2_690_687;
/// get_app_state of app A, then of app B: the cache holds B's nodes.
fn backend_after_snapshots_of_a_then_b() -> ComputerUseLinux {
let backend = ComputerUseLinux::default();
backend.cache_snapshot(
&[cached_node(2, ":1.7/org/a11y/atspi/accessible/7")],
Some(APP_A),
);
backend.cache_snapshot(
&[cached_node(2, ":1.9/org/a11y/atspi/accessible/42")],
Some(APP_B),
);
backend
}
fn pid_target(pid: u32) -> ElementActionTarget {
ElementActionTarget {
pid: Some(pid),
..Default::default()
}
}
async fn perform_action_dispatches(
backend: &ComputerUseLinux,
params: ActionParams,
) -> (ActionOutput, Vec<String>) {
let dispatched = Arc::new(Mutex::new(Vec::new()));
let (Json(output), _) = backend
.perform_element_action_with(
¶ms,
Some("click"),
{
let dispatched = Arc::clone(&dispatched);
move |object_ref, _| {
dispatched.lock().unwrap().push(object_ref);
async {
Ok(crate::atspi_tree_impl::ActionInvocation {
action_index: 0,
action_name: Some("click".to_string()),
ok: true,
})
}
}
},
|_| async { panic!("an indexed action must not refresh") },
)
.await;
let dispatched = dispatched.lock().unwrap().clone();
(output, dispatched)
}
#[tokio::test]
async fn perform_action_refuses_an_index_from_another_apps_snapshot() {
let backend = backend_after_snapshots_of_a_then_b();
let (output, dispatched) = perform_action_dispatches(
&backend,
ActionParams {
element_index: Some(2),
target: pid_target(APP_A),
..Default::default()
},
)
.await;
assert!(!output.ok);
assert!(dispatched.is_empty(), "dispatched to {dispatched:?}");
assert_eq!(
output.message,
node_target_mismatch_message(2, APP_B, APP_A),
"perform_action must refuse with the wording click uses"
);
}
#[tokio::test]
async fn perform_action_refuses_a_selector_from_another_apps_snapshot() {
let backend = backend_after_snapshots_of_a_then_b();
let (output, dispatched) = perform_action_dispatches(
&backend,
ActionParams {
role: Some("push button".to_string()),
name: Some("Hit me".to_string()),
target: pid_target(APP_A),
..Default::default()
},
)
.await;
assert!(!output.ok);
assert!(dispatched.is_empty(), "dispatched to {dispatched:?}");
assert!(
output.message.contains("target pid 2690664"),
"{}",
output.message
);
}
#[tokio::test]
async fn perform_action_on_the_snapshot_target_or_without_one_dispatches() {
let backend = backend_after_snapshots_of_a_then_b();
let b_ref = ":1.9/org/a11y/atspi/accessible/42".to_string();
for target in [pid_target(APP_B), ElementActionTarget::default()] {
let (output, dispatched) = perform_action_dispatches(
&backend,
ActionParams {
element_index: Some(2),
target,
..Default::default()
},
)
.await;
assert!(output.ok, "{}", output.message);
assert_eq!(dispatched, vec![b_ref.clone()]);
}
}
#[tokio::test]
async fn element_observation_stays_with_the_dispatched_object_owner() {
let backend = ComputerUseLinux::default();
let object_ref = ":1.9/org/a11y/atspi/accessible/42";
let mut node = cached_node(2, object_ref);
node.parent_index = Some(1);
node.depth = 2;
let mut frame = cached_node(1, ":1.9/window_20");
frame.role = "frame".to_string();
frame.name = Some("Window 20".to_string());
frame.child_count = 1;
let mut window = pidless_window(20);
window.pid = Some(APP_B);
backend.store_snapshot(
&[frame, node],
Some(APP_B),
None,
Some(&window),
TreeShape {
detail: NodeDetail::Compact,
window_scope: Some("Window 20".to_string()),
},
);
let target = backend
.element_observation_target_with(
&ElementActionTarget::default(),
object_ref,
None,
|| async { panic!("the observed element already has an owner") },
)
.await
.unwrap();
assert_eq!(target.pid, Some(APP_B));
assert_eq!(target.window_id, Some(20));
backend.cache_snapshot(&[cached_node(3, ":1.22/chrome")], Some(APP_A));
let target = backend
.element_observation_target_with(
&ElementActionTarget::default(),
object_ref,
None,
|| async { Ok(Some(APP_B)) },
)
.await
.unwrap();
assert_eq!(target.pid, Some(APP_B));
assert_eq!(target.window_id, None);
let explicit = backend
.element_observation_target_with(&pid_target(APP_B), object_ref, None, || async {
panic!("the explicitly checked target must be preserved")
})
.await
.unwrap();
assert_eq!(explicit.pid, Some(APP_B));
}
#[tokio::test]
async fn element_observation_preserves_a_verified_pidless_window() {
let backend = ComputerUseLinux::default();
let object_ref = ":1.9/org/a11y/atspi/accessible/42";
let mut node = cached_node(2, object_ref);
node.parent_index = Some(1);
node.depth = 2;
let mut frame = cached_node(1, ":1.9/window_20");
frame.role = "frame".to_string();
frame.name = Some("Window 20".to_string());
frame.child_count = 1;
backend.store_snapshot(
&[frame, node],
None,
None,
Some(&pidless_window(20)),
TreeShape {
detail: NodeDetail::Compact,
window_scope: Some("Window 20".to_string()),
},
);
let target = backend
.element_observation_target_with(
&ElementActionTarget::default(),
object_ref,
None,
|| async { Ok(Some(APP_B)) },
)
.await
.unwrap();
assert_eq!(target.pid, Some(APP_B));
assert_eq!(target.window_id, Some(20));
let explicit = ElementActionTarget {
window_id: Some(20),
..Default::default()
};
let target = backend
.element_observation_target_with(&explicit, object_ref, None, || async {
Ok(Some(APP_B))
})
.await
.unwrap();
assert_eq!(target.pid, Some(APP_B));
assert_eq!(target.window_id, Some(20));
}
#[tokio::test]
async fn element_observation_does_not_keep_a_pid_overridden_by_window_id() {
let object_ref = ":1.9/org/a11y/atspi/accessible/42";
let mut window_a = pidless_window(10);
window_a.pid = Some(APP_A);
let mut window_b = pidless_window(20);
window_b.pid = Some(APP_B);
let requested = ElementActionTarget {
window_id: Some(window_a.window_id),
pid: Some(APP_B),
..Default::default()
};
let requested_window = requested.window_target().unwrap();
assert_eq!(
resolve_window_target(&[window_a.clone(), window_b.clone()], &requested_window)
.unwrap()
.pid,
Some(APP_A)
);
for cached_pid in [None, Some(APP_A)] {
let backend = ComputerUseLinux::default();
backend.cache_snapshot(&[cached_node(2, object_ref)], cached_pid);
let target = backend
.element_observation_target_with(&requested, object_ref, None, || async {
assert!(cached_pid.is_none(), "reuse the observed object owner");
Ok(Some(APP_A))
})
.await
.unwrap();
assert_eq!(target.window_id, Some(window_a.window_id));
// Once the action closes A's window, the tree falls back to this
// pid because the window can no longer be resolved.
assert!(resolve_window_target(std::slice::from_ref(&window_b), &target).is_err());
assert_eq!(target.pid, Some(APP_A));
}
}
#[tokio::test]
async fn app_wide_element_observation_does_not_inherit_a_sibling_window() {
let backend = ComputerUseLinux::default();
let object_ref = ":1.9/org/a11y/atspi/accessible/window_20_button";
let mut window_10 = pidless_window(10);
window_10.pid = Some(APP_B);
let mut window_20 = pidless_window(20);
window_20.pid = Some(APP_B);
backend.cache_snapshot_with_window(
&[cached_node(2, object_ref)],
Some(APP_B),
None,
Some(&window_10),
);
let target = backend
.element_observation_target_with(
&ElementActionTarget::default(),
object_ref,
None,
|| async { panic!("the app-wide snapshot establishes the process owner") },
)
.await
.unwrap();
assert_eq!(target.pid, Some(APP_B));
assert_eq!(target.window_id, None);
assert!(resolve_window_target(&[window_10, window_20], &target).is_err());
assert_eq!(
post_action_window_title(Some(&target), None, target.pid, None),
None
);
}
#[test]
fn recovery_keeps_the_tree_scope_without_reusing_the_old_element_window() {
let previous = ElementObservationTarget {
owner: WindowTarget {
pid: Some(APP_B),
window_id: Some(20),
..Default::default()
},
tree_window: Some(CachedTreeWindow {
scope: "Window 20".to_string(),
pid: Some(APP_B),
window: Some((20, KWIN_BACKEND.to_string())),
}),
};
for pid in [APP_B, APP_A] {
let mut recovered = ElementObservationTarget {
owner: WindowTarget {
pid: Some(pid),
..Default::default()
},
..Default::default()
};
recovered.inherit_tree_window(&previous);
assert_eq!(recovered.owner.window_id, None);
assert_eq!(recovered.tree_window.is_some(), pid == APP_B);
if let Some(tree) = recovered.tree_window {
assert_eq!(tree.scope, "Window 20");
assert_eq!(tree.window, Some((20, KWIN_BACKEND.to_string())));
}
}
}
#[tokio::test]
async fn element_observation_does_not_bind_retained_dialogs_to_the_narrowed_window() {
let main_ref = ":1.9/main_button";
let dialog_ref = ":1.10/dialog_button";
for separate_root in [false, true] {
let backend = ComputerUseLinux::default();
let mut root = cached_node(0, ":1.9/root");
root.role = "application".to_string();
root.depth = 0;
let mut frame = cached_node(1, ":1.9/window_10");
frame.role = "frame".to_string();
frame.name = Some("Window 10".to_string());
frame.parent_index = Some(0);
let mut main_button = cached_node(2, main_ref);
main_button.depth = 2;
main_button.parent_index = Some(1);
let mut dialog_root = cached_node(3, ":1.10/root");
dialog_root.role = "application".to_string();
dialog_root.depth = 0;
let mut dialog = cached_node(4, ":1.10/dialog");
dialog.role = "file chooser".to_string();
dialog.name = Some("File dialog".to_string());
dialog.parent_index = Some(if separate_root { 3 } else { 0 });
let mut dialog_button = cached_node(5, dialog_ref);
dialog_button.depth = 2;
dialog_button.parent_index = Some(4);
let mut nodes = vec![root, frame, main_button];
if separate_root {
nodes.push(dialog_root);
}
nodes.extend([dialog, dialog_button]);
let mut window = pidless_window(10);
window.pid = Some(APP_B);
backend.store_snapshot(
&nodes,
Some(APP_B),
None,
Some(&window),
TreeShape {
detail: NodeDetail::Compact,
window_scope: Some("Window 10".to_string()),
},
);
for (requested, object_ref, expected_window) in [
(ElementActionTarget::default(), main_ref, Some(10)),
(ElementActionTarget::default(), dialog_ref, None),
(pid_target(APP_B), dialog_ref, None),
] {
let target = backend
.element_observation_target(&requested, object_ref, None)
.await
.unwrap();
assert_eq!(target.owner.pid, Some(APP_B));
assert_eq!(target.owner.window_id, expected_window);
let tree_window = target.tree_window.unwrap();
assert_eq!(tree_window.scope, "Window 10");
assert_eq!(tree_window.pid, Some(APP_B));
assert_eq!(tree_window.window, Some((10, window.backend.clone())));
}
}
}
#[tokio::test]
async fn element_observation_without_an_owner_reports_failure_without_foreground_fallback() {
let backend = backend_after_snapshots_of_a_then_b();
let error = backend
.element_observation_target_with(
&ElementActionTarget::default(),
":1.99/uncached",
None,
|| async { Ok(None) },
)
.await
.unwrap_err();
let capture: PostActionCaptureParams = serde_json::from_value(serde_json::json!({
"include_tree": true,
}))
.unwrap();
let (observation, screenshot, notes) = backend
.capture_element_observation(&capture, None, Err(&error))
.await;
// No foreground tree: the empty tree fields of any failed read, and
// the replaced snapshot is gone.
assert!(observation.accessibility_tree.unwrap().is_empty());
assert_eq!(observation.accessibility_tree_raw_count, Some(0));
assert_eq!(observation.tree_scoped, Some(false));
assert!(observation.accessibility_error.is_some());
assert!(observation.screenshot_error.is_none());
assert!(screenshot.is_none());
assert!(notes[0].contains("no verified app or window owner"));
assert!(backend.cached_snapshot_id().is_empty());
}
#[tokio::test]
async fn http_owner_binding_reads_only_a_named_snapshot() {
let http = ComputerUseLinux {
require_snapshot_ids: true,
..Default::default()
};
let object_ref = ":1.9/org/a11y/atspi/accessible/42";
http.cache_snapshot(&[cached_node(2, object_ref)], Some(APP_B));
// Another client's snapshot holds the object: its owner comes from
// the bus, not from that client's tree.
for named in [None, Some("other")] {
let owner = http
.element_observation_target_with(
&ElementActionTarget::default(),
object_ref,
named,
|| async { Ok(Some(APP_A)) },
)
.await
.unwrap();
assert_eq!(owner.pid, Some(APP_A));
}
// The call named the snapshot, so its observed owner holds.
let id = http.cached_snapshot_id();
let owner = http
.element_observation_target_with(
&ElementActionTarget::default(),
object_ref,
Some(&id),
|| async { panic!("the named snapshot establishes the owner") },
)
.await
.unwrap();
assert_eq!(owner.pid, Some(APP_B));
}
#[tokio::test]
async fn set_value_refuses_an_index_from_another_apps_snapshot() {
let backend = backend_after_snapshots_of_a_then_b();
let (output, verified, _) = backend
.dispatch_set_value(SetValueParams {
element_index: Some(2),
value: "written for app A".to_string(),
target: pid_target(APP_A),
..Default::default()
})
.await;
assert!(!output.ok);
assert_eq!(verified, None);
assert_eq!(
output.message,
node_target_mismatch_message(2, APP_B, APP_A),
"set_value must refuse with the wording click uses"
);
}
/// An i3 (no xprop) or COSMIC window: the backend reports no pid.
fn pidless_window(window_id: u64) -> WindowInfo {
WindowInfo {
window_id,
title: Some(format!("window {window_id}")),
app_id: None,
wm_class: None,
pid: None,
bounds: None,
workspace: None,
focused: false,
hidden: false,
client_type: None,
backend: "i3".to_string(),
terminal: None,
}
}
fn pidless_owner(window_id: u64) -> ElementOwnerTarget {
let windows = vec![pidless_window(10), pidless_window(20)];
let target = WindowTarget {
window_id: Some(window_id),
..Default::default()
};
element_action_owner_from(Ok(windows), &target).unwrap()
}
#[tokio::test]
async fn pidless_window_target_refuses_another_windows_snapshot() {
// get_app_state of window 20, then an action that targets window 10.
let backend = ComputerUseLinux::default();
let node = cached_node(2, ":1.9/org/a11y/atspi/accessible/42");
backend.cache_snapshot_with_window(
std::slice::from_ref(&node),
None,
None,
Some(&pidless_window(20)),
);
assert_eq!(
backend
.check_node_target(&node, &pidless_owner(10))
.await
.unwrap_err(),
node_window_mismatch_message(2, 20, 10)
);
assert!(backend
.check_node_target(&node, &pidless_owner(20))
.await
.is_ok());
}
#[tokio::test]
async fn pidless_window_target_fails_closed_without_a_snapshot_window() {
let backend = ComputerUseLinux::default();
let node = cached_node(2, ":1.9/org/a11y/atspi/accessible/42");
backend.cache_snapshot(std::slice::from_ref(&node), None);
assert_eq!(
backend
.check_node_target(&node, &pidless_owner(10))
.await
.unwrap_err(),
node_window_unverified_message(2, 10)
);
}
#[test]
fn unlisted_windows_fall_back_to_a_passed_pid_only() {
let with_pid = WindowTarget {
app_id: Some("org.gnome.TextEditor".to_string()),
pid: Some(7),
..Default::default()
};
assert_eq!(
element_action_owner_from(Err(anyhow::anyhow!("no window backend")), &with_pid),
Ok(ElementOwnerTarget::pid(Some(7)))
);
let without_pid = WindowTarget {
app_id: Some("org.gnome.TextEditor".to_string()),
..Default::default()
};
assert!(
element_action_owner_from(Err(anyhow::anyhow!("no window backend")), &without_pid)
.is_err()
);
// A listing that works but does not match still fails closed.
assert!(element_action_owner_from(Ok(vec![pidless_window(20)]), &with_pid).is_err());
}
#[test]
fn unraised_target_window_crop_needs_focus() {
let target = WindowTarget {
window_id: Some(20),
..Default::default()
};
let mut window = pidless_window(20);
assert!(
ensure_post_action_crop_shows_target(TargetRaised::No, Some(&target), &window).is_err()
);
assert!(
ensure_post_action_crop_shows_target(TargetRaised::Yes, Some(&target), &window).is_ok()
);
// No target: the crop is of the focused window anyway.
assert!(ensure_post_action_crop_shows_target(TargetRaised::No, None, &window).is_ok());
window.focused = true;
assert!(
ensure_post_action_crop_shows_target(TargetRaised::No, Some(&target), &window).is_ok()
);
}
#[tokio::test]
async fn same_pid_window_target_refuses_another_windows_snapshot() {
// Chromium: windows 20 and 10 share pid 7. get_app_state of window 20,
// then an action that targets window 10.
let backend = ComputerUseLinux::default();
let node = cached_node(2, ":1.9/org/a11y/atspi/accessible/42");
let mut window_20 = pidless_window(20);
window_20.pid = Some(7);
let mut window_10 = pidless_window(10);
window_10.pid = Some(7);
backend.store_snapshot(
std::slice::from_ref(&node),
Some(7),
None,
Some(&window_20),
TreeShape {
detail: NodeDetail::Compact,
window_scope: Some("Window 20".to_string()),
},
);
assert_eq!(
backend
.check_node_target(&node, &ElementOwnerTarget::window(&window_10, None, true))
.await
.unwrap_err(),
node_window_mismatch_message(2, 20, 10)
);
assert!(backend
.check_node_target(&node, &ElementOwnerTarget::window(&window_20, None, true))
.await
.is_ok());
// A pid-only target cannot tell the windows apart and passes.
assert!(backend
.check_node_target(&node, &ElementOwnerTarget::pid(Some(7)))
.await
.is_ok());
// An app-wide tree holds every window: its nodes pass on the pid.
backend.cache_snapshot_with_window(
std::slice::from_ref(&node),
Some(7),
None,
Some(&window_20),
);
assert!(backend
.check_node_target(&node, &ElementOwnerTarget::window(&window_10, None, true))
.await
.is_ok());
}
#[tokio::test]
async fn same_pid_exact_target_fails_closed_when_the_narrowed_tree_recorded_no_window() {
// A tree narrowed through the cached scope that came from another pid
// or frame records no window (post_action_tree_window). An exact
// target then cannot be checked against it.
let backend = ComputerUseLinux::default();
let node = cached_node(2, ":1.9/org/a11y/atspi/accessible/42");
let mut window_10 = pidless_window(10);
window_10.pid = Some(7);
backend.store_snapshot(
std::slice::from_ref(&node),
Some(7),
None,
None,
TreeShape {
detail: NodeDetail::Compact,
window_scope: Some("Window 20".to_string()),
},
);
assert_eq!(
backend
.check_node_target(&node, &ElementOwnerTarget::window(&window_10, None, true))
.await
.unwrap_err(),
node_window_unrecorded_message(2, 10)
);
// An inexact target and a pid-only target still pass on the pid.
assert!(backend
.check_node_target(&node, &ElementOwnerTarget::window(&window_10, None, false))
.await
.is_ok());
assert!(backend
.check_node_target(&node, &ElementOwnerTarget::pid(Some(7)))
.await
.is_ok());
// A narrowed tree whose window another backend listed (the GNOME
// extension failed and Introspect answered) is not comparable either.
let mut window_10_other_backend = window_10.clone();
window_10_other_backend.backend = "gnome-shell-introspect".to_string();
backend.store_snapshot(
std::slice::from_ref(&node),
Some(7),
None,
Some(&window_10_other_backend),
TreeShape {
detail: NodeDetail::Compact,
window_scope: Some("Window 10".to_string()),
},
);
assert_eq!(
backend
.check_node_target(&node, &ElementOwnerTarget::window(&window_10, None, true))
.await
.unwrap_err(),
node_window_unrecorded_message(2, 10)
);
}
#[tokio::test]
async fn an_inexact_window_target_passes_on_the_pid() {
// Chromium: windows 10 and 20 share pid 7 and app_id. The tree was
// narrowed to window 20; app_id resolves to the first window, 10.
let chromium = |window_id: u64, title: &str| {
let mut window = pidless_window(window_id);
window.pid = Some(7);
window.app_id = Some("chromium".to_string());
window.title = Some(title.to_string());
window
};
let windows = vec![chromium(10, "GitHub"), chromium(20, "PlanetScale")];
let backend = ComputerUseLinux::default();
let node = cached_node(2, ":1.9/org/a11y/atspi/accessible/42");
backend.store_snapshot(
std::slice::from_ref(&node),
Some(7),
None,
Some(&windows[1]),
TreeShape {
detail: NodeDetail::Compact,
window_scope: Some("PlanetScale".to_string()),
},
);
let by_app_id = element_action_owner_from(
Ok(windows.clone()),
&WindowTarget {
app_id: Some("chromium".to_string()),
..Default::default()
},
)
.unwrap();
assert!(!by_app_id.exact_window);
assert!(backend.check_node_target(&node, &by_app_id).await.is_ok());
// A title that names window 10 alone is exact and still refused.
let by_title = element_action_owner_from(
Ok(windows),
&WindowTarget {
title: Some("GitHub".to_string()),
..Default::default()
},
)
.unwrap();
assert!(by_title.exact_window);
assert_eq!(
backend
.check_node_target(&node, &by_title)
.await
.unwrap_err(),
node_window_mismatch_message(2, 20, 10)
);
}
#[test]
fn post_action_tree_records_only_a_window_it_holds() {
let mut with_pid = pidless_window(20);
with_pid.pid = Some(7);
assert!(post_action_tree_window(Some(&with_pid), Some(7), None, None).is_some());
assert!(post_action_tree_window(Some(&with_pid), Some(8), None, None).is_none());
// No pid: only a tree narrowed to the window's frame is recorded.
let pidless = pidless_window(20);
assert!(post_action_tree_window(Some(&pidless), Some(8), None, None).is_none());
assert!(post_action_tree_window(Some(&pidless), None, Some("Window 20"), None).is_some());
assert!(post_action_tree_window(None, Some(7), Some("Window 20"), None).is_none());
// app_id resolved window 10, but the tree was narrowed through the
// cached scope of window 20: it is window 20's tree.
let mut window_10 = pidless_window(10);
window_10.pid = Some(7);
let cached = CachedTreeWindow {
scope: "Window 20".to_string(),
pid: Some(7),
window: Some((20, "i3".to_string())),
};
let record = |root_pid, scope, cached: &CachedTreeWindow| {
post_action_tree_window(Some(&window_10), root_pid, scope, Some(cached))
};
assert_eq!(
record(Some(7), Some("Window 20"), &cached),
Some((20, "i3".to_string()))
);
// A tree from no pid or another pid is not the cached window's.
assert_eq!(record(None, Some("Window 20"), &cached), None);
assert_eq!(record(Some(8), Some("Window 20"), &cached), None);
// Another frame matched this time, or none did: record nothing.
assert_eq!(record(Some(7), Some("Window 2"), &cached), None);
assert_eq!(record(Some(7), None, &cached), None);
// A cached scope with no cached window never falls back to window 10.
let no_window = CachedTreeWindow {
window: None,
..cached.clone()
};
assert_eq!(record(Some(7), Some("Window 20"), &no_window), None);
}
#[tokio::test]
async fn post_action_tree_keeps_its_window_for_the_next_action() {
// sway: click(window_id=20, include_tree=true), then an action on
// window 20 with an index from that tree.
let backend = ComputerUseLinux::default();
let node = cached_node(2, ":1.9/org/a11y/atspi/accessible/42");
backend
.commit_observed_snapshot(
std::slice::from_ref(&node),
None,
Some((20, "i3".to_string())),
Some("Window 20".to_string()),
)
.await;
assert!(backend
.check_node_target(&node, &pidless_owner(20))
.await
.is_ok());
let cached = backend.last_snapshot.lock().unwrap();
assert_eq!(cached.window_scope.as_deref(), Some("Window 20"));
}
#[test]
fn identifier_owned_by_another_pid_is_refused() {
assert_eq!(
check_identifier_owner(Some(APP_B), Some(APP_A)).unwrap_err(),
identifier_target_mismatch_message(APP_B, APP_A)
);
assert!(check_identifier_owner(Some(APP_A), Some(APP_A)).is_ok());
assert!(check_identifier_owner(None, Some(APP_A)).is_ok());
assert!(check_identifier_owner(Some(APP_B), None).is_ok());
}
}
#[cfg(test)]
mod post_action_capture_tests {
use super::*;
use std::os::unix::fs::PermissionsExt;
use std::sync::atomic::{AtomicUsize, Ordering};
type ToolCall<'a> =
std::pin::Pin<Box<dyn Future<Output = Result<CallToolResult, ErrorData>> + 'a>>;
fn arguments<T: serde::de::DeserializeOwned>(value: serde_json::Value) -> Parameters<T> {
Parameters(serde_json::from_value(value).unwrap())
}
/// Runs `action` against a freshly cached report, then reads readiness the
/// way get_app_state does. Returns the action's result and whether that
/// read had to probe again.
async fn readiness_probed_after(
server: &ComputerUseLinux,
probes: &AtomicUsize,
action: impl Future<Output = Result<CallToolResult, ErrorData>>,
) -> (CallToolResult, bool) {
server.diagnostics.report(false).await;
let cached = probes.load(Ordering::SeqCst);
let result = action.await.unwrap();
server.diagnostics.report(false).await;
(result, probes.load(Ordering::SeqCst) != cached)
}
fn solid_png(width: u32, height: u32) -> Vec<u8> {
let img = image::RgbaImage::from_pixel(width, height, image::Rgba([32, 128, 192, 255]));
let mut out = Vec::new();
image::DynamicImage::ImageRgba8(img)
.write_to(&mut std::io::Cursor::new(&mut out), image::ImageFormat::Png)
.unwrap();
out
}
#[test]
fn screenshot_after_deserializes_bool_and_scope_strings() {
let empty: PostActionCaptureParams = serde_json::from_value(serde_json::json!({})).unwrap();
assert_eq!(empty.screenshot_scope(), None);
assert_eq!(empty.include_tree, None);
let disabled: PostActionCaptureParams =
serde_json::from_value(serde_json::json!({"screenshot_after": false})).unwrap();
assert_eq!(disabled.screenshot_scope(), None);
let enabled: PostActionCaptureParams =
serde_json::from_value(serde_json::json!({"screenshot_after": true})).unwrap();
assert_eq!(
enabled.screenshot_scope(),
Some(ScreenshotAfterScope::Screen)
);
let screen: PostActionCaptureParams =
serde_json::from_value(serde_json::json!({"screenshot_after": "screen"})).unwrap();
assert_eq!(
screen.screenshot_scope(),
Some(ScreenshotAfterScope::Screen)
);
let window: PostActionCaptureParams =
serde_json::from_value(serde_json::json!({"screenshot_after": "window"})).unwrap();
assert_eq!(
window.screenshot_scope(),
Some(ScreenshotAfterScope::Window)
);
let tree_only: PostActionCaptureParams =
serde_json::from_value(serde_json::json!({"include_tree": true})).unwrap();
assert_eq!(tree_only.screenshot_scope(), None);
assert_eq!(tree_only.include_tree, Some(true));
assert!(serde_json::from_value::<PostActionCaptureParams>(
serde_json::json!({"screenshot_after": "invalid"})
)
.is_err());
}
#[test]
fn editable_text_results_map_ok_verified_and_mcp_error_status() {
for (outcome, expected_ok, expected_verified) in [
(EditableTextSetOutcome::Verified, true, Some(true)),
(
EditableTextSetOutcome::MismatchAfterDeadline,
false,
Some(false),
),
(EditableTextSetOutcome::ReadbackFailed, false, Some(false)),
(EditableTextSetOutcome::ReadbackTimedOut, false, Some(false)),
(
EditableTextSetOutcome::PasswordUnverifiable,
true,
Some(false),
),
] {
let (output, verified) = editable_text_action_result(outcome, None);
assert_eq!(output.ok, expected_ok);
assert_eq!(verified, expected_verified);
let result = structured_tool_result(
SetValueToolOutput {
action: output,
verified,
observation: PostActionObservation::default(),
},
None,
)
.unwrap();
assert_eq!(result.is_error, Some(!expected_ok));
let structured = result.structured_content.unwrap();
assert_eq!(structured["ok"], serde_json::json!(expected_ok));
assert_eq!(structured["verified"], serde_json::json!(expected_verified));
}
}
#[test]
fn set_value_output_omits_verified_for_numeric_and_early_failures() {
let output = SetValueToolOutput {
action: action_result("set_value", Ok(Vec::new()), None),
verified: None,
observation: PostActionObservation::default(),
};
let value = serde_json::to_value(output).unwrap();
assert!(!value.as_object().unwrap().contains_key("verified"));
}
#[test]
fn only_set_value_schema_exposes_verified() {
let tools = ComputerUseLinux::default().mcp_tool_router().list_all();
for tool in tools {
let value = serde_json::to_value(&tool).unwrap();
let Some(properties) = value["outputSchema"]["properties"].as_object() else {
continue;
};
if tool.name == "set_value" {
assert!(properties.contains_key("verified"));
} else {
assert!(!properties.contains_key("verified"), "tool {}", tool.name);
}
}
}
#[test]
fn action_tool_output_omits_observation_fields_when_not_requested() {
let output = ActionToolOutput {
action: action_result("click", Ok(Vec::new()), None),
observation: PostActionObservation::default(),
};
let value = serde_json::to_value(output).unwrap();
let object = value.as_object().unwrap();
for key in [
"screenshot",
"screenshot_error",
"accessibility_tree",
"accessibility_tree_raw_count",
"tree_scoped",
"accessibility_tree_truncated",
"accessibility_error",
] {
assert!(!object.contains_key(key), "unexpected key {key} in {value}");
}
}
#[test]
fn mutating_tool_schemas_expose_post_action_capture_parameters() {
let router = ComputerUseLinux::default().mcp_tool_router();
let tools = router.list_all();
for name in [
"activate_window",
"move_window",
"resize_window",
"click",
"drag",
"scroll",
"press_key",
"type_text",
"perform_action",
"set_value",
] {
let tool = tools
.iter()
.find(|tool| tool.name == name)
.unwrap_or_else(|| panic!("missing tool {name}"));
let value = serde_json::to_value(tool).unwrap();
let properties = value["inputSchema"]["properties"]
.as_object()
.unwrap_or_else(|| panic!("{name} inputSchema.properties is not an object"));
for prop in [
"screenshot_after",
"include_tree",
"format",
"quality",
"scale",
"max_width",
"max_height",
"max_bytes",
"screenshot_file_path",
] {
assert!(
properties.contains_key(prop),
"{name} is missing post-action capture property {prop}"
);
}
}
}
#[test]
fn post_action_capture_executes_after_action_and_preserves_action_ok_on_capture_failure() {
let dir = std::env::temp_dir().join(format!(
"cul-post-action-capture-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir(&dir).unwrap();
std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).unwrap();
struct Cleanup(std::path::PathBuf);
impl Drop for Cleanup {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.0);
}
}
let _cleanup = Cleanup(dir.clone());
let sample_png = dir.join("sample.png");
std::fs::write(&sample_png, solid_png(64, 48)).unwrap();
let log_path = dir.join("events.log");
let xdotool = dir.join("xdotool");
std::fs::write(
&xdotool,
"#!/bin/sh\nprintf 'xdotool\\n' >> \"$CUL_POST_ACTION_LOG\"\nexit \"${CUL_POST_ACTION_FAIL_INPUT:-0}\"\n",
)
.unwrap();
std::fs::set_permissions(&xdotool, std::fs::Permissions::from_mode(0o700)).unwrap();
let gnome_screenshot = dir.join("gnome-screenshot");
std::fs::write(
&gnome_screenshot,
r#"#!/bin/sh
printf 'gnome-screenshot\n' >> "$CUL_POST_ACTION_LOG"
if [ "${CUL_POST_ACTION_FAIL_SCREENSHOT:-0}" = "1" ]; then
echo "simulated capture failure" >&2
exit 1
fi
while [ "$#" -gt 0 ]; do
if [ "$1" = "-f" ] && [ "$#" -ge 2 ]; then
/bin/cp "$CUL_POST_ACTION_PNG" "$2"
exit 0
fi
shift
done
exit 1
"#,
)
.unwrap();
std::fs::set_permissions(&gnome_screenshot, std::fs::Permissions::from_mode(0o700))
.unwrap();
let hyprctl = dir.join("hyprctl");
std::fs::write(
&hyprctl,
r#"#!/bin/sh
[ -n "$CUL_POST_ACTION_WINDOW_PID" ] || exit 1
case "$1" in
clients)
printf '[{"address":"0x1","title":"Background","pid":%s,"at":[0,0],"size":[16,12],"monitor":0,"focusHistoryID":1},{"address":"0x2","title":"Foreground","pid":%s,"at":[8,8],"size":[32,24],"monitor":0,"focusHistoryID":0}]\n' "$CUL_POST_ACTION_WINDOW_PID" "$CUL_POST_ACTION_WINDOW_PID"
;;
monitors) echo '[{"id":0,"x":0,"y":0,"scale":1}]' ;;
*) exit 1 ;;
esac
"#,
)
.unwrap();
std::fs::set_permissions(&hyprctl, std::fs::Permissions::from_mode(0o700)).unwrap();
// wmctrl answers only while CUL_WMCTRL_STATE names a file. It lists
// window 7, applies a resize, and fails every listing after the one
// that verified the resize.
let wmctrl = dir.join("wmctrl");
std::fs::write(
&wmctrl,
r#"#!/bin/sh
[ -n "$CUL_WMCTRL_STATE" ] || exit 1
read -r state < "$CUL_WMCTRL_STATE"
case "$1:$state" in
-l:verified) exit 1 ;;
-l:resized) echo verified > "$CUL_WMCTRL_STATE"; size="40 30" ;;
-l:*) size="32 24" ;;
*) case "$*" in *" -e "*) echo resized > "$CUL_WMCTRL_STATE" ;; esac; exit 0 ;;
esac
echo "0x00000007 0 0 0 0 $size fixture.Fixture host Fixture"
"#,
)
.unwrap();
std::fs::set_permissions(&wmctrl, std::fs::Permissions::from_mode(0o700)).unwrap();
let mut command = Command::new(std::env::current_exe().unwrap());
command
.args([
"--exact",
"server::post_action_capture_tests::post_action_capture_child",
"--ignored",
"--nocapture",
])
.env_clear()
.env("PATH", &dir)
.env("HOME", &dir)
.env("XDG_RUNTIME_DIR", &dir)
.env("DISPLAY", ":9876")
.env("WAYLAND_DISPLAY", "")
.env("XDG_SESSION_TYPE", "x11")
.env("CU_DISABLE_ABS_POINTER", "1")
.env("COMPUTER_USE_LINUX_SCREENSHOT_BACKEND", "gnome-screenshot")
.env("CUL_POST_ACTION_CHILD", "1")
.env("CUL_POST_ACTION_LOG", &log_path)
.env("CUL_POST_ACTION_PNG", &sample_png)
.env(
"DBUS_SESSION_BUS_ADDRESS",
format!("unix:path={}/no-bus", dir.display()),
);
for key in [
"DESKTOP_SESSION",
"HYPRLAND_INSTANCE_SIGNATURE",
"XAUTHORITY",
"XDG_SESSION_DESKTOP",
"XDG_CURRENT_DESKTOP",
] {
command.env(key, "post-action-test");
}
let output = crate::command_runner::output_blocking_with_timeout(
&mut command,
"post-action capture child",
Duration::from_secs(15),
)
.unwrap();
assert!(
String::from_utf8_lossy(&output.stdout).contains("running 1 test"),
"post-action capture child must execute its assertions"
);
assert!(
output.status.success(),
"{}{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
}
#[tokio::test]
#[ignore = "isolated environment subprocess; invoked by post_action_capture_executes_after_action_and_preserves_action_ok_on_capture_failure"]
async fn post_action_capture_child() {
assert_eq!(std::env::var("CUL_POST_ACTION_CHILD").as_deref(), Ok("1"));
let log_path = std::path::PathBuf::from(std::env::var("CUL_POST_ACTION_LOG").unwrap());
// The fake desktop has a window backend exactly while wmctrl answers.
let (server, diagnostics_probes) = super::tests::server_with_labelled_diagnostics(
|| false,
|| std::env::var_os("CUL_WMCTRL_STATE").is_some(),
);
server.diagnostics.report(false).await;
// 1. Successful click + post-action JPEG screenshot.
let click_params: ClickParams = serde_json::from_value(serde_json::json!({
"x": 120,
"y": 240,
"screenshot_after": true,
"format": "jpeg",
"quality": 75
}))
.unwrap();
let result = server.click(Parameters(click_params)).await.unwrap();
assert_ne!(result.is_error, Some(true));
assert_eq!(result.content.len(), 2);
let image = result.content[0]
.as_image()
.expect("first content block must be the post-action image");
assert_eq!(image.mime_type, "image/jpeg");
assert!(!image.data.is_empty());
let structured = result.structured_content.expect("structured content");
assert_eq!(structured["ok"], serde_json::json!(true));
assert_eq!(
structured["screenshot"]["mime_type"],
serde_json::json!("image/jpeg")
);
assert_eq!(
structured["screenshot"]["cropped_to_window"],
serde_json::json!(false)
);
assert_eq!(structured["screenshot"]["width"], serde_json::json!(64));
assert_eq!(structured["screenshot"]["height"], serde_json::json!(48));
let events = std::fs::read_to_string(&log_path).unwrap();
assert_eq!(
events, "xdotool\ngnome-screenshot\n",
"action must execute strictly before post-action screenshot capture"
);
server.diagnostics.report(false).await;
assert_eq!(
diagnostics_probes.load(std::sync::atomic::Ordering::SeqCst),
1,
"a successful post-action capture keeps the cached diagnostics"
);
// 2. Screenshot failure after a successful action must keep ok=true.
std::fs::write(&log_path, "").unwrap();
std::env::set_var("CUL_POST_ACTION_FAIL_SCREENSHOT", "1");
let click_params: ClickParams = serde_json::from_value(serde_json::json!({
"x": 120,
"y": 240,
"screenshot_after": "screen"
}))
.unwrap();
let result = server.click(Parameters(click_params)).await.unwrap();
std::env::remove_var("CUL_POST_ACTION_FAIL_SCREENSHOT");
assert_ne!(result.is_error, Some(true));
assert_eq!(result.content.len(), 1);
let structured = result.structured_content.expect("structured content");
assert_eq!(structured["ok"], serde_json::json!(true));
assert!(structured.get("screenshot").is_none());
assert!(structured["screenshot_error"]
.as_str()
.unwrap()
.contains("gnome-screenshot"));
assert!(structured["message"]
.as_str()
.unwrap()
.contains("Post-action screenshot failed:"));
server.diagnostics.report(false).await;
assert_eq!(
diagnostics_probes.load(std::sync::atomic::Ordering::SeqCst),
2,
"a failed post-action capture makes the next readiness read probe again"
);
// 3. Failed action (relative coordinates without a target) must not capture.
std::fs::write(&log_path, "").unwrap();
let bad_click: ClickParams = serde_json::from_value(serde_json::json!({
"x": 120,
"y": 240,
"relative": true,
"screenshot_after": true
}))
.unwrap();
let result = server.click(Parameters(bad_click)).await.unwrap();
assert_eq!(result.is_error, Some(true));
let structured = result.structured_content.expect("structured content");
assert_eq!(structured["ok"], serde_json::json!(false));
assert!(structured.get("screenshot").is_none());
assert!(structured.get("screenshot_error").is_none());
let events = std::fs::read_to_string(&log_path).unwrap();
assert!(
events.is_empty(),
"failed action must not run xdotool or gnome-screenshot: {events}"
);
// 4. activate_window preserves post-action observation notes in message.
std::env::set_var("CUL_POST_ACTION_FAIL_SCREENSHOT", "1");
let activate_capture: PostActionCaptureParams =
serde_json::from_value(serde_json::json!({"screenshot_after": "screen"})).unwrap();
let activate_result = server
.finish_activate_window_output(
ActivateWindowOutput {
ok: true,
implemented: true,
backend: "test".to_string(),
focus: None,
error: None,
permissions_hint: None,
received: None,
},
&activate_capture,
&WindowTarget::default(),
)
.await
.unwrap();
std::env::remove_var("CUL_POST_ACTION_FAIL_SCREENSHOT");
let activate_structured = activate_result
.structured_content
.expect("structured activate_window content");
assert_eq!(activate_structured["ok"], serde_json::json!(true));
assert!(activate_structured["message"]
.as_str()
.unwrap()
.contains("Post-action screenshot failed:"));
// 5. A window capture that falls back to the activated window when the
// window list fails still makes the next readiness read probe again.
server.diagnostics.report(false).await;
let probes = diagnostics_probes.load(std::sync::atomic::Ordering::SeqCst);
let window: WindowInfo = serde_json::from_value(serde_json::json!({
"window_id": 7,
"bounds": {"x": 8, "y": 8, "width": 32, "height": 24},
"focused": true,
"hidden": false,
"backend": "test"
}))
.unwrap();
let window_capture: PostActionCaptureParams =
serde_json::from_value(serde_json::json!({"screenshot_after": "window"})).unwrap();
let activate_result = server
.finish_activate_window_output(
ActivateWindowOutput {
ok: true,
implemented: true,
backend: "test".to_string(),
focus: Some(WindowFocusResult {
requested_window: window.clone(),
focused_window: Some(window),
exact_window_focused: true,
app_focused: true,
backend: "test".to_string(),
note: String::new(),
}),
error: None,
permissions_hint: None,
received: None,
},
&window_capture,
&WindowTarget {
window_id: Some(7),
..Default::default()
},
)
.await
.unwrap();
let activate_structured = activate_result
.structured_content
.expect("structured activate_window content");
assert!(activate_structured.get("screenshot_error").is_none());
assert_eq!(
activate_structured["screenshot"]["cropped_to_window"],
serde_json::json!(true)
);
server.diagnostics.report(false).await;
assert_eq!(
diagnostics_probes.load(std::sync::atomic::Ordering::SeqCst),
probes + 1,
"a failed window list makes the next readiness read probe again"
);
// 6. Every action that fails, whatever the reason, makes the next
// readiness read probe again. Input commands fail or are missing, no
// window backend answers, and no observation has cached an element
// index.
std::env::set_var("CUL_POST_ACTION_FAIL_INPUT", "9");
let failed_actions: Vec<(&str, ToolCall<'_>)> = vec![
(
"click",
Box::pin(server.click(arguments(serde_json::json!({"x": 120, "y": 240})))),
),
(
"type_text",
Box::pin(server.type_text(arguments(serde_json::json!({"text": "a"})))),
),
(
"press_key",
Box::pin(server.press_key(arguments(serde_json::json!({"key": "Escape"})))),
),
(
"scroll",
Box::pin(server.scroll(arguments(
serde_json::json!({"x": 120, "y": 240, "direction": "down"}),
))),
),
(
"drag",
Box::pin(server.drag(arguments(serde_json::json!({
"start_x": 10, "start_y": 10, "end_x": 40, "end_y": 40
})))),
),
(
"perform_action",
Box::pin(server.perform_action(arguments(serde_json::json!({"element_index": 3})))),
),
(
"set_value",
Box::pin(server.set_value(arguments(
serde_json::json!({"element_index": 3, "value": "x"}),
))),
),
(
"activate_window",
Box::pin(server.activate_window(arguments(serde_json::json!({"window_id": 7})))),
),
(
"move_window",
Box::pin(server.move_window(arguments(
serde_json::json!({"window_id": 7, "x": 0, "y": 0}),
))),
),
(
"resize_window",
Box::pin(server.resize_window(arguments(
serde_json::json!({"window_id": 7, "width": 40, "height": 30}),
))),
),
];
for (tool, action) in failed_actions {
let (result, probed) =
readiness_probed_after(&server, &diagnostics_probes, action).await;
assert_eq!(
result.is_error,
Some(true),
"{tool} must fail here: {:?}",
result.structured_content
);
assert!(
probed,
"a failed {tool} makes the next readiness read probe again"
);
}
std::env::remove_var("CUL_POST_ACTION_FAIL_INPUT");
// 7. A resize that lands but whose geometry re-query finds no window
// backend stays ok and still makes the next readiness read probe again.
let wmctrl_state = log_path.with_file_name("wmctrl.state");
std::fs::write(&wmctrl_state, "").unwrap();
std::env::set_var("CUL_WMCTRL_STATE", &wmctrl_state);
let (result, probed) = readiness_probed_after(
&server,
&diagnostics_probes,
server.resize_window(arguments(
serde_json::json!({"window_id": 7, "width": 40, "height": 30}),
)),
)
.await;
std::env::remove_var("CUL_WMCTRL_STATE");
let structured = result.structured_content.unwrap();
assert_eq!(structured["ok"], true, "{structured}");
assert!(structured["window"].is_null(), "{structured}");
assert!(
probed,
"a failed geometry re-query makes the next readiness read probe again"
);
// 8. A get_app_state whose window lookup fails reports readiness from a
// probe it ran itself, that report carries the call's own screenshot
// result, and the next call probes again. The raw capture succeeds
// before the unresolved target refuses the crop.
server.diagnostics.report(false).await;
let probes = diagnostics_probes.load(Ordering::SeqCst);
let state =
super::tests::app_state_value(&server, serde_json::json!({"window_id": 7})).await;
assert!(state["window_error"].is_string(), "{state}");
assert_eq!(
state["readiness"]["recommended_next_step"],
format!("probe run {}", probes + 1)
);
assert_eq!(state["readiness"]["screenshot_capture_status"], "verified");
let next = super::tests::app_state_value(
&server,
serde_json::json!({"include_screenshot": false}),
)
.await;
assert_eq!(
next["readiness"]["recommended_next_step"],
format!("probe run {}", probes + 2),
"the call after a failed one probes again"
);
// 9. A get_app_state that resolves a window the cached report says no
// backend can list reports refreshed readiness, and the next call
// reuses that report.
let cached = server.diagnostics.report(false).await;
assert!(!cached.readiness.can_query_windows);
std::fs::write(&wmctrl_state, "").unwrap();
std::env::set_var("CUL_WMCTRL_STATE", &wmctrl_state);
let probes = diagnostics_probes.load(Ordering::SeqCst);
let window_7 = serde_json::json!({"window_id": 7, "include_screenshot": false});
let state = super::tests::app_state_value(&server, window_7.clone()).await;
assert_eq!(state["window_context"]["window_id"], 7, "{state}");
assert_eq!(state["readiness"]["can_query_windows"], true, "{state}");
assert_eq!(
state["readiness"]["recommended_next_step"],
format!("probe run {}", probes + 1)
);
let next = super::tests::app_state_value(&server, window_7).await;
std::env::remove_var("CUL_WMCTRL_STATE");
assert_eq!(next["window_context"]["window_id"], 7, "{next}");
assert_eq!(
next["readiness"]["recommended_next_step"],
format!("probe run {}", probes + 1),
"the refreshed report already shows the window backend"
);
// 10. A window that wmctrl lists after the GNOME extension failed. A
// cached report that calls the extension healthy is refreshed for this
// call, and the next call probes again. One that calls it unhealthy,
// as on KDE, where it fails on every call, stays cached.
for (extension_ok, expected) in [
(true, ["probe run 2", "probe run 3"]),
(false, ["probe run 1", "probe run 1"]),
] {
let (server, _) = super::tests::server_with_labelled_reports(
|| false,
move || {
let mut report =
crate::diagnostics_impl::tests::doctor_report_fixture(false, true);
report.windowing.backends.insert(
GNOME_SHELL_EXTENSION_BACKEND.to_string(),
crate::diagnostics::Check {
ok: extension_ok,
detail: String::new(),
},
);
report
},
);
server.diagnostics.report(false).await;
std::fs::write(&wmctrl_state, "").unwrap();
std::env::set_var("CUL_WMCTRL_STATE", &wmctrl_state);
let state = super::tests::app_state_value(
&server,
serde_json::json!({"window_id": 7, "include_screenshot": false}),
)
.await;
std::env::remove_var("CUL_WMCTRL_STATE");
assert_eq!(
state["window_context"]["backend"],
crate::windows::X11_BACKEND,
"{state}"
);
let next = super::tests::app_state_value(
&server,
serde_json::json!({"include_screenshot": false}),
)
.await;
assert_eq!(
[
&state["readiness"]["recommended_next_step"],
&next["readiness"]["recommended_next_step"],
],
expected,
"cached report calls the failed GNOME extension healthy: {extension_ok}"
);
}
// The fake display cannot monitor XKB. Preserve xdotool compatibility
// and make the unavailable detection explicit, including after capture.
std::fs::write(&log_path, "").unwrap();
let params = serde_json::from_value(serde_json::json!({
"text": "quotes '\"; : / date +%H:%M:%S",
"screenshot_after": true
}))
.unwrap();
let (result, probed) = readiness_probed_after(
&server,
&diagnostics_probes,
server.type_text(Parameters(params)),
)
.await;
assert!(
!probed,
"a successful type_text keeps the cached diagnostics"
);
let structured = result.structured_content.unwrap();
assert_eq!(structured["ok"], true);
assert!(structured.get("screenshot").is_some());
let message = structured["message"].as_str().unwrap();
// The layout warning survives the post-action capture.
for expected in [
"WARNING: X11 keyboard layout",
"br/ABNT2",
"before pressing Enter",
"Field contents were not verified",
] {
assert!(message.contains(expected), "missing {expected}: {message}");
}
assert!(
message.contains("layout-change detection was unavailable"),
"{message}"
);
assert_eq!(
std::fs::read_to_string(&log_path).unwrap(),
"xdotool\ngnome-screenshot\n"
);
// The text-specific risk does not apply to named key dispatch.
let (result, probed) = readiness_probed_after(
&server,
&diagnostics_probes,
server.press_key(arguments(serde_json::json!({"key": "Escape"}))),
)
.await;
assert!(
!probed,
"a successful press_key keeps the cached diagnostics"
);
assert_eq!(result.structured_content.as_ref().unwrap()["ok"], true);
assert!(!result.structured_content.unwrap()["message"]
.as_str()
.unwrap()
.contains("WARNING: X11 keyboard layout"));
// A launched failure stays an error and must not claim dispatch.
std::env::set_var("CUL_POST_ACTION_FAIL_INPUT", "9");
let params = serde_json::from_value(serde_json::json!({"key": "Escape"})).unwrap();
let result = server.press_key(Parameters(params)).await.unwrap();
std::env::remove_var("CUL_POST_ACTION_FAIL_INPUT");
let structured = result.structured_content.unwrap();
assert_eq!(structured["ok"], false);
assert!(!structured["message"]
.as_str()
.unwrap()
.contains("WARNING: X11 keyboard layout"));
// Screenshots keep their focused-window fallback when the tree owner
// has several windows or cannot be established on a pidless desktop.
let owner_error = "no verified app or window owner".to_string();
let tree_owner = ElementObservationTarget {
owner: WindowTarget {
pid: Some(12345),
..Default::default()
},
..Default::default()
};
let window_capture: PostActionCaptureParams = serde_json::from_value(
serde_json::json!({"screenshot_after": "window", "include_tree": true}),
)
.unwrap();
for pid in [Some(12345), None] {
std::env::set_var(
"CUL_POST_ACTION_WINDOW_PID",
pid.map_or_else(|| "null".to_string(), |pid| pid.to_string()),
);
let tree_target = if pid.is_some() {
Ok(&tree_owner)
} else {
Err(&owner_error)
};
let (observation, screenshot, _) = server
.capture_element_observation(&window_capture, None, tree_target)
.await;
assert!(observation.screenshot_error.is_none());
assert!(observation.accessibility_error.is_some());
let summary = observation.screenshot.unwrap();
assert_eq!(summary.window_title.as_deref(), Some("Foreground"));
assert_eq!((summary.width, summary.height), (32, 24));
assert_eq!(summary.cropped_to_window, Some(true));
assert!(screenshot.is_some());
let background = WindowTarget {
window_id: Some(1),
..Default::default()
};
let (observation, screenshot, _) = server
.capture_element_observation(&window_capture, Some(&background), tree_target)
.await;
assert!(observation.screenshot_error.is_some());
assert!(observation.screenshot.is_none());
assert!(screenshot.is_none());
}
std::env::remove_var("CUL_POST_ACTION_WINDOW_PID");
std::fs::write(&log_path, "").unwrap();
let set_value_capture: PostActionCaptureParams = serde_json::from_value(
serde_json::json!({"screenshot_after": "screen", "include_tree": true}),
)
.unwrap();
let (output, verified) =
editable_text_action_result(EditableTextSetOutcome::MismatchAfterDeadline, None);
// A failed owner binding drops the replaced snapshot, as every
// failed post-action tree read does.
server.commit_snapshot(&[], Some(12345)).await;
assert!(!server.cached_snapshot_id().is_empty());
let result = server
.finish_set_value_output(
output,
verified,
&set_value_capture,
None,
Err(&owner_error),
)
.await
.unwrap();
assert_eq!(result.is_error, Some(true));
assert_eq!(result.content.len(), 2);
let structured = result.structured_content.unwrap();
assert_eq!(structured["ok"], false);
assert_eq!(structured["verified"], false);
assert_eq!(structured["screenshot"]["width"], 64);
assert!(structured.get("screenshot_error").is_none());
assert!(structured["accessibility_error"].is_string());
assert_eq!(structured["accessibility_tree_raw_count"], 0);
assert_eq!(structured["tree_scoped"], false);
assert!(server.cached_snapshot_id().is_empty());
assert_eq!(
std::fs::read_to_string(&log_path).unwrap(),
"gnome-screenshot\n"
);
std::fs::write(&log_path, "").unwrap();
let output = action_result("set_value", Err("pre-dispatch failure".to_string()), None);
let result = server
.finish_set_value_output(
output,
None,
&set_value_capture,
None,
Ok(&ElementObservationTarget::default()),
)
.await
.unwrap();
assert_eq!(result.is_error, Some(true));
let structured = result.structured_content.unwrap();
assert_eq!(structured["ok"], false);
assert!(structured.get("screenshot").is_none());
assert!(structured.get("accessibility_tree").is_none());
assert!(structured.get("accessibility_error").is_none());
assert!(std::fs::read_to_string(&log_path).unwrap().is_empty());
}
}
#[cfg(test)]
mod screenshot_file_tests {
use super::*;
use std::os::unix::fs::PermissionsExt;
fn solid_png(width: u32, height: u32) -> Vec<u8> {
let img = image::RgbaImage::from_pixel(width, height, image::Rgba([32, 128, 192, 255]));
let mut out = Vec::new();
image::DynamicImage::ImageRgba8(img)
.write_to(&mut std::io::Cursor::new(&mut out), image::ImageFormat::Png)
.unwrap();
out
}
fn refusal(tool: &str, arguments: serde_json::Value, remote: bool) -> Option<String> {
let serde_json::Value::Object(arguments) = arguments else {
panic!("arguments must be an object");
};
screenshot_file_refusal(tool, Some(&arguments), true, remote)
}
#[test]
fn file_output_parameters_are_in_every_screenshot_schema() {
let server = ComputerUseLinux::default();
let router = server.mcp_tool_router();
for name in [
"screenshot",
"get_app_state",
"click",
"type_text",
"press_key",
"scroll",
"drag",
"perform_action",
"set_value",
"activate_window",
"move_window",
"resize_window",
] {
let tool = router
.get(name)
.unwrap_or_else(|| panic!("{name} is registered"));
assert!(
tool.input_schema["properties"]
.as_object()
.unwrap()
.contains_key("screenshot_file_path"),
"{name} lacks screenshot_file_path"
);
}
}
#[test]
fn preflight_refuses_unusable_screenshot_file_paths_before_the_tool_runs() {
let dir = std::env::temp_dir().join(format!(
"cul-shot-preflight-{}-{:016x}",
std::process::id(),
getrandom::u64().unwrap()
));
std::fs::create_dir(&dir).unwrap();
struct Cleanup(std::path::PathBuf);
impl Drop for Cleanup {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.0);
}
}
let _cleanup = Cleanup(dir.clone());
let png = dir.join("shot.png");
let png = png.to_str().unwrap();
// Usable paths pass, and a call without the parameter is untouched.
assert_eq!(refusal("screenshot", serde_json::json!({}), true), None);
assert_eq!(
refusal(
"screenshot",
serde_json::json!({"screenshot_file_path": null}),
true
),
None
);
assert_eq!(
refusal(
"screenshot",
serde_json::json!({"screenshot_file_path": png}),
false
),
None
);
assert_eq!(
refusal(
"get_app_state",
serde_json::json!({"screenshot_file_path": png}),
false
),
None
);
assert_eq!(
refusal(
"click",
serde_json::json!({"x": 1, "y": 1, "screenshot_after": "window", "screenshot_file_path": png}),
false
),
None
);
// A tool without the parameter ignores it like any unknown argument.
let serde_json::Value::Object(arguments) =
serde_json::json!({"screenshot_file_path": "relative.png"})
else {
unreachable!()
};
assert_eq!(
screenshot_file_refusal("list_windows", Some(&arguments), false, true),
None
);
// Over HTTP every use is refused, even a valid path.
for (tool, arguments) in [
(
"screenshot",
serde_json::json!({"screenshot_file_path": png}),
),
(
"get_app_state",
serde_json::json!({"screenshot_file_path": png}),
),
(
"click",
serde_json::json!({"x": 1, "y": 1, "screenshot_after": true, "screenshot_file_path": png}),
),
] {
let message = refusal(tool, arguments, true).expect("refused over HTTP");
assert!(
message.starts_with(&format!("{tool} was not run: "))
&& message.contains("refused over HTTP (mcp --listen)"),
"{message}"
);
}
// Without a capture there is nothing to save.
assert!(refusal(
"get_app_state",
serde_json::json!({"include_screenshot": false, "screenshot_file_path": png}),
false
)
.unwrap()
.contains("needs include_screenshot=true"));
for screenshot_after in [serde_json::Value::Null, serde_json::json!(false)] {
assert!(refusal(
"click",
serde_json::json!({"x": 1, "y": 1, "screenshot_after": screenshot_after, "screenshot_file_path": png}),
false
)
.unwrap()
.contains("needs screenshot_after"));
}
// Bad paths, with the format the call selects.
let missing = dir.join("missing").join("shot.png");
for (arguments, expected) in [
(
serde_json::json!({"screenshot_file_path": "shot.png"}),
"must be an absolute path",
),
(
serde_json::json!({"screenshot_file_path": missing}),
"does not exist",
),
(
serde_json::json!({"screenshot_file_path": dir}),
"does not match format png",
),
(
serde_json::json!({"screenshot_file_path": png, "format": "jpeg"}),
"does not match format jpeg",
),
] {
let message = refusal("screenshot", arguments.clone(), false)
.unwrap_or_else(|| panic!("{arguments} must be refused"));
assert!(message.contains(expected), "{arguments}: {message}");
}
// File output only creates: an existing entry is refused before the
// tool runs, on read-only and mutating tools alike.
let as_dir = dir.join("dir.png");
std::fs::create_dir(&as_dir).unwrap();
let existing = dir.join("existing.png");
std::fs::write(&existing, b"keep").unwrap();
for (tool, arguments) in [
(
"screenshot",
serde_json::json!({"screenshot_file_path": as_dir}),
),
(
"get_app_state",
serde_json::json!({"screenshot_file_path": existing}),
),
(
"click",
serde_json::json!({"x": 1, "y": 1, "screenshot_after": true, "screenshot_file_path": existing}),
),
] {
let message = refusal(tool, arguments, false).expect("an existing entry is refused");
assert!(message.contains("already exists"), "{message}");
}
assert_eq!(std::fs::read(&existing).unwrap(), b"keep");
}
#[test]
fn screenshot_file_output_writes_files_and_returns_no_image_content() {
let dir = std::env::temp_dir().join(format!(
"cul-shot-file-{}-{:016x}",
std::process::id(),
getrandom::u64().unwrap()
));
std::fs::create_dir(&dir).unwrap();
std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).unwrap();
struct Cleanup(std::path::PathBuf);
impl Drop for Cleanup {
fn drop(&mut self) {
let locked = self.0.join("locked");
let _ = std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o700));
let _ = std::fs::remove_dir_all(&self.0);
}
}
let _cleanup = Cleanup(dir.clone());
let sample_png = dir.join("sample.png");
std::fs::write(&sample_png, solid_png(64, 48)).unwrap();
let xdotool = dir.join("xdotool");
std::fs::write(
&xdotool,
"#!/bin/sh\nprintf 'xdotool\\n' >> \"$CUL_SHOT_FILE_LOG\"\nexit 0\n",
)
.unwrap();
std::fs::set_permissions(&xdotool, std::fs::Permissions::from_mode(0o700)).unwrap();
let gnome_screenshot = dir.join("gnome-screenshot");
std::fs::write(
&gnome_screenshot,
r#"#!/bin/sh
printf 'gnome-screenshot\n' >> "$CUL_SHOT_FILE_LOG"
while [ "$#" -gt 0 ]; do
if [ "$1" = "-f" ] && [ "$#" -ge 2 ]; then
/bin/cp "$CUL_SHOT_FILE_PNG" "$2"
exit 0
fi
shift
done
exit 1
"#,
)
.unwrap();
std::fs::set_permissions(&gnome_screenshot, std::fs::Permissions::from_mode(0o700))
.unwrap();
let out = dir.join("out");
std::fs::create_dir(&out).unwrap();
let mut command = Command::new(std::env::current_exe().unwrap());
command
.args([
"--exact",
"server::screenshot_file_tests::screenshot_file_output_child",
"--ignored",
"--nocapture",
])
.env_clear()
.env("PATH", &dir)
.env("HOME", &dir)
.env("XDG_RUNTIME_DIR", &dir)
.env("DISPLAY", ":9876")
.env("WAYLAND_DISPLAY", "")
.env("XDG_SESSION_TYPE", "x11")
.env("CU_DISABLE_ABS_POINTER", "1")
.env("COMPUTER_USE_LINUX_SCREENSHOT_BACKEND", "gnome-screenshot")
.env("CUL_SHOT_FILE_CHILD", "1")
.env("CUL_SHOT_FILE_LOG", dir.join("events.log"))
.env("CUL_SHOT_FILE_PNG", &sample_png)
.env("CUL_SHOT_FILE_OUT", &out)
.env(
"DBUS_SESSION_BUS_ADDRESS",
format!("unix:path={}/no-bus", dir.display()),
);
for key in [
"DESKTOP_SESSION",
"HYPRLAND_INSTANCE_SIGNATURE",
"XAUTHORITY",
"XDG_SESSION_DESKTOP",
"XDG_CURRENT_DESKTOP",
] {
command.env(key, "screenshot-file-test");
}
let output = crate::command_runner::output_blocking_with_timeout(
&mut command,
"screenshot file child",
Duration::from_secs(20),
)
.unwrap();
assert!(
String::from_utf8_lossy(&output.stdout).contains("running 1 test"),
"screenshot file child must execute its assertions"
);
assert!(
output.status.success(),
"{}{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
}
fn caption(result: &CallToolResult) -> serde_json::Value {
assert_eq!(result.content.len(), 1, "only the caption: {result:?}");
assert!(result.content[0].as_image().is_none());
serde_json::from_str(&result.content[0].as_text().unwrap().text).unwrap()
}
fn assert_no_image(result: &CallToolResult) {
assert!(
result
.content
.iter()
.all(|block| block.as_image().is_none()),
"no image content: {result:?}"
);
}
#[tokio::test]
#[ignore = "isolated environment subprocess; invoked by screenshot_file_output_writes_files_and_returns_no_image_content"]
async fn screenshot_file_output_child() {
use image::GenericImageView as _;
assert_eq!(std::env::var("CUL_SHOT_FILE_CHILD").as_deref(), Ok("1"));
let log = std::path::PathBuf::from(std::env::var("CUL_SHOT_FILE_LOG").unwrap());
let out = std::path::PathBuf::from(std::env::var("CUL_SHOT_FILE_OUT").unwrap());
let events = || std::fs::read_to_string(&log).unwrap_or_default();
let clear = || std::fs::write(&log, "").unwrap();
let (server, probes) = super::tests::server_with_labelled_diagnostics(|| false, || false);
let unprivileged = unsafe { libc::geteuid() } != 0;
let locked = out.join("locked");
std::fs::create_dir(&locked).unwrap();
std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o500)).unwrap();
// 1. The screenshot tool saves a png and returns only the caption.
let path = out.join("shot.png");
let result = server
.screenshot(Parameters(
serde_json::from_value(serde_json::json!({"screenshot_file_path": path})).unwrap(),
))
.await;
assert_ne!(result.is_error, Some(true), "{result:?}");
let caption_value = caption(&result);
let written = std::fs::read(&path).unwrap();
assert_eq!(caption_value["file_path"], serde_json::json!(path));
assert_eq!(caption_value["bytes"], serde_json::json!(written.len()));
assert_eq!(caption_value["format"], serde_json::json!("png"));
assert_eq!(caption_value["width"], serde_json::json!(64));
assert_eq!(caption_value["height"], serde_json::json!(48));
assert_eq!(caption_value["coordinate_width"], serde_json::json!(64));
assert_eq!(caption_value["coordinate_height"], serde_json::json!(48));
assert_eq!(caption_value["scale"], serde_json::json!(1.0));
assert_eq!(
image::load_from_memory(&written).unwrap().dimensions(),
(64, 48)
);
assert_eq!(events(), "gnome-screenshot\n");
let written_png = written;
// A jpeg, scaled down.
let jpeg = out.join("shot.jpg");
let result = server
.screenshot(Parameters(
serde_json::from_value(serde_json::json!({
"screenshot_file_path": jpeg,
"format": "jpeg",
"scale": 0.5
}))
.unwrap(),
))
.await;
let caption_value = caption(&result);
let written = std::fs::read(&jpeg).unwrap();
assert!(written.starts_with(&[0xff, 0xd8]), "a jpeg file");
assert_eq!(caption_value["format"], serde_json::json!("jpeg"));
assert_eq!(caption_value["width"], serde_json::json!(32));
assert_eq!(caption_value["coordinate_width"], serde_json::json!(64));
assert_eq!(caption_value["scale"], serde_json::json!(0.5));
assert_eq!(caption_value["bytes"], serde_json::json!(written.len()));
// 2. Bad paths are refused before any capture.
clear();
let as_dir = out.join("dir.png");
std::fs::create_dir(&as_dir).unwrap();
for (arguments, expected) in [
(
serde_json::json!({"screenshot_file_path": "shot.png"}),
"must be an absolute path",
),
(
serde_json::json!({"screenshot_file_path": out.join("missing").join("shot.png")}),
"does not exist",
),
(
serde_json::json!({"screenshot_file_path": as_dir}),
"already exists",
),
(
serde_json::json!({"screenshot_file_path": out.join("shot.jpeg")}),
"does not match format png",
),
// File output never replaces a file, so an earlier capture stays.
(
serde_json::json!({"screenshot_file_path": path}),
"already exists",
),
(
serde_json::json!({"screenshot_file_path": format!("{}/", out.join("slash.png").display())}),
"must end with the file name",
),
] {
let result = server
.screenshot(Parameters(
serde_json::from_value(arguments.clone()).unwrap(),
))
.await;
assert_eq!(result.is_error, Some(true), "{arguments}");
assert_no_image(&result);
let text = &result.content[0].as_text().unwrap().text;
assert!(text.contains(expected), "{arguments}: {text}");
}
assert_eq!(events(), "", "a refused path captures nothing");
assert!(!out.join("missing").exists());
assert_eq!(std::fs::read(&path).unwrap(), written_png);
// 3. Over HTTP the screenshot tool refuses file output.
let http = ComputerUseLinux {
http_transport: true,
..Default::default()
};
let remote = out.join("remote.png");
let result = http
.screenshot(Parameters(
serde_json::from_value(serde_json::json!({"screenshot_file_path": remote}))
.unwrap(),
))
.await;
assert_eq!(result.is_error, Some(true));
assert!(result.content[0]
.as_text()
.unwrap()
.text
.contains("refused over HTTP"));
assert!(!remote.exists());
assert_eq!(events(), "");
// A failed write is the call's error, with no inline fallback.
if unprivileged {
let result = server
.screenshot(Parameters(
serde_json::from_value(
serde_json::json!({"screenshot_file_path": locked.join("shot.png")}),
)
.unwrap(),
))
.await;
assert_eq!(result.is_error, Some(true));
assert_no_image(&result);
assert!(result.content[0]
.as_text()
.unwrap()
.text
.contains("failed to write screenshot file"));
}
// 4. get_app_state saves its screenshot and names the file.
let state = out.join("state.png");
let params: GetAppStateParams =
serde_json::from_value(serde_json::json!({"screenshot_file_path": state})).unwrap();
let result = server.app_state(params).await.unwrap();
assert_no_image(&result);
let structured = result.structured_content.unwrap();
let written = std::fs::read(&state).unwrap();
assert_eq!(
structured["screenshot"]["file_path"],
serde_json::json!(state)
);
assert_eq!(
structured["screenshot"]["bytes"],
serde_json::json!(written.len())
);
assert_eq!(
structured["screenshot"]["coordinate_width"],
serde_json::json!(64)
);
assert_eq!(structured["screenshot"]["scale"], serde_json::json!(1.0));
assert!(structured
.get("screenshot_error")
.is_none_or(serde_json::Value::is_null));
assert!(structured["message"]
.as_str()
.unwrap()
.contains("The screenshot was saved to"));
// A failed write is reported in screenshot_error, without the image.
if unprivileged {
let params: GetAppStateParams = serde_json::from_value(
serde_json::json!({"screenshot_file_path": locked.join("state.png")}),
)
.unwrap();
let result = server.app_state(params).await.unwrap();
assert_no_image(&result);
let structured = result.structured_content.unwrap();
assert!(structured["screenshot"].is_null());
assert!(structured["screenshot_error"]
.as_str()
.unwrap()
.contains("failed to write screenshot file"));
}
// 5. A post-action screenshot is saved after the action.
clear();
let after = out.join("after.png");
let click: ClickParams = serde_json::from_value(serde_json::json!({
"x": 10,
"y": 10,
"screenshot_after": true,
"screenshot_file_path": after
}))
.unwrap();
let result = server.click(Parameters(click)).await.unwrap();
assert_ne!(result.is_error, Some(true), "{result:?}");
assert_no_image(&result);
let structured = result.structured_content.unwrap();
assert_eq!(structured["ok"], serde_json::json!(true));
assert_eq!(
structured["screenshot"]["file_path"],
serde_json::json!(after)
);
let after_bytes = std::fs::read(&after).unwrap();
assert_eq!(
structured["screenshot"]["bytes"],
serde_json::json!(after_bytes.len())
);
assert_eq!(
structured["screenshot"]["coordinate_height"],
serde_json::json!(48)
);
assert_eq!(events(), "xdotool\ngnome-screenshot\n");
// A failed post-action write keeps the action's ok and reports
// screenshot_error like any failed post-action capture. It says
// nothing about the desktop, so the cached diagnostics stay.
server.diagnostics.report(false).await;
let probes_before = probes.load(std::sync::atomic::Ordering::SeqCst);
if unprivileged {
let click: ClickParams = serde_json::from_value(serde_json::json!({
"x": 10,
"y": 10,
"screenshot_after": "screen",
"screenshot_file_path": locked.join("after.png")
}))
.unwrap();
let result = server.click(Parameters(click)).await.unwrap();
assert_ne!(result.is_error, Some(true));
assert_no_image(&result);
let structured = result.structured_content.unwrap();
assert_eq!(structured["ok"], serde_json::json!(true));
assert!(structured.get("screenshot").is_none());
assert!(structured["screenshot_error"]
.as_str()
.unwrap()
.contains("failed to write screenshot file"));
assert!(structured["message"]
.as_str()
.unwrap()
.contains("Post-action screenshot failed:"));
}
// A bad post-action path is reported without a capture, and an
// existing file is never replaced.
for (bad, expected) in [
(serde_json::json!("after.png"), "must be an absolute path"),
(serde_json::json!(after), "already exists"),
] {
clear();
let click: ClickParams = serde_json::from_value(serde_json::json!({
"x": 10,
"y": 10,
"screenshot_after": true,
"screenshot_file_path": bad
}))
.unwrap();
let result = server.click(Parameters(click)).await.unwrap();
let structured = result.structured_content.unwrap();
assert_eq!(structured["ok"], serde_json::json!(true));
assert!(structured["screenshot_error"]
.as_str()
.unwrap()
.contains(expected));
assert_eq!(events(), "xdotool\n");
}
assert_eq!(std::fs::read(&after).unwrap(), after_bytes);
server.diagnostics.report(false).await;
assert_eq!(
probes.load(std::sync::atomic::Ordering::SeqCst),
probes_before,
"a file output failure keeps the cached diagnostics"
);
std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o700)).unwrap();
}
}
#[cfg(test)]
mod focus_feedback_tests {
use super::*;
#[test]
fn document_focus_warns_without_claiming_text_was_lost() {
// A container is only reported once its subtree was searched and nothing
// inside it holds focus, so the document itself has the keys.
let element = FocusedElementSummary {
role: "document web".into(),
name: Some("Synthetic host".into()),
editable: false,
states: vec!["focused".into()],
is_terminal: false,
};
let feedback = describe_focused_element(&element, true);
assert!(feedback.starts_with("WARNING:"));
assert!(!feedback.contains("went nowhere"));
assert_eq!(
describe_focused_element(&element, false),
"Focused element: document web \"Synthetic host\" (not editable).",
"press_key feedback on a page is unchanged"
);
}
#[test]
fn editable_state_is_enough_without_editable_text_interface() {
let element = FocusedElementSummary {
role: "entry".into(),
name: Some("Synthetic composer".into()),
editable: false,
states: vec!["focused".into(), "editable".into()],
is_terminal: false,
};
assert!(describe_focused_element(&element, true).contains("(editable)"));
}
#[test]
fn concrete_non_editable_control_keeps_warning_without_claiming_delivery() {
let element = FocusedElementSummary {
role: "button".into(),
name: None,
editable: false,
states: vec!["focused".into()],
is_terminal: false,
};
let feedback = describe_focused_element(&element, true);
assert!(feedback.starts_with("WARNING:"));
assert!(feedback.contains("not editable"));
assert!(!feedback.contains("went nowhere"));
}
}
#[cfg(test)]
mod tree_spill_server_tests {
use super::*;
use std::os::unix::fs::DirBuilderExt;
fn node(index: u32) -> AccessibilityNode {
AccessibilityNode {
index,
parent_index: index.checked_sub(1),
depth: 1,
object_ref: format!(":1.1/org/a11y/atspi/accessible/{index}"),
role: "push button".into(),
name: Some(format!("Button {index} {}", "x".repeat(300))),
description: None,
child_count: 0,
bounds: Some(Bounds {
x: 10,
y: i32::try_from(index).unwrap() * 20,
width: 80,
height: 18,
}),
states: vec!["enabled".into(), "visible".into(), "showing".into()],
actions: Vec::new(),
value: None,
text: None,
supports_editable_text: false,
}
}
fn backend(name: &str) -> (ComputerUseLinux, std::path::PathBuf) {
let root =
std::env::temp_dir().join(format!("cul-spill-server-{name}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&root);
std::fs::DirBuilder::new()
.recursive(true)
.mode(0o700)
.create(&root)
.unwrap();
let spiller = crate::tree_spill::TreeSpiller::new(Some(root.join("cul").join("trees")));
(
ComputerUseLinux {
tree_spiller: Arc::new(spiller),
..ComputerUseLinux::default()
},
root,
)
}
#[test]
fn an_index_past_the_inline_prefix_resolves_against_the_cached_tree() {
let (backend, root) = backend("index");
let tree: Vec<_> = (0..300).map(node).collect();
backend.cache_snapshot(&tree, None);
let mut inline = TreeNodes::new(tree.clone(), NodeDetail::Full);
let (spill, note) = fit_tree(&backend.tree_spiller, &mut inline, tree_budget(2_000));
let spill = spill.expect("a 300-node tree spills");
assert!(note.unwrap().contains("get_tree_page"));
assert!(
inline.len() < 250,
"inline prefix is short: {}",
inline.len()
);
// The last node is only in the file, yet element_index actions still find it.
assert_eq!(
backend.center_for_cached_node(299),
Some((50, 299 * 20 + 9))
);
assert!(std::path::Path::new(&spill.path).exists());
// The next snapshot, even a small one, removes the file it no longer matches.
backend.cache_snapshot(&tree[..3], None);
assert!(!std::path::Path::new(&spill.path).exists());
let _ = std::fs::remove_dir_all(root);
}
#[test]
fn tree_pages_cover_the_whole_tree_in_order() {
let tree: Vec<_> = (0..300).map(node).collect();
let mut seen = Vec::new();
let mut offset = 0;
loop {
let page = tree_page(&tree, NodeDetail::Full, offset);
assert!(!page.nodes.is_empty());
assert!(
serde_json::to_string(&page).unwrap().len()
<= crate::tree_spill::INLINE_RESULT_BYTES
);
assert_eq!(page.node_count, 300);
seen.extend(page.nodes.iter().map(|node| node.index));
match page.next_offset {
Some(next) => offset = next,
None => break,
}
}
assert_eq!(seen, (0..300).collect::<Vec<_>>());
assert!(tree_page(&tree, NodeDetail::Full, 999).nodes.is_empty());
assert!(tree_page(&[], NodeDetail::Full, 0)
.message
.contains("call get_app_state"));
}
#[test]
fn a_page_holds_one_node_even_past_the_budget() {
let mut huge = node(0);
huge.name = Some("x".repeat(crate::tree_spill::INLINE_RESULT_BYTES * 2));
let page = tree_page(&[huge, node(1)], NodeDetail::Full, 0);
assert_eq!(page.nodes.len(), 1);
assert_eq!(page.next_offset, Some(1));
}
}
#[cfg(test)]
mod pixelflux_routing_tests {
use super::pixelflux_point_for_element;
use crate::remote_desktop::LogicalMonitor;
use std::os::unix::fs::PermissionsExt;
#[tokio::test]
async fn cancelled_pixelflux_typing_keeps_input_locked() {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
async fn read_request(stream: &mut tokio::net::TcpStream) {
let mut request = Vec::new();
let mut bytes = [0; 4096];
loop {
let n = stream.read(&mut bytes).await.unwrap();
assert!(n > 0, "fixture request ended early");
request.extend_from_slice(&bytes[..n]);
if let Some(end) = request.windows(4).position(|part| part == b"\r\n\r\n") {
let headers = String::from_utf8_lossy(&request[..end]);
let length: usize = headers
.lines()
.find_map(|line| line.strip_prefix("Content-Length: "))
.unwrap()
.parse()
.unwrap();
if request.len() >= end + 4 + length {
return;
}
}
}
}
const TEST: &str =
"server::pixelflux_routing_tests::cancelled_pixelflux_typing_keeps_input_locked";
if std::env::var_os("CUL_CANCEL_PIXELFLUX_CHILD").is_none() {
let root = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("target/pixelflux-cancel-fixture");
std::fs::create_dir_all(&root).unwrap();
let systemctl = root.join("systemctl");
std::fs::write(&systemctl, "#!/bin/sh\nexit 0\n").unwrap();
std::fs::set_permissions(&systemctl, std::fs::Permissions::from_mode(0o700)).unwrap();
let status = std::process::Command::new(std::env::current_exe().unwrap())
.args(["--exact", TEST, "--nocapture"])
.env_clear()
.env("CUL_CANCEL_PIXELFLUX_CHILD", "1")
.env("PATH", &root)
.env("HOME", &root)
.env("XDG_RUNTIME_DIR", &root)
.env(
"DBUS_SESSION_BUS_ADDRESS",
format!("unix:path={}/no-bus", root.display()),
)
.env("DISPLAY", ":9876")
.env("WAYLAND_DISPLAY", "no-wayland")
.env("XDG_CURRENT_DESKTOP", "fixture")
.env("XDG_SESSION_TYPE", "wayland")
.env("COMPUTER_USE_LINUX_FORCE_YDOTOOL_POINTER", "1")
.env("COMPUTER_USE_LINUX_FORCE_YDOTOOL_KEYBOARD", "1")
.env("CU_DISABLE_ABS_POINTER", "1")
.status()
.unwrap();
assert!(
status.success(),
"cancelled pixelflux request unlocked input"
);
return;
}
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
std::env::set_var(
"PIXELFLUX_CU",
listener.local_addr().unwrap().port().to_string(),
);
std::env::set_var("PIXELFLUX_CU_TOKEN", "fixture-token");
let (sent, started) = tokio::sync::oneshot::channel();
let (finish, finished) = tokio::sync::oneshot::channel();
let peer = tokio::spawn(async move {
let reply = b"HTTP/1.1 200 OK\r\nContent-Length: 15\r\n\r\n{\"result\":\"ok\"}";
let (mut probe, _) = listener.accept().await.unwrap();
read_request(&mut probe).await;
probe.write_all(reply).await.unwrap();
drop(probe);
let (mut action, _) = listener.accept().await.unwrap();
read_request(&mut action).await;
sent.send(()).unwrap();
finished.await.unwrap();
action.write_all(reply).await.unwrap();
});
let server = super::ComputerUseLinux::default();
let input_lock = std::sync::Arc::clone(&server.input_operation_lock);
let caller = tokio::spawn(async move {
let params =
serde_json::from_value(serde_json::json!({"text": "fixture typing"})).unwrap();
server.dispatch_type_text(params).await
});
tokio::time::timeout(std::time::Duration::from_secs(5), started)
.await
.unwrap()
.unwrap();
caller.abort();
let _ = caller.await;
let still_locked = input_lock.try_lock().is_err();
finish.send(()).unwrap();
tokio::time::timeout(std::time::Duration::from_secs(2), peer)
.await
.unwrap()
.unwrap();
let _guard = tokio::time::timeout(std::time::Duration::from_secs(2), input_lock.lock())
.await
.unwrap();
assert!(
still_locked,
"typing continued after its operation lock was released"
);
}
fn monitor(width: i32, height: i32) -> LogicalMonitor {
LogicalMonitor {
x: 0,
y: 0,
width,
height,
scale: 2.0,
}
}
#[test]
fn pixelflux_point_for_element_scales() {
let bounds = (166, 714, 54, 46);
let layout = [monitor(1432, 768)];
assert_eq!(
pixelflux_point_for_element(bounds, (2864, 1536), &layout),
Some((386, 1474))
);
assert_eq!(
pixelflux_point_for_element(bounds, (1432, 768), &layout),
Some((193, 737))
);
assert_eq!(
pixelflux_point_for_element(bounds, (2864, 1000), &layout),
None
);
assert_eq!(pixelflux_point_for_element(bounds, (2864, 1536), &[]), None);
}
#[test]
fn pixelflux_click_uses_new_geometry_after_resize() {
let server = super::ComputerUseLinux::default();
let bounds = (166, 714, 54, 46);
server.cache_desktop_size(2864, 1536);
server.cache_desktop_size(1428, 1536);
let capture = server.desktop_size.lock().unwrap().unwrap();
let layout = [monitor(714, 768)];
assert_eq!(
pixelflux_point_for_element(bounds, capture, &layout),
Some((386, 1474))
);
assert_eq!(
pixelflux_point_for_element(bounds, (2864, 1536), &layout),
None
);
}
/// Clicks in a child process against a fake pixelflux that answers the probe, then
/// `action_reply` (or hangs up after reading the action when `None`).
/// Returns the click message and what a recording `ydotool` received.
fn click_after_probe(
test: &str,
mode: &str,
action_reply: Option<&'static str>,
) -> (String, String) {
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let port = listener.local_addr().unwrap().port();
std::thread::spawn(move || {
let ok = "HTTP/1.1 200 X\r\nContent-Length: 18\r\nConnection: close\r\n\r\n{\"text\":\"X=0,Y=0\"}";
for reply in [Some(ok), action_reply] {
let (mut sock, _) = listener.accept().unwrap();
let mut buf = [0u8; 2048];
let _ = std::io::Read::read(&mut sock, &mut buf);
if let Some(reply) = reply {
let _ = std::io::Write::write_all(&mut sock, reply.as_bytes());
}
}
});
let dir = std::env::temp_dir().join(format!("cul-pf-{mode}-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let log = dir.join("ydotool.log");
let out = dir.join("out.txt");
let ydotool = dir.join("ydotool");
std::fs::write(
&ydotool,
"#!/bin/sh\nprintf '%s\\n' \"$*\" >> \"$CUL_PIXELFLUX_LOG\"\nexit 0\n",
)
.unwrap();
std::fs::set_permissions(&ydotool, std::fs::Permissions::from_mode(0o700)).unwrap();
let systemctl = dir.join("systemctl");
std::fs::write(&systemctl, "#!/bin/sh\nexit 0\n").unwrap();
std::fs::set_permissions(&systemctl, std::fs::Permissions::from_mode(0o700)).unwrap();
let status = std::process::Command::new(std::env::current_exe().unwrap())
.args(["--exact", test, "--nocapture"])
.env_clear()
.env("PATH", &dir)
.env("HOME", &dir)
.env("XDG_RUNTIME_DIR", &dir)
.env(
"DBUS_SESSION_BUS_ADDRESS",
format!("unix:path={}/no-bus", dir.display()),
)
.env("DISPLAY", ":9876")
.env("WAYLAND_DISPLAY", "no-wayland")
.env("XDG_CURRENT_DESKTOP", "fixture")
.env("XDG_SESSION_TYPE", "wayland")
.env("COMPUTER_USE_LINUX_FORCE_YDOTOOL_POINTER", "1")
.env("CUL_PIXELFLUX_CHILD", mode)
.env("CUL_PIXELFLUX_LOG", &log)
.env("CUL_PIXELFLUX_OUT", &out)
.env("PIXELFLUX_CU", port.to_string())
.env("PIXELFLUX_CU_TOKEN", "secret")
.env("CU_DISABLE_ABS_POINTER", "1")
.status()
.unwrap();
assert!(status.success(), "{status}");
let message = std::fs::read_to_string(&out).unwrap();
let recorded = std::fs::read_to_string(&log).unwrap_or_default();
let _ = std::fs::remove_dir_all(&dir);
(message, recorded)
}
async fn record_click_message() {
for (button, count) in [("right", 2), ("middle", 3)] {
let guard = std::sync::Arc::new(
std::sync::Arc::new(tokio::sync::Mutex::new(()))
.lock_owned()
.await,
);
assert!(super::ComputerUseLinux::default()
.try_pixelflux_click(10, 20, false, None, Some(button), count, None, guard)
.await
.is_none());
}
if std::env::var("CUL_PIXELFLUX_CHILD").as_deref() == Ok("scroll") {
let params = serde_json::from_value(serde_json::json!({"direction": "down"})).unwrap();
let result = super::ComputerUseLinux::default()
.scroll(rmcp::handler::server::wrapper::Parameters(params))
.await
.unwrap();
std::fs::write(
std::env::var("CUL_PIXELFLUX_OUT").unwrap(),
result.structured_content.unwrap()["message"]
.as_str()
.unwrap(),
)
.unwrap();
return;
}
let params = serde_json::from_value(serde_json::json!({"x": 10, "y": 20})).unwrap();
let result = super::ComputerUseLinux::default()
.click(rmcp::handler::server::wrapper::Parameters(params))
.await
.unwrap();
let message = result.structured_content.unwrap()["message"]
.as_str()
.unwrap()
.to_string();
std::fs::write(std::env::var("CUL_PIXELFLUX_OUT").unwrap(), message).unwrap();
}
#[tokio::test]
async fn pixelflux_untargeted_scroll_uses_current_cursor() {
if std::env::var("CUL_PIXELFLUX_CHILD").as_deref() == Ok("scroll") {
return record_click_message().await;
}
let (message, recorded) = click_after_probe(
"server::pixelflux_routing_tests::pixelflux_untargeted_scroll_uses_current_cursor",
"scroll",
Some("HTTP/1.1 200 X\r\nContent-Length: 15\r\n\r\n{\"result\":\"ok\"}"),
);
assert!(
message.contains("Action sent through pixelflux"),
"{message}"
);
assert!(recorded.is_empty(), "{recorded}");
}
#[tokio::test]
async fn pixelflux_malformed_reply_reports_unknown_outcome() {
if std::env::var("CUL_PIXELFLUX_CHILD").as_deref() == Ok("malformed") {
return record_click_message().await;
}
let (message, recorded) = click_after_probe(
"server::pixelflux_routing_tests::pixelflux_malformed_reply_reports_unknown_outcome",
"malformed",
Some("HTTP/1.1 200 X\r\nContent-Length: 4\r\n\r\noops"),
);
assert!(message.contains("it may have run"), "{message}");
assert!(!message.contains("did not accept"), "{message}");
assert!(recorded.is_empty(), "{recorded}");
}
#[tokio::test]
async fn pixelflux_failure_after_send_is_not_replayed() {
if std::env::var("CUL_PIXELFLUX_CHILD").as_deref() == Ok("500") {
return record_click_message().await;
}
let (message, recorded) = click_after_probe(
"server::pixelflux_routing_tests::pixelflux_failure_after_send_is_not_replayed",
"500",
Some("HTTP/1.1 500 X\r\nContent-Length: 14\r\nConnection: close\r\n\r\n{\"error\":\"no\"}"),
);
assert!(message.contains("pixelflux did not accept"), "{message}");
assert!(!message.contains("secret"), "{message}");
assert!(recorded.is_empty(), "{recorded}");
}
#[tokio::test]
async fn pixelflux_hangup_after_send_reports_unknown_outcome() {
if std::env::var("CUL_PIXELFLUX_CHILD").as_deref() == Ok("hangup") {
return record_click_message().await;
}
let (message, recorded) = click_after_probe(
"server::pixelflux_routing_tests::pixelflux_hangup_after_send_reports_unknown_outcome",
"hangup",
None,
);
assert!(message.contains("it may have run"), "{message}");
assert!(!message.contains("did not accept"), "{message}");
assert!(recorded.is_empty(), "{recorded}");
}
#[tokio::test]
async fn pixelflux_probe_failure_falls_through() {
if std::env::var("CUL_PIXELFLUX_CHILD").as_deref() == Ok("down") {
let server = super::ComputerUseLinux::default();
let params = serde_json::from_value(serde_json::json!({"x": 10, "y": 20})).unwrap();
let without = server
.click(rmcp::handler::server::wrapper::Parameters(params))
.await
.unwrap();
std::env::set_var("PIXELFLUX_CU", "1");
std::env::set_var("PIXELFLUX_CU_TOKEN", "secret");
let params = serde_json::from_value(serde_json::json!({"x": 10, "y": 20})).unwrap();
let with = server
.click(rmcp::handler::server::wrapper::Parameters(params))
.await
.unwrap();
let left = without.structured_content.unwrap()["message"].to_string();
let right = with.structured_content.unwrap()["message"].to_string();
assert_eq!(left, right, "unreachable pixelflux changed the fallback");
assert!(!right.contains("secret"));
return;
}
let status = std::process::Command::new(std::env::current_exe().unwrap())
.args([
"--exact",
"server::pixelflux_routing_tests::pixelflux_probe_failure_falls_through",
"--nocapture",
])
.env_clear()
.env("PATH", "/usr/bin:/bin")
.env("CUL_PIXELFLUX_CHILD", "down")
.env("CU_DISABLE_ABS_POINTER", "1")
.env("DBUS_SESSION_BUS_ADDRESS", "unix:path=/tmp/cul-no-bus")
.env("XDG_SESSION_TYPE", "tty")
.status()
.unwrap();
assert!(status.success(), "{status}");
}
}
#[cfg(test)]
mod snapshot_handle_tests {
use super::*;
const FIRST_REF: &str = ":1.10/org/a11y/atspi/accessible/7";
const SECOND_REF: &str = ":1.20/org/a11y/atspi/accessible/7";
const IDENTIFIER_REF: &str = ":1.30/org/a11y/atspi/accessible/9";
fn node(object_ref: &str) -> AccessibilityNode {
AccessibilityNode {
index: 0,
parent_index: None,
depth: 1,
object_ref: object_ref.to_string(),
role: "push button".to_string(),
name: Some("Save".to_string()),
description: None,
child_count: 0,
bounds: None,
states: vec!["enabled".to_string()],
actions: vec![AccessibilityAction {
index: 0,
name: "click".to_string(),
description: String::new(),
keybinding: String::new(),
}],
value: None,
text: None,
supports_editable_text: false,
}
}
fn invocation() -> crate::atspi_tree_impl::ActionInvocation {
crate::atspi_tree_impl::ActionInvocation {
action_index: 0,
action_name: Some("click".to_string()),
ok: true,
}
}
#[test]
fn every_store_gets_a_new_id_and_servers_do_not_share_ids() {
let backend = ComputerUseLinux::default();
assert_eq!(backend.cached_snapshot_id(), "");
let first = backend.cache_snapshot(&[node(FIRST_REF)], None);
let second = backend.cache_snapshot(&[node(FIRST_REF)], None);
assert_ne!(first, second);
assert_eq!(backend.cached_snapshot_id(), second);
let (prefix, _) = first.split_once('-').unwrap();
assert!(second.starts_with(&format!("{prefix}-")), "{second}");
// A server started later, or another server, names its own snapshots.
let other = ComputerUseLinux::default().cache_snapshot(&[node(FIRST_REF)], None);
assert!(!other.starts_with(&format!("{prefix}-")), "{other}");
}
#[test]
fn snapshot_id_check() {
let check =
|requested: Option<&str>, current: &str| check_snapshot_id(requested, current, "ab");
assert_eq!(check(None, "ab-2"), Ok(()));
assert_eq!(check(Some(" "), "ab-2"), Ok(()));
assert_eq!(check(Some("ab-2"), "ab-2"), Ok(()));
assert_eq!(check(Some(" ab-2 "), "ab-2"), Ok(()));
assert_eq!(
check(Some("ab-1"), "ab-2"),
Err("snapshot_id \"ab-1\" is not the cached snapshot: a later get_app_state, include_tree or perform_action recovery replaced it, and its element indices could name other elements now. Call get_app_state and use the element_index values and snapshot_id it returns.".to_string())
);
let reason = |requested: &str, current: &str| {
let message = check(Some(requested), current).unwrap_err();
let (_, reason) = message.split_once("is not the cached snapshot: ").unwrap();
reason.split_once(", and its").unwrap().0.to_string()
};
assert_eq!(
reason("ab-1", ""),
"no tree is cached now, since the last tree read failed or perform_action recovery cleared it"
);
let foreign =
"this server did not issue it; it may come from a server process that has since exited";
assert_eq!(reason("cd-1", "ab-1"), foreign);
assert_eq!(reason("cd-1", ""), foreign);
assert_eq!(reason("ab1", "ab-1"), foreign);
}
#[test]
fn only_calls_that_resolve_a_cached_element_name_a_snapshot() {
let click =
|value: serde_json::Value| -> ClickParams { serde_json::from_value(value).unwrap() };
let scroll = |value: serde_json::Value| -> ScrollParams {
let mut value = value;
value["direction"] = "down".into();
serde_json::from_value(value).unwrap()
};
use serde_json::json;
assert!(click_resolves_cached_element(&click(
json!({"element_index": 1})
)));
assert!(click_resolves_cached_element(&click(
json!({"role": "button"})
)));
// A lone x falls back to the element, as the click target resolves it.
assert!(click_resolves_cached_element(&click(
json!({"element_index": 1, "x": 5})
)));
assert!(click_resolves_cached_element(&click(
json!({"element_index": 1, "x": 5, "y": 6, "coordinate_space": "atspi"})
)));
assert!(!click_resolves_cached_element(&click(
json!({"element_index": 1, "x": 5, "y": 6})
)));
assert!(!click_resolves_cached_element(&click(
json!({"x": 5, "y": 6})
)));
assert!(scroll_resolves_cached_element(&scroll(
json!({"element_index": 1})
)));
assert!(scroll_resolves_cached_element(&scroll(
json!({"element_index": 1, "x": 5})
)));
assert!(scroll_resolves_cached_element(&scroll(
json!({"element_index": 1, "y": 6})
)));
assert!(!scroll_resolves_cached_element(&scroll(
json!({"element_index": 1, "x": 5, "y": 6})
)));
assert!(!scroll_resolves_cached_element(&scroll(json!({"x": 5}))));
}
/// Two callers share one server and its one cached snapshot. After the
/// second caller's get_app_state, the first caller's index names a node
/// of the second tree; naming the first snapshot refuses it.
#[tokio::test]
async fn an_index_from_a_replaced_snapshot_is_refused() {
let backend = ComputerUseLinux::default();
let first = backend.cache_snapshot(&[node(FIRST_REF)], None);
let second = backend.cache_snapshot(&[node(SECOND_REF)], None);
let dispatched = Arc::new(Mutex::new(Vec::new()));
let run = |params: ActionParams| {
let dispatched = Arc::clone(&dispatched);
let backend = backend.clone();
async move {
backend
.perform_element_action_with(
¶ms,
Some("click"),
move |object_ref, _| {
dispatched.lock().unwrap().push(object_ref);
async { Ok(invocation()) }
},
|_| async { Err(anyhow::anyhow!("no recovery expected")) },
)
.await
.0
.0
}
};
let by_index = |snapshot_id: Option<&str>| ActionParams {
element_index: Some(0),
snapshot_id: snapshot_id.map(str::to_string),
..Default::default()
};
let refused = run(by_index(Some(&first))).await;
assert!(!refused.ok);
assert!(
refused.message.starts_with(&format!(
"perform_action was not sent: snapshot_id {first:?} is not the cached snapshot: a later get_app_state"
)),
"{}",
refused.message
);
// A selector resolves in the cached tree too.
let by_selector = ActionParams {
role: Some("push button".to_string()),
snapshot_id: Some(first.clone()),
..Default::default()
};
assert!(!run(by_selector).await.ok);
assert!(dispatched.lock().unwrap().is_empty());
assert!(run(by_index(Some(&second))).await.ok);
// Without snapshot_id the latest snapshot answers, as before.
assert!(run(by_index(None)).await.ok);
// An element_identifier names the object itself, not a snapshot node.
let by_identifier = ActionParams {
element_identifier: Some(IDENTIFIER_REF.to_string()),
snapshot_id: Some(first.clone()),
..Default::default()
};
assert!(run(by_identifier).await.ok);
assert_eq!(
*dispatched.lock().unwrap(),
[SECOND_REF, SECOND_REF, IDENTIFIER_REF]
);
}
#[tokio::test]
async fn tree_pages_name_their_snapshot_and_refuse_a_replaced_one() {
let backend = ComputerUseLinux::default();
let page = |snapshot_id: Option<&str>| {
let backend = backend.clone();
let snapshot_id = snapshot_id.map(str::to_string);
async move {
backend
.get_tree_page(Parameters(GetTreePageParams {
offset: None,
snapshot_id,
}))
.await
.0
}
};
assert_eq!(page(None).await.snapshot_id, None);
let first = backend.cache_snapshot(&[node(FIRST_REF)], None);
let second = backend.cache_snapshot(&[node(SECOND_REF)], None);
for requested in [None, Some(second.as_str())] {
let current = page(requested).await;
assert_eq!(current.snapshot_id.as_deref(), Some(second.as_str()));
assert_eq!(current.node_count, 1);
assert_eq!(current.nodes.len(), 1);
}
let stale = page(Some(&first)).await;
assert_eq!(stale.snapshot_id, None);
assert_eq!(stale.node_count, 0);
assert!(stale.nodes.is_empty());
assert_eq!(stale.next_offset, None);
assert!(
stale
.message
.starts_with(&format!("snapshot_id {first:?} is not the cached snapshot")),
"{}",
stale.message
);
// A cleared cache names no snapshot and refuses every id.
backend.clear_snapshot();
assert_eq!(backend.cached_snapshot_id(), "");
let cleared = page(None).await;
assert_eq!(cleared.snapshot_id, None);
assert_eq!(cleared.node_count, 0);
let refused = page(Some(&second)).await;
assert!(
refused.message.starts_with(&format!(
"snapshot_id {second:?} is not the cached snapshot: no tree is cached now"
)),
"{}",
refused.message
);
}
const HTTP_REQUIRED: &str = "click was not sent: snapshot_id is required over HTTP for element_index and selector calls, because another client's get_app_state can replace the cached tree. Pass the snapshot_id from your get_app_state result.";
/// The click checks of `http_clients_must_name_their_snapshot`, on any
/// server; returns each call's check result.
fn click_checks(backend: &ComputerUseLinux) -> (String, Vec<std::result::Result<(), String>>) {
let a = backend.clone();
let b = backend.clone();
let first = a.cache_snapshot(&[node(FIRST_REF)], None);
b.cache_snapshot(&[node(SECOND_REF)], None);
let check = |value: serde_json::Value| {
let params: ClickParams = serde_json::from_value(value).unwrap();
a.check_element_snapshot(
"click",
params.snapshot_id.as_deref(),
click_resolves_cached_element(¶ms),
)
.map(drop)
};
use serde_json::json;
let results = vec![
check(json!({"element_index": 0, "snapshot_id": first})),
check(json!({"element_index": 0})),
check(json!({"element_index": 0, "snapshot_id": " "})),
check(json!({"element_identifier": IDENTIFIER_REF})),
check(json!({"x": 5, "y": 6})),
];
(first, results)
}
#[test]
fn http_clients_must_name_their_snapshot() {
let backend = ComputerUseLinux {
require_snapshot_ids: true,
..Default::default()
};
let (first, results) = click_checks(&backend);
assert!(results[0].as_ref().unwrap_err().starts_with(&format!(
"click was not sent: snapshot_id {first:?} is not the cached snapshot: a later get_app_state"
)));
assert_eq!(results[1], Err(HTTP_REQUIRED.to_string()));
assert_eq!(results[2], Err(HTTP_REQUIRED.to_string()));
assert_eq!(results[3], Ok(()));
assert_eq!(results[4], Ok(()));
}
#[test]
fn stdio_snapshot_id_stays_optional() {
let (_, results) = click_checks(&ComputerUseLinux::default());
assert!(results[0].is_err());
assert_eq!(results[1..], [Ok(()), Ok(()), Ok(()), Ok(())]);
}
#[tokio::test]
async fn http_tree_page_needs_a_snapshot_id() {
let backend = ComputerUseLinux {
require_snapshot_ids: true,
..Default::default()
};
let id = backend.cache_snapshot(&[node(FIRST_REF)], None);
for missing in [None, Some(" ".to_string())] {
let page = backend
.get_tree_page(Parameters(GetTreePageParams {
offset: None,
snapshot_id: missing,
}))
.await
.0;
assert_eq!(page.node_count, 0);
assert!(page.nodes.is_empty());
assert_eq!(page.snapshot_id, None);
assert_eq!(
page.message,
HTTP_REQUIRED.replacen("click", "get_tree_page", 1)
);
}
let page = backend
.get_tree_page(Parameters(GetTreePageParams {
offset: None,
snapshot_id: Some(id.clone()),
}))
.await
.0;
assert_eq!(page.node_count, 1);
assert_eq!(page.snapshot_id, Some(id));
}
/// Client B's tree: its app is pid 42 and it was narrowed to "Window 20".
fn cache_client_b_tree(backend: &ComputerUseLinux) -> String {
let id = backend.cache_snapshot(&[node(FIRST_REF)], Some(42));
backend.last_snapshot.lock().unwrap().window_scope = Some("Window 20".to_string());
id
}
#[test]
fn http_post_action_tree_ignores_another_clients_scope() {
let http = ComputerUseLinux {
require_snapshot_ids: true,
..Default::default()
};
let b_id = cache_client_b_tree(&http);
// Client A's action names no snapshot, or not B's: nothing comes from
// B's tree. Coordinate calls resolve no cached element, so they pass.
for named in [None, Some(" "), Some("other")] {
let scope = http.check_element_snapshot("click", named, false).unwrap();
assert_eq!(scope.pid, None);
assert!(scope.tree.is_none(), "{:?}", scope.tree);
}
// Client B named its own snapshot, so its scope carries over.
let scope = http
.check_element_snapshot("click", Some(&format!(" {b_id} ")), false)
.unwrap();
assert_eq!(scope.pid, Some(42));
assert_eq!(scope.tree.unwrap().scope, "Window 20");
// Stdio reads the cached scope at capture time, as before.
let stdio = ComputerUseLinux::default();
cache_client_b_tree(&stdio);
let scope = stdio.cached_post_action_context();
assert_eq!(scope.pid, Some(42));
let window = scope.tree.unwrap();
assert_eq!(window.scope, "Window 20");
assert_eq!(window.pid, Some(42));
}
#[test]
fn post_action_scope_for_capture_keeps_the_carried_scope_over_http() {
// Client B stores a newer tree for another app and window.
let cache_newer_tree = |backend: &ComputerUseLinux| {
backend.cache_snapshot(&[node(SECOND_REF)], Some(7));
backend.last_snapshot.lock().unwrap().window_scope = Some("Window 30".to_string());
};
let http = ComputerUseLinux {
require_snapshot_ids: true,
..Default::default()
};
let a_id = cache_client_b_tree(&http);
// Client A's action takes its scope under the input lock.
let carried = http
.check_element_snapshot("click", Some(&a_id), true)
.unwrap();
cache_newer_tree(&http);
let scope = http.post_action_scope_for_capture(carried);
assert_eq!(scope.pid, Some(42));
assert_eq!(scope.tree.unwrap().scope, "Window 20");
// Stdio reads the cache at capture time, as on main.
let stdio = ComputerUseLinux::default();
let a_id = cache_client_b_tree(&stdio);
let carried = stdio
.check_element_snapshot("click", Some(&a_id), true)
.unwrap();
cache_newer_tree(&stdio);
let scope = stdio.post_action_scope_for_capture(carried);
assert_eq!(scope.pid, Some(7));
assert_eq!(scope.tree.unwrap().scope, "Window 30");
}
#[tokio::test]
async fn http_failed_read_keeps_the_cached_tree() {
let http = ComputerUseLinux {
require_snapshot_ids: true,
..Default::default()
};
let id = http.cache_snapshot(&[node(FIRST_REF)], None);
http.commit_cleared_snapshot().await;
assert!(http
.check_element_snapshot("click", Some(&id), true)
.is_ok());
let stdio = ComputerUseLinux::default();
let id = stdio.cache_snapshot(&[node(FIRST_REF)], None);
stdio.commit_cleared_snapshot().await;
let refused = stdio
.check_element_snapshot("click", Some(&id), true)
.unwrap_err();
assert!(refused.contains("no tree is cached now"), "{refused}");
}
}
#[cfg(test)]
mod type_text_atspi_fallback_tests {
use super::*;
use crate::atspi_tree_impl::insert_fixture::{
start_field_fixture, FieldFixture, FIXTURE_WINDOW,
};
use crate::keyboard_keymap::{MissingKeysymReason, MissingLayoutKeysym};
const REFUSAL: &str =
"Literal text was not sent: GNOME portal cannot verify U+002F in keyboard layout group 2.";
const MISSING: MissingLayoutKeysym = MissingLayoutKeysym {
ch: '/',
group: 2,
reason: MissingKeysymReason::Absent,
};
fn result(insert: Result<FocusedTextInsert>) -> std::result::Result<String, String> {
type_text_atspi_result(REFUSAL, MISSING, "/home/x", insert)
}
#[test]
fn a_verified_insert_names_the_route_and_skips_the_unverified_note() {
let message = result(Ok(FocusedTextInsert::Verified {
role: "text".to_string(),
replaced_selection: true,
caret_after_text: true,
}))
.unwrap();
// type_text adds "Field contents were not verified" unless this holds.
assert!(message.starts_with(TYPE_TEXT_ATSPI_ROUTE), "{message}");
assert!(message.contains("group 2 lacks U+002F"), "{message}");
assert!(message.contains("reads back the text"), "{message}");
assert!(message.contains("replaced the selection"), "{message}");
assert!(!message.contains("caret"), "{message}");
}
#[test]
fn no_editable_focus_and_password_fields_keep_the_refusal() {
let error = result(Ok(FocusedTextInsert::NotEditable(
"no element in the target app has keyboard focus".to_string(),
)))
.unwrap_err();
assert!(error.starts_with(REFUSAL), "{error}");
assert!(error.contains("no element in the target app has keyboard focus"));
let error = result(Ok(FocusedTextInsert::PasswordField)).unwrap_err();
assert!(error.starts_with(REFUSAL), "{error}");
assert!(error.contains("password field"), "{error}");
assert!(error.contains("set_value can fill it"), "{error}");
let error = result(Err(anyhow::anyhow!("failed to connect to AT-SPI bus"))).unwrap_err();
assert!(error.starts_with(REFUSAL), "{error}");
}
#[test]
fn a_readback_mismatch_fails_and_quotes_the_field() {
let error = result(Ok(FocusedTextInsert::Mismatch {
expected_count: 7,
read_count: 6,
read_text: "/home/".to_string(),
read_truncated: false,
}))
.unwrap_err();
// A failure must not carry the success marker.
assert!(error.starts_with(TYPE_TEXT_ATSPI_UNVERIFIED), "{error}");
assert!(!error.starts_with(TYPE_TEXT_ATSPI_ROUTE), "{error}");
assert!(error.contains("does not read it back"), "{error}");
assert!(error.contains("\"/home/x\""), "{error}");
assert!(error.contains("read \"/home/\""), "{error}");
assert!(error.contains("may have changed"), "{error}");
let error = result(Ok(FocusedTextInsert::EditFailed {
reason: "the app refused InsertText".to_string(),
may_have_changed: false,
}))
.unwrap_err();
assert!(error.starts_with(TYPE_TEXT_ATSPI_UNVERIFIED), "{error}");
assert!(error.contains("No text was changed"), "{error}");
for outcome in [
FocusedTextInsert::ReadbackFailed,
FocusedTextInsert::ReadbackTimedOut,
] {
let error = result(Ok(outcome)).unwrap_err();
assert!(error.starts_with(TYPE_TEXT_ATSPI_UNVERIFIED), "{error}");
}
// An unreachable shift level is not worded as a missing character.
let unreachable = MissingLayoutKeysym {
reason: MissingKeysymReason::UnreachableLevel,
..MISSING
};
let error = type_text_atspi_result(
REFUSAL,
unreachable,
"/home/x",
Ok(FocusedTextInsert::ReadbackFailed),
)
.unwrap_err();
assert!(error.contains("only at a shift level"), "{error}");
assert!(!error.contains("lacks"), "{error}");
}
#[test]
fn only_a_missing_keysym_takes_the_atspi_route() {
let missing = anyhow::Error::new(MISSING).context("keyboard keymap preflight failed");
assert_eq!(atspi_fallback_trigger(&missing), Some(MISSING));
for error in [
anyhow::anyhow!("WAYLAND_DISPLAY is missing; cannot verify the keyboard keymap"),
anyhow::anyhow!("keyboard keymap read timed out; no text was sent"),
anyhow::anyhow!("cannot verify the complete text against the keyboard keymap"),
] {
assert_eq!(atspi_fallback_trigger(&error), None, "{error}");
}
}
/// The issue's case: with us + il groups the portal preflight refuses
/// `/home/x`, and the text goes into the focused entry and reads back.
#[tokio::test]
async fn a_two_group_keymap_refusal_types_through_atspi() {
let keymap = include_bytes!("../tests/fixtures/keyboard-us-il.xkb");
assert!(crate::keyboard_keymap::validate_text_with_keymap(keymap, "/").is_ok());
let error =
crate::keyboard_keymap::validate_text_with_keymap(keymap, "/home/x").unwrap_err();
let refusal = format!("Literal text was not sent: {error:#}");
let missing = atspi_fallback_trigger(&error).expect("a layout refusal");
assert_eq!(
missing,
MissingLayoutKeysym {
ch: 'h',
group: 2,
reason: MissingKeysymReason::Absent,
}
);
let running = start_field_fixture(FieldFixture {
role: atspi::Role::Entry,
role_name: "entry",
..Default::default()
})
.await;
let insert = crate::atspi_tree_impl::insert_text_at_focus_on(
&running.conn,
std::process::id(),
Some(FIXTURE_WINDOW),
"/home/x",
)
.await;
let message = type_text_atspi_result(&refusal, missing, "/home/x", insert).unwrap();
assert!(message.starts_with(TYPE_TEXT_ATSPI_ROUTE), "{message}");
assert!(message.contains("group 2 lacks U+0068"), "{message}");
assert_eq!(running.take_state(0).contents, "/home/x");
}
#[test]
fn control_characters_are_not_inserted() {
assert_eq!(key_press_character("/home/x y.txt"), None);
assert_eq!(key_press_character("שלום"), None);
for (text, code) in [
("a\nb", "U+000A"),
("a\tb", "U+0009"),
("a\rb", "U+000D"),
("a\u{8}", "U+0008"),
("\u{1b}", "U+001B"),
("\u{7f}", "U+007F"),
] {
let key = key_press_character(text).unwrap();
let error = type_text_control_character_refusal(REFUSAL, key);
assert!(error.starts_with(REFUSAL), "{error}");
assert!(error.contains(code), "{error}");
assert!(error.contains("press_key"), "{error}");
}
}
}