use crate::atspi_tree::{
accessibility_chrome_context, focused_element_summary_in_app, list_accessible_apps,
object_ref_owner_pid, perform_action as invoke_accessibility_action, perform_named_action,
probe_focused_element, set_element_value, snapshot_accessibility_tree, AccessibilityAction,
AccessibilityNode, AccessibleAppSummary, Bounds, FocusProbe, FocusedElementSummary,
ValueSetInvocation,
};
use crate::diagnostics::{doctor_report, setup_accessibility_report, DoctorReport, SetupReport};
use crate::gnome_extension::{setup_window_targeting_report, WindowTargetingSetupReport};
use crate::remote_desktop::{
click as portal_click, drag as portal_drag, keysyms_for_text, press_key_chord,
press_keycode_chord, scroll as portal_scroll, start_portal_keyboard_session,
start_portal_pointer_session, type_text_with_keysyms, PointerButton, PortalKey,
PortalKeyboardSession, PortalPointerSession, ScrollDirection,
};
use crate::screenshot::{
capture_screenshot_raw, prepare_screenshot_payload, RawScreenshotCapture, ScreenshotCapture,
ScreenshotOutputFormat, ScreenshotPayloadOptions,
};
use crate::terminal::{terminal_paste_shortcut, TerminalPasteShortcut};
use crate::windowing::registry;
use crate::windows::{
focus_window_target, focused_window, list_windows, resolve_window_target,
window_permission_hint, WindowFocusResult, WindowInfo, WindowTarget,
GNOME_SHELL_EXTENSION_BACKEND, GNOME_SHELL_INTROSPECT_BACKEND, KWIN_BACKEND,
};
use crate::ydotool;
use anyhow::{Context, Result};
use rmcp::{
handler::server::wrapper::{Json, Parameters},
model::{CallToolResult, ContentBlock},
schemars::JsonSchema,
tool, tool_handler, tool_router, ErrorData, ServerHandler, ServiceExt,
};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::{
collections::BTreeMap,
env,
ffi::OsString,
future::Future,
path::Path,
process::{Command, Output, Stdio},
sync::{Arc, Mutex},
time::Duration,
};
use tokio::{
process::Command as TokioCommand,
time::{sleep, timeout},
};
use xkeysym::key as xkey;
use zbus::{Connection as ZbusConnection, Proxy as ZbusProxy};
const INPUT_COMMAND_TIMEOUT: Duration = Duration::from_secs(10);
// Linux limits each exec argument (including its NUL) to 32 pages, at least
// 128 KiB on supported platforms. Reject oversized text before sending input.
const WTYPE_MAX_TEXT_BYTES: usize = 128 * 1024 - 1;
const WTYPE_STDIN_MAX_CHARS: usize = 100;
const WTYPE_FOCUS_TIMEOUT: Duration = Duration::from_millis(1500);
const MAX_SCROLL_PAGES: f64 = 100.0;
const YDOTOOL_TYPE_CHARS_PER_SECOND: u64 = 20;
const KDE_CLIPBOARD_DBUS_TIMEOUT: Duration = Duration::from_secs(3);
const KDE_KLIPPER_SERVICE: &str = "org.kde.klipper";
const KDE_KLIPPER_PATH: &str = "/klipper";
const KDE_KLIPPER_INTERFACE: &str = "org.kde.klipper.klipper";
const SHELL_ENABLE_ENV: &str = "COMPUTER_USE_LINUX_ENABLE_SHELL";
const COMPLETION_ENABLE_ENV: &str = "COMPUTER_USE_LINUX_NOTIFY_ON_COMPLETE";
const SHELL_DEFAULT_TIMEOUT_SECS: u64 = 30;
const SHELL_MAX_TIMEOUT_SECS: u64 = 120;
const SHELL_MAX_COMMAND_BYTES: usize = 64 * 1024;
const SHELL_MAX_CWD_BYTES: usize = 4096;
const SHELL_MAX_ENV_ENTRIES: usize = 64;
const SHELL_MAX_ENV_BYTES: usize = 64 * 1024;
const SHELL_RESPONSE_STREAM_BYTES: usize = 512 * 1024;
/// Nodes and their app ownership must come from the same snapshot.
#[derive(Default)]
struct CachedAccessibilitySnapshot {
nodes: Vec<AccessibilityNode>,
pid: Option<u32>,
}
#[derive(Clone, Default)]
pub struct ComputerUseLinux {
last_snapshot: Arc<Mutex<CachedAccessibilitySnapshot>>,
portal_pointer_session: Arc<Mutex<Option<PortalPointerSession>>>,
portal_keyboard_session: Arc<Mutex<Option<PortalKeyboardSession>>>,
/// Lazily-created uinput absolute pointer (preferred coordinate backend).
abs_pointer: Arc<Mutex<Option<crate::abs_pointer::AbsPointer>>>,
portal_session_init_lock: Arc<tokio::sync::Mutex<()>>,
input_operation_lock: Arc<tokio::sync::Mutex<()>>,
kde_clipboard_lock: Arc<tokio::sync::Mutex<()>>,
/// Cached physical desktop size from the most recent full-frame capture;
/// used for off-screen warnings and portal logical-coordinate mapping.
desktop_size: Arc<Mutex<Option<(u32, u32)>>>,
/// On-screen activity overlay (opt-out via `COMPUTER_USE_LINUX_INDICATOR=0`).
indicator: Arc<crate::indicator::Indicator>,
}
fn sanitize_unsigned_integer_formats(value: &mut serde_json::Value) {
let serde_json::Value::Object(object) = value else {
return;
};
let has_unsigned_format = matches!(
object.get("format").and_then(serde_json::Value::as_str),
Some("uint" | "uint8" | "uint16" | "uint32" | "uint64" | "usize")
);
if has_unsigned_format {
object.remove("format");
}
for nested in object.values_mut() {
match nested {
serde_json::Value::Object(_) => sanitize_unsigned_integer_formats(nested),
serde_json::Value::Array(items) => {
for item in items {
sanitize_unsigned_integer_formats(item);
}
}
_ => {}
}
}
}
impl ComputerUseLinux {
fn mcp_tool_router(&self) -> rmcp::handler::server::router::tool::ToolRouter<Self> {
self.router_with_completion(env::var(COMPLETION_ENABLE_ENV).as_deref() == Ok("1"))
}
fn router_with_completion(
&self,
enabled: bool,
) -> rmcp::handler::server::router::tool::ToolRouter<Self> {
let mut router = Self::tool_router();
if !enabled {
router.map.remove("complete_interaction");
}
if !shell_execution_enabled() {
router.map.remove("run_shell");
}
for route in router.map.values_mut() {
let input_schema = Arc::make_mut(&mut route.attr.input_schema);
for value in input_schema.values_mut() {
sanitize_unsigned_integer_formats(value);
}
if let Some(output_schema) = route.attr.output_schema.as_mut() {
for value in Arc::make_mut(output_schema).values_mut() {
sanitize_unsigned_integer_formats(value);
}
}
}
router
}
}
#[tool_router]
impl ComputerUseLinux {
#[tool(
name = "complete_interaction",
description = "Send a desktop notification that this agent has finished its interaction. Available only with COMPUTER_USE_LINUX_NOTIFY_ON_COMPLETE=1. This cue does not acquire or release an exclusive desktop lock. Delivery is best effort and bounded; it may be suppressed by desktop notification settings.",
annotations(
read_only_hint = false,
destructive_hint = false,
idempotent_hint = false,
open_world_hint = true
)
)]
async fn complete_interaction(&self) -> Json<CompletionOutput> {
if env::var(COMPLETION_ENABLE_ENV).as_deref() != Ok("1") {
return Json(CompletionOutput {
ok: true,
cue: "skipped".into(),
note: "Completion notifications are disabled.".into(),
});
}
Json(send_completion_notification(Path::new("notify-send"), Duration::from_secs(2)).await)
}
#[tool(
name = "doctor",
description = "Report Linux Computer Use desktop integration readiness.",
annotations(
read_only_hint = true,
destructive_hint = false,
idempotent_hint = true,
open_world_hint = false
)
)]
async fn doctor(&self) -> Json<DoctorReport> {
Json(
tokio::task::spawn_blocking(doctor_report)
.await
.expect("diagnostics task panicked"),
)
}
#[tool(
name = "setup_accessibility",
description = "Enable GNOME accessibility through gsettings so Linux Computer Use can read AT-SPI trees.",
annotations(
read_only_hint = false,
destructive_hint = false,
idempotent_hint = true,
open_world_hint = false
)
)]
async fn setup_accessibility(&self) -> Json<SetupReport> {
Json(
tokio::task::spawn_blocking(setup_accessibility_report)
.await
.expect("accessibility setup task panicked"),
)
}
#[tool(
name = "setup_window_targeting",
description = "Install and enable the optional GNOME Shell extension used for exact window list/focus targeting when GNOME blocks native introspection.",
annotations(
read_only_hint = false,
destructive_hint = false,
idempotent_hint = true,
open_world_hint = false
)
)]
async fn setup_window_targeting(&self) -> Json<WindowTargetingSetupReport> {
Json(setup_window_targeting_report().await)
}
#[tool(
name = "list_apps",
description = "List running Linux desktop app candidates visible to the Computer Use backend.",
annotations(
read_only_hint = true,
destructive_hint = false,
idempotent_hint = true,
open_world_hint = true
)
)]
async fn list_apps(&self) -> Json<ListAppsOutput> {
let (accessible_apps, accessibility_error) = match list_accessible_apps(50).await {
Ok(apps) => (apps, None),
Err(error) => (Vec::new(), Some(format!("{error:#}"))),
};
Json(ListAppsOutput {
apps: list_process_apps(),
accessible_apps,
accessibility_error,
note: "Linux Computer Use lists process candidates plus AT-SPI application roots when accessibility is enabled.".to_string(),
})
}
#[tool(
name = "list_windows",
description = "List compositor windows with title, app id, class, focus state, client type, and known bounds.",
annotations(
read_only_hint = true,
destructive_hint = false,
idempotent_hint = true,
open_world_hint = true
)
)]
async fn list_windows(&self) -> Json<ListWindowsOutput> {
Json(window_list_output().await)
}
#[tool(
name = "focused_window",
description = "Return the compositor window that currently has keyboard focus.",
annotations(
read_only_hint = true,
destructive_hint = false,
idempotent_hint = true,
open_world_hint = true
)
)]
async fn focused_window(&self) -> Json<FocusedWindowOutput> {
match focused_window().await {
Ok(window) => {
let backend = window_backend(window.as_ref().into_iter());
Json(FocusedWindowOutput {
backend,
focused_window: window,
error: None,
permissions_hint: None,
message:
"Focused window query completed through the available compositor window backend."
.to_string(),
})
}
Err(error) => {
let error = format!("{error:#}");
Json(FocusedWindowOutput {
backend: GNOME_SHELL_INTROSPECT_BACKEND.to_string(),
focused_window: None,
permissions_hint: window_permission_hint(&error),
error: Some(error),
message: "Focused window query failed; targeted keyboard input is unavailable until window introspection works.".to_string(),
})
}
}
}
#[tool(
name = "activate_window",
description = "Focus a Linux desktop window by window_id, pid, app_id, wm_class, title, or terminal selectors when the compositor permits it.",
annotations(
read_only_hint = false,
destructive_hint = false,
idempotent_hint = true,
open_world_hint = true
)
)]
async fn activate_window(
&self,
Parameters(params): Parameters<ActivateWindowParams>,
) -> Json<ActivateWindowOutput> {
let target = params.into_target();
let received = Some(serde_json::json!(target.clone()));
self.indicator.action("activate_window").await;
match focus_window_target(&target).await {
Ok(focus) => {
let ok = focus_satisfies_target(&focus, &target);
Json(ActivateWindowOutput {
ok,
implemented: true,
backend: focus.backend.clone(),
focus: Some(focus),
error: None,
permissions_hint: None,
received,
})
}
Err(error) => {
let error = format!("{error:#}");
Json(ActivateWindowOutput {
ok: false,
implemented: true,
backend: GNOME_SHELL_INTROSPECT_BACKEND.to_string(),
focus: None,
permissions_hint: window_permission_hint(&error),
error: Some(error),
received,
})
}
}
}
#[tool(
name = "get_app_state",
output_schema = rmcp::handler::server::tool::schema_for_output::<GetAppStateOutput>()
.expect("get_app_state output schema"),
description = "Start an app use session if needed, then get a size-bounded screenshot and accessibility state for a Linux app. Scope the accessibility tree with app_name_or_bundle_identifier or a window_id/pid/app_id/wm_class/title target; omitting a target returns the whole desktop tree and can flood context. Screenshot results include coordinate_width, coordinate_height, scale, format, and quality when the returned image is downscaled or compressed; callers can request jpeg/quality for compression before resizing.",
annotations(
read_only_hint = true,
destructive_hint = false,
idempotent_hint = true,
open_world_hint = true
)
)]
async fn get_app_state(
&self,
Parameters(params): Parameters<GetAppStateParams>,
) -> Result<CallToolResult, ErrorData> {
let captures = params.include_screenshot.unwrap_or(true);
let result = self.app_state(params).await;
if captures {
self.indicator.action("get_app_state").await;
}
result
}
async fn app_state(&self, params: GetAppStateParams) -> Result<CallToolResult, ErrorData> {
let verbose = params.verbose.unwrap_or(false);
let mut diagnostics = tokio::task::spawn_blocking(doctor_report)
.await
.expect("diagnostics task panicked");
let (window_context, window_error, window_permissions_hint) =
self.resolve_window_context(¶ms).await;
let (max_nodes, max_depth) =
crate::atspi_tree::snapshot_limits(params.max_nodes, params.max_depth);
let include_screenshot = params.include_screenshot.unwrap_or(true);
let screenshot_options = params.screenshot_options();
let screenshot_target_requested = params.window_target().has_target();
// Whether the caller asked for any scope at all. `tree_scoped` can still
// come back false when a target matched no AT-SPI root, and that case
// needs different advice than "pass a target".
let accessibility_target_requested = screenshot_target_requested
|| params
.app_name_or_bundle_identifier
.as_deref()
.is_some_and(|name| !name.trim().is_empty());
let app_filter = self
.resolve_accessibility_app_filter(¶ms, window_context.as_ref())
.await;
let (screenshot, screenshot_error) = if include_screenshot {
let result: Result<ScreenshotCapture> = async {
let _pipeline = crate::screenshot_impl::screenshot_pipeline_permit().await?;
let raw = {
let _hold = self.indicator.hold_for_capture().await?;
capture_screenshot_raw().await
};
diagnostics.readiness.record_screenshot_result(raw.is_ok());
let raw = raw?;
self.cache_desktop_size(raw.width, raw.height);
let crop = if let Some(window) = window_context.as_ref() {
ensure_readonly_screenshot_target_is_visible(window)?;
Some(self.window_crop_rect_for_capture(window, &raw).await?)
} else {
None
};
crate::screenshot_impl::run_image_task(move || {
prepare_app_state_screenshot(
raw,
crop,
screenshot_target_requested,
screenshot_options,
)
})
.await
}
.await;
match result {
Ok(capture) => (Some(capture), None),
Err(error) => (None, Some(format!("{error:#}"))),
}
} else {
(None, None)
};
let mut tree_scoped = false;
let mut tree_root_pid = None;
let mut accessibility_tree_truncated = false;
let mut shell_only_note = None;
let (accessibility_tree, accessibility_tree_raw_count, accessibility_error) =
if diagnostics.readiness.can_build_accessibility_tree {
let target_pid = window_context.as_ref().and_then(|window| window.pid);
match snapshot_accessibility_tree(
app_filter.as_deref(),
target_pid,
max_nodes,
max_depth,
)
.await
{
Ok(snapshot) => {
tree_scoped = snapshot.scoped;
tree_root_pid = snapshot.root_pid;
accessibility_tree_truncated = snapshot.truncated;
let raw_count = snapshot.nodes.len();
let (nodes, note) = compact_accessibility_tree_with_note(
snapshot.nodes,
tree_scoped,
accessibility_tree_truncated,
);
shell_only_note = note;
(nodes, raw_count, None)
}
Err(error) => (Vec::new(), 0, Some(format!("{error:#}"))),
}
} else {
(
Vec::new(),
0,
Some(
"GNOME accessibility is disabled; call setup_accessibility first."
.to_string(),
),
)
};
if accessibility_error.is_none() {
// Record a pid only when the tree's roots were matched by it. A pid
// with no AT-SPI root falls back to every app, or to an app-name
// match that can be another app entirely; recording the pid then
// would let that app's index pass the target check instead of
// taking the per-node owner lookup.
self.commit_snapshot(&accessibility_tree, tree_root_pid)
.await;
} else {
self.commit_snapshot(&[], None).await;
}
let mut message = if let Some(error) = &accessibility_error {
format!("MCP registration is working, but AT-SPI tree extraction failed: {error}")
} else if let Some(capture) = &screenshot {
format!(
"MCP registration, screenshot capture, and AT-SPI tree extraction are working. Captured {} accessibility nodes (compacted from {}) and a screenshot through {}.",
accessibility_tree.len(),
accessibility_tree_raw_count,
capture.source
)
} else if let Some(error) = &screenshot_error {
format!(
"MCP registration and AT-SPI tree extraction are working. Captured {} accessibility nodes (compacted from {}). Screenshot capture failed: {error}",
accessibility_tree.len(),
accessibility_tree_raw_count,
)
} else {
format!(
"MCP registration and AT-SPI tree extraction are working. Captured {} accessibility nodes (compacted from {}). Screenshot capture was not requested.",
accessibility_tree.len(),
accessibility_tree_raw_count,
)
};
if let Some(window) = &window_context {
message.push_str(&format!(
" Window target resolved to window_id {}.",
window.window_id
));
} else if let Some(error) = &window_error {
message.push_str(&format!(" Window target resolution failed: {error}"));
}
if let Some(warning) = unscoped_accessibility_tree_warning(
tree_scoped,
accessibility_error.is_none(),
accessibility_target_requested,
) {
message.push(' ');
message.push_str(warning);
}
if let Some(note) = truncated_accessibility_tree_note(accessibility_tree_truncated) {
message.push(' ');
message.push_str(note);
}
if let Some(note) = shell_only_note {
message.push(' ');
message.push_str(note);
}
// Full diagnostics are huge (portal/process dumps); emit them only on
// request. The compact readiness block always travels, and failures get
// a pointer to verbose=true instead of an automatic dump.
let readiness = diagnostics.readiness.clone();
let include_full = verbose;
if !include_full
&& (accessibility_error.is_some()
|| screenshot_error.is_some()
|| window_error.is_some())
{
message.push_str(" Pass verbose=true for full diagnostics.");
}
let output = GetAppStateOutput {
app_name_or_bundle_identifier: params.app_name_or_bundle_identifier,
window_context,
window_error,
window_permissions_hint,
backend: "linux-atspi".to_string(),
screenshot: screenshot.as_ref().map(ScreenshotSummary::from),
screenshot_error,
accessibility_tree,
accessibility_tree_raw_count,
tree_scoped,
accessibility_tree_truncated,
accessibility_error,
readiness,
diagnostics: include_full.then_some(diagnostics),
message,
};
app_state_result(output, screenshot)
}
#[tool(
name = "screenshot",
description = "Capture the screen and return it as a viewable, size-bounded image. Optionally target a window (window_id/pid/wm_class/title/app_id): the window is raised to the front and the image is cropped before any resize. Returns the image plus a short caption with returned dimensions, coordinate dimensions, scale, format, quality, source, and crop bounds; callers can request jpeg/quality for compression before resizing.",
annotations(
read_only_hint = false,
destructive_hint = false,
idempotent_hint = false,
open_world_hint = true
)
)]
async fn screenshot(&self, Parameters(params): Parameters<ScreenshotParams>) -> CallToolResult {
let result = match self.capture_screenshot(params).await {
Ok(result) => result,
Err(error) => CallToolResult::error(vec![ContentBlock::text(format!("{error:#}"))]),
};
self.indicator.action("screenshot").await;
result
}
async fn capture_screenshot(&self, params: ScreenshotParams) -> Result<CallToolResult> {
let _pipeline = crate::screenshot_impl::screenshot_pipeline_permit().await?;
let target = params.window_target();
let target_window = match target.as_ref() {
Some(target) => Some(
self.resolve_screenshot_window(target, params.raise_window.unwrap_or(true))
.await
.context("targeted screenshot failed")?,
),
None => None,
};
let crop_window = (!params.full_screen.unwrap_or(false))
.then_some(target_window.as_ref())
.flatten();
let window_label = target_window
.as_ref()
.and_then(|window| window.title.clone());
let raw_capture = {
let _hold = self.indicator.hold_for_capture().await?;
capture_screenshot_raw()
.await
.context("screenshot failed")?
};
self.cache_desktop_size(raw_capture.width, raw_capture.height);
// Warn when the target window extends past the visible desktop: the
// portal only captures on-screen pixels, so the crop silently loses the
// off-screen region while coordinate metadata still claims full size.
let off_screen_note = match crop_window.and_then(|window| window.bounds.as_ref()) {
Some(bounds) => self.off_screen_note_for_bounds(bounds).await,
None => None,
};
let (capture, cropped) = match crop_window {
Some(window) => {
let (x, y, width, height) = self
.window_crop_rect_for_capture(window, &raw_capture)
.await
.context("targeted screenshot crop failed")?;
let (raw_capture, bytes, width, height) =
crate::screenshot_impl::run_image_task(move || {
let (bytes, width, height) =
crop_png(&raw_capture.bytes, x, y, width, height).map_err(|error| {
anyhow::anyhow!("targeted screenshot crop failed: {error}")
})?;
Ok((raw_capture, bytes, width, height))
})
.await?;
(
RawScreenshotCapture {
mime_type: raw_capture.mime_type,
bytes,
source: raw_capture.source,
width,
height,
},
true,
)
}
None => (raw_capture, false),
};
let options = params.screenshot_options();
let capture = crate::screenshot_impl::run_image_task(move || {
prepare_screenshot_payload(capture, options)
})
.await
.context("screenshot resize failed")?;
let mut caption = serde_json::json!({
"width": capture.width,
"height": capture.height,
"coordinate_width": capture.coordinate_width,
"coordinate_height": capture.coordinate_height,
"scale": capture.scale,
"resized": capture.resized,
"bytes": capture.bytes,
"original_bytes": capture.original_bytes,
"max_bytes": capture.max_bytes,
"format": capture.format,
"quality": capture.quality,
"source": capture.source,
"cropped_to_window": cropped,
"window_title": window_label,
});
if let Some(note) = off_screen_note {
caption["window_off_screen"] = serde_json::json!(true);
caption["off_screen_note"] = serde_json::json!(note);
}
Ok(CallToolResult::success(vec![
ContentBlock::image(data_url_payload(&capture.data_url), capture.mime_type),
ContentBlock::text(caption.to_string()),
]))
}
/// Lazily create the uinput absolute pointer, sizing its ABS range to the
/// full capture's pixel dimensions. This preserves the coordinate space
/// used by click/scroll/drag, independently of logical monitor scaling.
/// Explicit backend overrides skip this device, including a cached one.
async fn ensure_abs_pointer(&self) -> bool {
if env_flag_enabled("CU_DISABLE_ABS_POINTER")
|| env_flag_enabled("COMPUTER_USE_LINUX_FORCE_YDOTOOL_POINTER")
|| (env_flag_enabled("COMPUTER_USE_LINUX_FORCE_PORTAL_POINTER")
&& self.is_wayland_session())
{
return false;
}
if self
.abs_pointer
.lock()
.map(|g| g.is_some())
.unwrap_or(false)
{
return true;
}
let Ok(cap) = capture_screenshot_raw().await else {
return false;
};
self.cache_desktop_size(cap.width, cap.height);
match tokio::task::spawn_blocking(move || {
crate::abs_pointer::AbsPointer::create(cap.width as i32, cap.height as i32)
})
.await
{
Ok(Ok(pointer)) => {
if let Ok(mut guard) = self.abs_pointer.lock() {
*guard = Some(pointer);
return true;
}
false
}
_ => false,
}
}
#[tool(
name = "click",
description = "Click an element by index, semantic selector, or desktop coordinate pixels from screenshot metadata. Plain left activation prefers a native AT-SPI click/press/toggle action, avoiding toolkit coordinate scaling. Entry activate and slider jump actions are not substituted for pointer clicks. Explicit coordinates, right clicks, and multi-clicks retain pointer semantics.",
annotations(
read_only_hint = false,
destructive_hint = true,
idempotent_hint = false,
open_world_hint = true
)
)]
async fn click(&self, Parameters(mut params): Parameters<ClickParams>) -> Json<ActionOutput> {
let received = Some(serde_json::json!(params.clone()));
let input_guard = Arc::clone(&self.input_operation_lock).lock_owned().await;
let mut portal_target_point = None;
let mut target_pid = params.pid;
// Raise the target window first (if specified) so the click lands on the
// intended app rather than whatever is stacked on top at that pixel.
let window_target = params.window_target();
if params.relative == Some(true) && window_target.is_none() {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "click".to_string(),
message: "Relative coordinate clicks require a window target.".to_string(),
received,
});
}
if let Some(target) = window_target {
let focus = match self.focus_target_for_input(&target).await {
Ok(focus) => focus,
Err(message) => {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "click".to_string(),
message,
received,
});
}
};
target_pid = focus_target_pid(focus.as_ref()).or(target_pid);
tokio::time::sleep(Duration::from_millis(120)).await;
// Window-relative coordinates: translate by the window's top-left so
// the agent can click the pixel it saw in a window-cropped screenshot.
if params.relative == Some(true) {
let Some(focus) = focus.as_ref() else {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "click".to_string(),
message: "Relative coordinate clicks require verified target-window focus."
.to_string(),
received,
});
};
let coordinate_map = match self.focused_window_coordinate_map(focus).await {
Ok(mapping) => mapping,
Err(message) => {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "click".to_string(),
message,
received,
});
}
};
if let Err(message) = apply_window_relative_click_coordinates(
&mut params,
coordinate_map.capture_rect,
) {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "click".to_string(),
message,
received,
});
}
portal_target_point = params
.x
.zip(params.y)
.and_then(|(x, y)| coordinate_map.portal_point(x, y));
}
}
if params.x.zip(params.y).is_none() {
if let Some(node) = self.cached_node_for(
params.element_index,
¶ms.selector(),
ElementResolvePurpose::Click,
) {
if let Err(message) = self.check_node_target(&node, target_pid).await {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "click".to_string(),
message,
received,
});
}
}
}
let target = match self.resolve_click_target(¶ms) {
Ok(target) => target,
Err(message) => {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "click".to_string(),
message,
received,
});
}
};
if let ClickTarget::PrimaryAction {
object_ref,
action_name,
action_index,
} = target
{
self.indicator.action("click").await;
let invocation = if let Some(name) = action_name
.as_deref()
.filter(|name| !name.trim().is_empty())
{
perform_named_action(&object_ref, name).await
} else {
invoke_accessibility_action(&object_ref, Some(&action_index.to_string())).await
};
return match invocation {
Ok(invocation) => Json(ActionOutput {
ok: invocation.ok,
implemented: true,
action: "click".to_string(),
message: if invocation.ok {
format!(
"Invoked the primary AT-SPI action{} without pointer-coordinate conversion.",
action_name
.as_deref()
.filter(|name| !name.is_empty())
.map(|name| format!(" ({name})"))
.unwrap_or_default()
)
} else {
format!(
"The primary AT-SPI action{} returned false.",
action_name
.as_deref()
.filter(|name| !name.is_empty())
.map(|name| format!(" ({name})"))
.unwrap_or_default()
)
},
received,
}),
Err(error) => Json(ActionOutput {
ok: false,
implemented: true,
action: "click".to_string(),
message: error.to_string(),
received,
}),
};
}
let ClickTarget::Coordinates(x, y) = target else {
unreachable!("click target must resolve to coordinates or an AT-SPI action");
};
self.indicator
.pointer("click", (x, y), self.indicator_space())
.await;
let button = mouse_button_code(params.button.as_deref());
let click_count = params.click_count.unwrap_or(1).clamp(1, 10).to_string();
// Preferred backend: the uinput absolute pointer. Unlike ydotool's
// relative-only device (faked `--absolute` via pin-to-corner + relative
// move, which acceleration + fractional scaling distort) and unlike the
// portal (per-monitor coordinate scaling + an approval dialog), the
// absolute pointer uses screenshot-pixel coordinates directly and
// reports the point it emitted after desktop-edge clamping.
if let Some(abs_button) =
crate::abs_pointer::PointerButton::from_name(params.button.as_deref())
{
if self.ensure_abs_pointer().await {
let pointer = Arc::clone(&self.abs_pointer);
let count = params.click_count.unwrap_or(1).clamp(1, 10);
let (input_guard, result) = run_cancellation_safe_input(input_guard, async move {
run_abs_pointer(pointer, move |pointer| {
pointer.click(x, y, abs_button, count)
})
.await
})
.await;
let _input_guard = input_guard;
return Json(abs_pointer_result("click", result, received));
}
}
let off_screen_note = self.off_screen_note_for_point(x, y).await;
if let Some(session) = self.cached_portal_pointer_session() {
let Some((portal_x, portal_y)) =
portal_target_point.or_else(|| self.logical_portal_point(&session, x, y))
else {
self.clear_portal_pointer_session(&session);
return Json(with_notes(
portal_coordinate_error("click", received),
off_screen_note.clone(),
));
};
match portal_click(
&session,
portal_x,
portal_y,
PointerButton::from_name(params.button.as_deref()),
params.click_count.unwrap_or(1).clamp(1, 10),
)
.await
{
Ok(()) => {
return Json(with_notes(
ActionOutput {
ok: true,
implemented: true,
action: "click".to_string(),
message: "Action sent through the remote desktop portal.".to_string(),
received,
},
off_screen_note.clone(),
));
}
Err(error) => {
self.clear_portal_pointer_session(&session);
return Json(with_notes(
portal_action_error("click", error, received),
off_screen_note.clone(),
));
}
}
} else if self.should_prefer_portal_pointer_backend().await {
match self.ensure_portal_pointer_session().await {
Ok(Some(session)) => {
let Some((portal_x, portal_y)) =
portal_target_point.or_else(|| self.logical_portal_point(&session, x, y))
else {
self.clear_portal_pointer_session(&session);
return Json(with_notes(
portal_coordinate_error("click", received),
off_screen_note.clone(),
));
};
match portal_click(
&session,
portal_x,
portal_y,
PointerButton::from_name(params.button.as_deref()),
params.click_count.unwrap_or(1).clamp(1, 10),
)
.await
{
Ok(()) => {
return Json(with_notes(
ActionOutput {
ok: true,
implemented: true,
action: "click".to_string(),
message: "Action sent through the remote desktop portal."
.to_string(),
received,
},
off_screen_note.clone(),
));
}
Err(error) => {
self.clear_portal_pointer_session(&session);
return Json(with_notes(
portal_action_error("click", error, received),
off_screen_note.clone(),
));
}
}
}
Ok(None) => {}
Err(error) => return Json(portal_start_error("click", error, received)),
}
}
if self.is_wayland_session() {
return Json(unsafe_pointer_fallback("click", received));
}
if self.should_prefer_xdotool_pointer() {
if let Some(xdotool_args) = xdotool_pointer_click_args(
x,
y,
params.click_count.unwrap_or(1).clamp(1, 10),
params.button.as_deref(),
) {
let ydotool_commands = vec![
absolute_mousemove_args(x, y),
vec![
"click".to_string(),
"--repeat".to_string(),
click_count.clone(),
button.clone(),
],
];
let (input_guard, result) = run_cancellation_safe_input(input_guard, async move {
run_xdotool_pointer_or_fallback(Path::new("xdotool"), &xdotool_args, || async {
run_ydotool_sequence(&ydotool_commands).await
})
.await
})
.await;
let _input_guard = input_guard;
let used_xdotool = result
.as_ref()
.is_ok_and(|result| result.backend == KeyboardCommandBackend::Xdotool);
let mut output =
action_result("click", result.map(|result| result.outputs), received);
if output.ok && used_xdotool {
output.message = "Action sent through xdotool (X11 XTEST).".to_string();
}
return Json(with_notes(output, off_screen_note));
}
}
let commands = vec![
absolute_mousemove_args(x, y),
vec![
"click".to_string(),
"--repeat".to_string(),
click_count,
button,
],
];
let (input_guard, result) = run_cancellation_safe_input(input_guard, async move {
run_ydotool_sequence(&commands).await
})
.await;
let _input_guard = input_guard;
Json(with_notes(
action_result("click", result, received),
off_screen_note,
))
}
#[tool(
name = "perform_action",
description = "Invoke an accessibility action exposed by an element selected by index, identifier, or semantic selector. Defaults to the primary action unless action is provided.",
annotations(
read_only_hint = false,
destructive_hint = true,
idempotent_hint = false,
open_world_hint = true
)
)]
async fn perform_action(
&self,
Parameters(params): Parameters<ActionParams>,
) -> Json<ActionOutput> {
let requested_action = requested_or_primary_action(params.action.as_deref());
self.perform_element_action(¶ms, Some(requested_action))
.await
}
#[tool(
name = "set_value",
description = "Set the value of a settable accessibility element selected by index, identifier, or semantic selector.",
annotations(
read_only_hint = false,
destructive_hint = true,
idempotent_hint = false,
open_world_hint = true
)
)]
async fn set_value(
&self,
Parameters(params): Parameters<SetValueParams>,
) -> Json<ActionOutput> {
let _input_guard = Arc::clone(&self.input_operation_lock).lock_owned().await;
let received = Some(serde_json::json!(params.clone()));
let object_ref = match self.resolve_object_ref(
params.element_index,
params.element_identifier.as_deref(),
¶ms.selector(),
ElementResolvePurpose::SetValue,
) {
Ok(object_ref) => object_ref,
Err(message) => {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "set_value".to_string(),
message,
received,
});
}
};
self.indicator.text("set_value", ¶ms.value, false).await;
match set_element_value(&object_ref, ¶ms.value).await {
Ok(ValueSetInvocation::Numeric { value }) => Json(ActionOutput {
ok: true,
implemented: true,
action: "set_value".to_string(),
message: format!("AT-SPI numeric value set to {value}."),
received,
}),
Ok(ValueSetInvocation::EditableText) => Json(ActionOutput {
ok: true,
implemented: true,
action: "set_value".to_string(),
message: "AT-SPI editable text contents set.".to_string(),
received,
}),
Err(error) => Json(ActionOutput {
ok: false,
implemented: true,
action: "set_value".to_string(),
message: error.to_string(),
received,
}),
}
}
#[tool(
name = "scroll",
description = "Scroll an element in a direction by a number of pages. With a window target and no x/y/element_index, scrolls at the centre of the targeted window.",
annotations(
read_only_hint = false,
destructive_hint = false,
idempotent_hint = false,
open_world_hint = true
)
)]
async fn scroll(&self, Parameters(mut params): Parameters<ScrollParams>) -> Json<ActionOutput> {
let received = Some(serde_json::json!(params.clone()));
let units = match scroll_units(params.pages) {
Ok(units) => units,
Err(message) => {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "scroll".to_string(),
message,
received,
});
}
};
let input_guard = Arc::clone(&self.input_operation_lock).lock_owned().await;
let mut portal_target_point = None;
let mut target_pid = params.pid;
// Raise/focus the target window first (parity with click) so wheel
// events land on the intended app.
let window_target = params.window_target();
if params.relative == Some(true) && window_target.is_none() {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "scroll".to_string(),
message: "Relative scroll coordinates require a window target.".to_string(),
received,
});
}
if let Some(target) = window_target {
let focus = match self.focus_target_for_input(&target).await {
Ok(focus) => focus,
Err(message) => {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "scroll".to_string(),
message,
received,
});
}
};
target_pid = focus_target_pid(focus.as_ref()).or(target_pid);
tokio::time::sleep(Duration::from_millis(120)).await;
if params.relative == Some(true) {
let Some(focus) = focus.as_ref() else {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "scroll".to_string(),
message:
"Relative scroll coordinates require verified target-window focus."
.to_string(),
received,
});
};
let coordinate_map = match self.focused_window_coordinate_map(focus).await {
Ok(mapping) => mapping,
Err(message) => {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "scroll".to_string(),
message,
received,
});
}
};
if let Err(message) = apply_window_relative_scroll_coordinates(
&mut params,
coordinate_map.capture_rect,
) {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "scroll".to_string(),
message,
received,
});
}
portal_target_point = params
.x
.zip(params.y)
.and_then(|(x, y)| coordinate_map.portal_point(x, y));
} else if params.x.is_none() && params.y.is_none() && params.element_index.is_none() {
// A window target without a point would otherwise scroll
// whatever happens to sit under the pointer: focusing does not
// move the cursor, and the wheel path never repositions it.
// Default to the centre of the resolved target window.
let Some(focus) = focus.as_ref() else {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "scroll".to_string(),
message: "Window-targeted scroll requires verified target-window focus."
.to_string(),
received,
});
};
let coordinate_map = match self.focused_window_coordinate_map(focus).await {
Ok(mapping) => mapping,
Err(message) => {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "scroll".to_string(),
message,
received,
});
}
};
if let Err(message) =
apply_window_center_scroll_point(&mut params, coordinate_map.capture_rect)
{
return Json(ActionOutput {
ok: false,
implemented: true,
action: "scroll".to_string(),
message,
received,
});
}
portal_target_point = params
.x
.zip(params.y)
.and_then(|(x, y)| coordinate_map.portal_point(x, y));
}
}
if params.x.is_none() && params.y.is_none() {
if let Some(node) = self.cached_node_for(
params.element_index,
&ElementSelector::default(),
ElementResolvePurpose::Click,
) {
if let Err(message) = self.check_node_target(&node, target_pid).await {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "scroll".to_string(),
message,
received,
});
}
}
}
let target_point =
match self.resolve_optional_target_point(params.x, params.y, params.element_index) {
Ok(point) => point,
Err(message) => {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "scroll".to_string(),
message,
received,
});
}
};
let direction = match params.direction.to_ascii_lowercase().as_str() {
"up" => ScrollDirection::Up,
"down" => ScrollDirection::Down,
"left" => ScrollDirection::Left,
"right" => ScrollDirection::Right,
_ => {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "scroll".to_string(),
message: "Unsupported scroll direction; expected up, down, left, or right."
.to_string(),
received,
});
}
};
match target_point {
Some(point) => {
self.indicator
.pointer("scroll", point, self.indicator_space())
.await
}
None => self.indicator.action("scroll").await,
}
let (dx, dy) = match direction {
ScrollDirection::Up => (0, units),
ScrollDirection::Down => (0, -units),
ScrollDirection::Left => (units, 0),
ScrollDirection::Right => (-units, 0),
};
// Coordinate scroll needs the same accurate positioning as click. Keep
// the move and wheel event under one cancellation-safe input guard.
if let Some((x, y)) = target_point {
if self.is_wayland_session()
&& ydotool_backend_available().await
&& self.ensure_abs_pointer().await
{
let pointer = Arc::clone(&self.abs_pointer);
let (input_guard, result) = run_cancellation_safe_input(input_guard, async move {
let landing =
run_abs_pointer(pointer, move |pointer| pointer.move_to(x, y)).await?;
sleep(Duration::from_millis(35)).await;
run_ydotool(&wheel_mousemove_args(dx, dy)).await?;
Ok(landing)
})
.await;
let _input_guard = input_guard;
return Json(abs_pointer_result("scroll", result, received));
}
}
let off_screen_note = match target_point {
Some((x, y)) => self.off_screen_note_for_point(x, y).await,
None => None,
};
if let Some(session) = self.cached_portal_pointer_session() {
let mapped_target = match (portal_target_point, target_point) {
(Some(point), _) => Some(Some(point)),
(None, Some((x, y))) => self.logical_portal_point(&session, x, y).map(Some),
(None, None) => Some(None),
};
let Some(portal_target_point) = mapped_target else {
self.clear_portal_pointer_session(&session);
return Json(with_notes(
portal_coordinate_error("scroll", received),
off_screen_note.clone(),
));
};
match portal_scroll(&session, portal_target_point, direction, units).await {
Ok(()) => {
return Json(with_notes(
ActionOutput {
ok: true,
implemented: true,
action: "scroll".to_string(),
message: "Action sent through the remote desktop portal.".to_string(),
received,
},
off_screen_note.clone(),
));
}
Err(error) => {
self.clear_portal_pointer_session(&session);
return Json(with_notes(
portal_action_error("scroll", error, received),
off_screen_note.clone(),
));
}
}
} else if self.should_prefer_portal_pointer_backend().await {
match self.ensure_portal_pointer_session().await {
Ok(Some(session)) => {
let mapped_target = match (portal_target_point, target_point) {
(Some(point), _) => Some(Some(point)),
(None, Some((x, y))) => self.logical_portal_point(&session, x, y).map(Some),
(None, None) => Some(None),
};
let Some(portal_target_point) = mapped_target else {
self.clear_portal_pointer_session(&session);
return Json(with_notes(
portal_coordinate_error("scroll", received),
off_screen_note.clone(),
));
};
match portal_scroll(&session, portal_target_point, direction, units).await {
Ok(()) => {
return Json(with_notes(
ActionOutput {
ok: true,
implemented: true,
action: "scroll".to_string(),
message: "Action sent through the remote desktop portal."
.to_string(),
received,
},
off_screen_note.clone(),
));
}
Err(error) => {
self.clear_portal_pointer_session(&session);
return Json(with_notes(
portal_action_error("scroll", error, received),
off_screen_note.clone(),
));
}
}
}
Ok(None) => {}
Err(error)
if target_point.is_some()
|| env_flag_enabled("COMPUTER_USE_LINUX_FORCE_PORTAL_POINTER") =>
{
return Json(portal_start_error("scroll", error, received));
}
Err(_) => {}
}
}
if target_point.is_some() && self.is_wayland_session() {
return Json(unsafe_pointer_fallback("scroll", received));
}
let mut sequence = Vec::new();
if let Some((x, y)) = target_point {
sequence.push(absolute_mousemove_args(x, y));
}
sequence.push(wheel_mousemove_args(dx, dy));
// X11: ydotool's absolute move warps through (0, 0), so the pointer
// re-enters the window. GTK 3 resets its XI2 scroll valuators on that
// enter, and the one wheel event only re-baselines them (delta 0).
// The call reported success while the window did not move (#184).
// XTEST wheel buttons have no valuators, so GTK takes the button path.
if self.should_prefer_xdotool_pointer() {
let repeat = u32::try_from(units.max(1)).unwrap_or(1);
let xdotool_args = xdotool_pointer_scroll_args(target_point, repeat, direction);
let command_timeout = xdotool_scroll_timeout(repeat, XDOTOOL_SCROLL_CLICK_DELAY_MS);
let (input_guard, result) = run_cancellation_safe_input(input_guard, async move {
run_xdotool_pointer_or_fallback_with_timeout(
Path::new("xdotool"),
&xdotool_args,
command_timeout,
|| async { run_ydotool_sequence(&sequence).await },
)
.await
})
.await;
let _input_guard = input_guard;
let used_xdotool = result
.as_ref()
.is_ok_and(|result| result.backend == KeyboardCommandBackend::Xdotool);
let mut output = action_result("scroll", result.map(|result| result.outputs), received);
if output.ok && used_xdotool {
output.message = "Action sent through xdotool (X11 XTEST).".to_string();
}
return Json(with_notes(output, off_screen_note));
}
let (input_guard, result) = run_cancellation_safe_input(input_guard, async move {
run_ydotool_sequence(&sequence).await
})
.await;
let _input_guard = input_guard;
Json(with_notes(
action_result("scroll", result, received),
off_screen_note,
))
}
#[tool(
name = "drag",
description = "Drag from one point to another using pixel coordinates.",
annotations(
read_only_hint = false,
destructive_hint = true,
idempotent_hint = false,
open_world_hint = true
)
)]
async fn drag(&self, Parameters(params): Parameters<DragParams>) -> Json<ActionOutput> {
let received = Some(serde_json::json!(params));
let input_guard = Arc::clone(&self.input_operation_lock).lock_owned().await;
let space = self.indicator_space();
self.indicator
.pointer("drag", (params.start_x, params.start_y), space)
.await;
// Lead the overlay cursor to the drop point while the drag runs.
self.indicator
.follow("drag", (params.end_x, params.end_y), space)
.await;
// Preferred backend: the uinput absolute pointer (accurate landing).
if self.ensure_abs_pointer().await {
let pointer = Arc::clone(&self.abs_pointer);
let (input_guard, result) = run_cancellation_safe_input(input_guard, async move {
run_abs_pointer(pointer, move |pointer| {
pointer.drag(
(params.start_x, params.start_y),
(params.end_x, params.end_y),
crate::abs_pointer::PointerButton::Left,
)
})
.await
})
.await;
let _input_guard = input_guard;
return Json(match result {
Ok(()) => ActionOutput {
ok: true,
implemented: true,
action: "drag".to_string(),
message: "Action sent through the uinput absolute pointer.".to_string(),
received,
},
Err(error) => abs_pointer_error("drag", error, received),
});
}
if let Some(session) = self.cached_portal_pointer_session() {
let _ = self.capture_space_rect().await;
let Some((start_x, start_y)) =
self.logical_portal_point(&session, params.start_x, params.start_y)
else {
self.clear_portal_pointer_session(&session);
return Json(portal_coordinate_error("drag", received));
};
let Some((end_x, end_y)) =
self.logical_portal_point(&session, params.end_x, params.end_y)
else {
self.clear_portal_pointer_session(&session);
return Json(portal_coordinate_error("drag", received));
};
match portal_drag(&session, start_x, start_y, end_x, end_y).await {
Ok(()) => {
return Json(ActionOutput {
ok: true,
implemented: true,
action: "drag".to_string(),
message: "Action sent through the remote desktop portal.".to_string(),
received,
});
}
Err(error) => {
self.clear_portal_pointer_session(&session);
return Json(portal_action_error("drag", error, received));
}
}
} else if self.should_prefer_portal_pointer_backend().await {
let _ = self.capture_space_rect().await;
match self.ensure_portal_pointer_session().await {
Ok(Some(session)) => {
let Some((start_x, start_y)) =
self.logical_portal_point(&session, params.start_x, params.start_y)
else {
self.clear_portal_pointer_session(&session);
return Json(portal_coordinate_error("drag", received));
};
let Some((end_x, end_y)) =
self.logical_portal_point(&session, params.end_x, params.end_y)
else {
self.clear_portal_pointer_session(&session);
return Json(portal_coordinate_error("drag", received));
};
match portal_drag(&session, start_x, start_y, end_x, end_y).await {
Ok(()) => {
return Json(ActionOutput {
ok: true,
implemented: true,
action: "drag".to_string(),
message: "Action sent through the remote desktop portal."
.to_string(),
received,
});
}
Err(error) => {
self.clear_portal_pointer_session(&session);
return Json(portal_action_error("drag", error, received));
}
}
}
Ok(None) => {}
Err(error) => return Json(portal_start_error("drag", error, received)),
}
}
if self.is_wayland_session() {
return Json(unsafe_pointer_fallback("drag", received));
}
let (input_guard, result) = run_cancellation_safe_input(input_guard, async move {
run_ydotool_drag(params.start_x, params.start_y, params.end_x, params.end_y).await
})
.await;
let _input_guard = input_guard;
Json(action_result("drag", result, received))
}
#[tool(
name = "press_key",
description = "Press a key or key-combination on the keyboard, optionally after focusing a target window or terminal selector. Key grammar (case-insensitive; hyphens/spaces ignored): combos join with '+', e.g. Ctrl+L or Ctrl+Shift+T. Modifiers: ctrl/control, alt/option, shift, meta/super/cmd/command. Named keys: enter/return, escape/esc, tab, backspace, delete/del, space, home, end, pageup, pagedown, arrowleft/left, arrowright/right, arrowup/up, arrowdown/down, f1-f12. Plus single US letters a-z and digits 0-9. Anything else returns an error (never silently dropped). On Wayland, chords are sent through an active remote desktop portal keyboard session when one is available (or when ydotool is absent), falling back to ydotool otherwise. Portal chords send modifiers and named keys as keysyms so remapped keys (e.g. Caps Lock swapped with Control) follow the active keymap; letters and digits are physical US positions. Note: compositor-level shortcuts (e.g. Super+Up) may be consumed by GNOME before reaching the app.",
annotations(
read_only_hint = false,
destructive_hint = true,
idempotent_hint = false,
open_world_hint = true
)
)]
async fn press_key(
&self,
Parameters(params): Parameters<PressKeyParams>,
) -> Json<ActionOutput> {
let received = Some(serde_json::json!(params.clone()));
let input_guard = Arc::clone(&self.input_operation_lock).lock_owned().await;
let focus = match self.focus_target_for_input(¶ms.window_target()).await {
Ok(focus) => focus,
Err(message) => {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "press_key".to_string(),
message,
received,
});
}
};
let Some((chord_modifiers, chord_key)) = key_chord(¶ms.key) else {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "press_key".to_string(),
message: "Unsupported key. Use names like Enter, Escape, Tab, ArrowLeft, Super, Ctrl+L, or a single US keyboard letter/digit.".to_string(),
received,
});
};
let secret = self.typing_into_secret(focus.as_ref()).await;
self.indicator.keys(¶ms.key, secret).await;
if self.should_prefer_portal_keyboard_for_chords().await {
match self.ensure_portal_keyboard_session().await {
Ok(Some(session)) => {
// KDE keeps physical keycodes: its keysym path is untested
// here, and #191 was reported against mutter.
let (modifiers, key) = portal_key_chord(¶ms.key)
.filter(|_| !self.is_kde_wayland_session())
.unwrap_or_else(|| {
let modifiers = chord_modifiers
.iter()
.map(|m| PortalKey::Keycode(i32::from(*m)))
.collect();
(modifiers, PortalKey::Keycode(i32::from(chord_key)))
});
match press_key_chord(&session, &modifiers, key).await {
Ok(()) => {
let notes = self.input_landing_notes(focus.as_ref(), false).await;
return Json(with_notes(
successful_action_with_focus(
"press_key",
"Action sent through the remote desktop portal.",
received,
focus,
),
notes,
));
}
Err(error) => {
self.clear_portal_keyboard_session(&session);
return Json(action_result_with_focus(
"press_key",
Err(format!("{error:#}")),
received,
focus,
));
}
}
}
Ok(None) => {}
Err(_) => {}
}
}
let Some(key_events) = key_sequence(¶ms.key) else {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "press_key".to_string(),
message: "Unsupported key. Use names like Enter, Escape, Tab, ArrowLeft, Super, Ctrl+L, or a single US keyboard letter/digit.".to_string(),
received,
});
};
// X11: prefer xdotool/XTEST. ydotool's raw evdev scancodes get
// re-mapped by the active XKB layout on X11, so named keys and chords
// arrive as stray glyphs instead of real key events (issue #58).
if self.should_prefer_xdotool_keyboard() {
if let Some(spec) = xdotool_key_spec(¶ms.key) {
let xdotool_args = vec!["key".to_string(), "--clearmodifiers".to_string(), spec];
let ydotool_args =
ydotool_key_args(key_events.clone(), !chord_modifiers.is_empty());
let (input_guard, result) = run_cancellation_safe_input(input_guard, async move {
run_xdotool_or_fallback(Path::new("xdotool"), &xdotool_args, || {
run_ydotool(&ydotool_args)
})
.await
})
.await;
let _input_guard = input_guard;
let used_xdotool = result
.as_ref()
.is_ok_and(|result| result.backend == KeyboardCommandBackend::Xdotool);
let mut output = action_result_with_focus(
"press_key",
result.map(|result| vec![result.output]),
received,
focus.clone(),
);
if used_xdotool {
output.message = "Action sent through xdotool (X11 XTEST).".to_string();
}
if output.ok && focus.is_some() {
let notes = self.input_landing_notes(focus.as_ref(), false).await;
output = with_notes(output, notes);
}
return Json(output);
}
}
let args = ydotool_key_args(key_events, !chord_modifiers.is_empty());
let (input_guard, result) = run_cancellation_safe_input(input_guard, async move {
run_ydotool(&args).await.map(|output| vec![output])
})
.await;
let _input_guard = input_guard;
let mut output = action_result_with_focus("press_key", result, received, focus.clone());
if output.ok && focus.is_some() {
let notes = self.input_landing_notes(focus.as_ref(), false).await;
output = with_notes(output, notes);
}
Json(output)
}
#[tool(
name = "run_shell",
description = "Execute one explicitly approved /bin/sh command with same-user host authority. This tool is absent unless the server operator starts computer-use-linux with COMPUTER_USE_LINUX_ENABLE_SHELL=1. It is not sandboxed: the command can read or modify files and use the network with the server user's permissions. The inherited environment is cleared to a small desktop/runtime allowlist; pass any additional variables explicitly. Execution time and output are bounded: returned streams are truncated to 512 KiB, while a stream exceeding the 8 MiB collection ceiling fails the call without returning partial output. An audit digest is written to server stderr.",
annotations(
read_only_hint = false,
destructive_hint = true,
idempotent_hint = false,
open_world_hint = true
)
)]
async fn run_shell(
&self,
Parameters(params): Parameters<RunShellParams>,
) -> Json<RunShellOutput> {
Json(execute_shell(params).await)
}
#[tool(
name = "type_text",
description = "Type literal text using keyboard input, optionally after focusing a target window or terminal selector. Success confirms dispatch, not field-content verification; re-observe the target before retrying. The wtype backend rejects NUL and text over 131071 UTF-8 bytes before focus or indicators; Hyprland rejects confirmed password fields; ambiguous fields and terminals use private stdin for at most 100 Unicode characters per call.",
annotations(
read_only_hint = false,
destructive_hint = true,
idempotent_hint = false,
open_world_hint = true
)
)]
async fn type_text(
&self,
Parameters(params): Parameters<TypeTextParams>,
) -> Json<ActionOutput> {
let Json(mut output) = self.dispatch_type_text(params).await;
if output.ok {
output.message.push_str(
" Field contents were not verified; re-observe the target before retrying.",
);
}
Json(output)
}
async fn dispatch_type_text(&self, params: TypeTextParams) -> Json<ActionOutput> {
let received = Some(serde_json::json!(params.clone()));
// Reject before focus changes or any indicator can display the text.
let wtype_mode = self.wtype_text_mode();
if let Err(message) = preflight_type_text(¶ms.text, wtype_mode) {
return Json(action_result("type_text", Err(message), received));
}
let input_guard = Arc::clone(&self.input_operation_lock).lock_owned().await;
let window_target = params.window_target();
let focus = match self.focus_target_for_input(&window_target).await {
Ok(focus) => focus,
Err(message) => {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "type_text".to_string(),
message,
received,
});
}
};
let hyprland_argument_mode = wtype_mode == Some(WtypeTextMode::Argument);
let (wtype_mode, secret) = if hyprland_argument_mode {
match hyprland_wtype_delivery(¶ms.text, self.input_focus_probe(focus.as_ref()))
.await
{
Ok(mode) => (Some(mode), mode == WtypeTextMode::Stdin),
Err(message) => {
return Json(action_result_with_focus(
"type_text",
Err(message),
received,
focus,
))
}
}
} else {
(wtype_mode, self.typing_into_secret(focus.as_ref()).await)
};
if let Some(mode) = wtype_mode {
if let Err(message) = validate_wtype_privacy(mode, secret) {
return Json(action_result_with_focus(
"type_text",
Err(message),
received,
focus,
));
}
}
self.indicator.text("type_text", ¶ms.text, secret).await;
if self.should_prefer_kde_clipboard_text_backend() {
match self.ensure_portal_keyboard_session().await {
Ok(Some(session)) => {
let kde_focus = if window_target.has_target() {
match self.focus_target_for_input(&window_target).await {
Ok(focus) => focus,
Err(message) => {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "type_text".to_string(),
message,
received,
});
}
}
} else {
focus.clone()
};
let paste_shortcut =
kde_clipboard_paste_shortcut(&window_target, kde_focus.as_ref()).await;
let _clipboard_guard = self.kde_clipboard_lock.lock().await;
match run_kde_clipboard_paste_text(&session, ¶ms.text, paste_shortcut).await
{
Ok(message) => {
let notes = self.input_landing_notes(kde_focus.as_ref(), true).await;
return Json(with_notes(
successful_action_with_focus(
"type_text",
&message,
received,
kde_focus,
),
notes,
));
}
Err(error) => {
if error.clear_portal_keyboard_session {
self.clear_portal_keyboard_session(&session);
}
if !error.can_fallback_to_ydotool {
return Json(action_result_with_focus(
"type_text",
Err(error.message),
received,
kde_focus,
));
}
}
}
}
Ok(None) => {}
Err(_) => {}
}
}
if self.should_prefer_portal_text_backend() {
let result = async {
let keysyms = keysyms_for_text(¶ms.text)
.map_err(|error| format!("Literal text was not sent: {error:#}"))?;
if self.is_gnome_wayland_session() {
crate::keyboard_keymap::validate_gnome_text(¶ms.text, &keysyms)
.await
.map_err(|error| format!("Literal text was not sent: {error:#}"))?;
}
let session = self
.ensure_portal_keyboard_session()
.await
.map_err(|error| format!("Portal text input could not start: {error:#}"))?
.ok_or_else(|| {
"Portal keyboard input is unavailable; no text was sent.".to_string()
})?;
type_text_with_keysyms(&session, &keysyms)
.await
.map_err(|error| {
self.clear_portal_keyboard_session(&session);
format!("{error:#}")
})
}
.await;
return Json(match result {
Ok(()) => {
let notes = self.input_landing_notes(focus.as_ref(), true).await;
with_notes(
successful_action_with_focus(
"type_text",
"Action sent through the remote desktop portal.",
received,
focus,
),
notes,
)
}
Err(error) => action_result_with_focus("type_text", Err(error), received, focus),
});
}
// X11: xdotool type resolves keysyms against the live XKB layout.
// ydotool's raw scancodes get re-mapped by X11 and mangle symbols and
// digits (`_` → `%`, `1` → `+`) even on a plain US layout (issue #58).
if self.should_prefer_xdotool_keyboard() {
let delay_ms = xdotool_type_delay_ms();
let args = xdotool_type_args_with_delay(¶ms.text, delay_ms);
let command_timeout = xdotool_type_timeout(¶ms.text, delay_ms);
let text = params.text.clone();
let (input_guard, result) = run_cancellation_safe_input(input_guard, async move {
run_xdotool_or_fallback_with_timeout(
Path::new("xdotool"),
&args,
command_timeout,
|| run_ydotool_type_text(&text),
)
.await
})
.await;
let _input_guard = input_guard;
let used_xdotool = result
.as_ref()
.is_ok_and(|result| result.backend == KeyboardCommandBackend::Xdotool);
let mut output = action_result_with_focus(
"type_text",
result.map(|result| vec![result.output]),
received,
focus.clone(),
);
if used_xdotool {
output.message = "Action sent through xdotool (X11 XTEST).".to_string();
}
if output.ok && focus.is_some() {
let notes = self.input_landing_notes(focus.as_ref(), true).await;
output = with_notes(output, notes);
}
return Json(output);
}
if let Some(mode) = wtype_mode {
let text = params.text.clone();
let (input_guard, result) = run_cancellation_safe_input(input_guard, async move {
run_wtype_type_text_or_fallback(Path::new("wtype"), &text, mode, secret, || {
run_ydotool_type_text(&text)
})
.await
})
.await;
let _input_guard = input_guard;
let used_wtype = result
.as_ref()
.is_ok_and(|result| result.backend == KeyboardCommandBackend::Wtype);
let mut output = action_result_with_focus(
"type_text",
result.map(|result| vec![result.output]),
received,
focus.clone(),
);
if used_wtype {
output.message =
"Action sent through wtype (Wayland virtual-keyboard protocol).".to_string();
if hyprland_argument_mode && mode == WtypeTextMode::Stdin {
output.message.push_str(" AT-SPI could not classify the field; text was sent through stdin, limited to 100 Unicode characters per call.");
}
}
if output.ok && focus.is_some() {
let notes = self.input_landing_notes(focus.as_ref(), true).await;
output = with_notes(output, notes);
}
return Json(output);
}
let text = params.text.clone();
let (input_guard, result) = run_cancellation_safe_input(input_guard, async move {
run_ydotool_type_text(&text)
.await
.map(|output| vec![output])
})
.await;
let _input_guard = input_guard;
let mut output = action_result_with_focus("type_text", result, received, focus.clone());
if output.ok && focus.is_some() {
let notes = self.input_landing_notes(focus.as_ref(), true).await;
output = with_notes(output, notes);
}
Json(output)
}
#[tool(
name = "move_window",
description = "Move a window to a new desktop position (frame top-left in desktop coordinates). Useful to recover windows that are partially off-screen. Works through the computer-use-linux GNOME Shell extension or a generic X11/EWMH window manager (wmctrl).",
annotations(
read_only_hint = false,
destructive_hint = false,
idempotent_hint = true,
open_world_hint = true
)
)]
async fn move_window(
&self,
Parameters(params): Parameters<MoveWindowParams>,
) -> Json<WindowGeometryOutput> {
let received = Some(serde_json::json!(params.clone()));
let target = params.target.clone().into_target();
self.indicator.action("move_window").await;
self.window_geometry_op(received, &target, |window| async move {
registry::move_window(&window, params.x, params.y).await
})
.await
}
#[tool(
name = "resize_window",
description = "Resize a window to a new frame width/height in desktop pixels, unmaximizing it first if needed. Useful to fit a window fully on-screen. Works through the computer-use-linux GNOME Shell extension or a generic X11/EWMH window manager (wmctrl).",
annotations(
read_only_hint = false,
destructive_hint = false,
idempotent_hint = true,
open_world_hint = true
)
)]
async fn resize_window(
&self,
Parameters(params): Parameters<ResizeWindowParams>,
) -> Json<WindowGeometryOutput> {
let received = Some(serde_json::json!(params.clone()));
let target = params.target.clone().into_target();
self.indicator.action("resize_window").await;
self.window_geometry_op(received, &target, |window| async move {
registry::resize_window(&window, params.width, params.height).await
})
.await
}
}
#[tool_handler(
router = self.mcp_tool_router(),
name = "computer-use-linux",
// NOTE: keep in lockstep with Cargo.toml + package.json on every release.
// The rmcp tool_handler macro only accepts a string literal here, so this
// can't be env!("CARGO_PKG_VERSION"); the MCP safety check (CI) fails the
// build if it drifts from the Cargo version.
version = "0.7.17",
instructions = "Begin every turn that uses Computer Use by calling get_app_state. If diagnostics report disabled GNOME accessibility, call setup_accessibility before asking the user to retry. Use list_windows/focused_window before targeted keyboard input. If diagnostics report windowing.can_list_windows=false on GNOME, call setup_window_targeting to install the optional GNOME Shell extension backend, then ask the user to log out and back in if the setup report says a shell reload is required. This Linux backend can capture size-bounded screenshots through GNOME Shell or XDG Desktop Portal, read AT-SPI trees with action/value metadata, invoke native AT-SPI actions, set AT-SPI values or editable text, list/focus compositor windows through registered Linux window backends when the session permits it, attach best-effort terminal tty/process metadata to terminal windows, send coordinate or element-targeted click/scroll/drag input through the Wayland remote desktop portal when available, and send literal type_text through KDE clipboard integration on Plasma Wayland, wtype on compatible Wayland compositors, or portal keysyms on other Wayland sessions. Portal text startup or conversion failures return an error without replaying through ydotool. GNOME preflights the complete text against every configured keyboard layout group because the background reader cannot establish the active group; rejected text can use set_value on an editable field. Raw ydotool text is limited to printable ASCII, tab and newline, uses US physical key positions, and requires checking the resulting field contents under the active layout. Screenshot results include width/height for the returned image plus coordinate_width/coordinate_height and scale for desktop coordinate conversion; request more detail with max_width, max_height, max_bytes, format=jpeg, quality, or a smaller target/crop instead of relying on unbounded screenshots. Tools with readOnlyHint=false may mutate local desktop or application state; hosts should require approval for actions that can submit, delete, send, purchase, or overwrite data. For element-targeted actions, prefer element_index from the latest get_app_state result; click, perform_action, and set_value can also use semantic role/name/text/states selectors when the target is unique. type_text and press_key accept optional window_id, pid, app_id, wm_class, title, tty, terminal_pid, terminal_command, or terminal_cwd selectors and refuse targeted input if focus cannot be verified. After targeted keyboard input, results append focused-element feedback from AT-SPI (role, name, editable) and warn when a concrete non-terminal focus is not editable. Exact window focus alone does not suppress a missing-focus warning; only a complete scoped tree containing window/menu chrome without content makes it informational. Terminals accept keyboard input without an EditableText interface. On Hyprland, wtype uses a single literal argument after -- to avoid stdin keymap-refresh corruption; text is visible in process arguments, so confirmed password fields are rejected (use set_value). Terminals and ambiguous fields, with or without AT-SPI, use private stdin for at most 100 Unicode characters per call; split larger input into smaller calls or expose accessibility. Input classification has its own 1500 ms budget, independent of the 250 ms cosmetic indicator timeout. Other compatible compositors retain stdin. The wtype path rejects NUL and more than 131071 UTF-8 bytes before focus or indicators. A successful type_text result confirms dispatch only; every text backend leaves field contents unverified. Re-observe the target before retrying. Screenshot, click, and input results warn when the target window or coordinate is partially or fully off-screen; use move_window/resize_window (GNOME Shell extension backend) to bring a window fully on-screen before retrying. scroll accepts the same window targeting and relative coordinates as click. get_app_state returns a compact readiness block by default; pass verbose=true for the full diagnostics dump. Scope get_app_state with app_name_or_bundle_identifier or a window target (window_id, pid, app_id, wm_class, title); without one it returns the whole desktop AT-SPI tree, reports tree_scoped=false, and warns in message, which can flood context. accessibility_tree_truncated=true means the node, depth, or read budget stopped traversal with unread elements left; recover by scoping to a narrower app or window target and raising max_nodes or max_depth (hard caps 2000 and 64), not by lowering max_nodes. Electron apps with only window/menu chrome may need --force-renderer-accessibility to expose renderer content."
)]
impl ServerHandler for ComputerUseLinux {}
fn shell_execution_enabled() -> bool {
shell_execution_enabled_value(env::var(SHELL_ENABLE_ENV).ok().as_deref())
}
#[derive(Debug, Serialize, JsonSchema)]
struct CompletionOutput {
ok: bool,
cue: String,
note: String,
}
async fn send_completion_notification(program: &Path, limit: Duration) -> CompletionOutput {
let mut command = TokioCommand::new(program);
command.args([
"--app-name=computer-use-linux",
"--expire-time=3000",
"Desktop interaction finished",
"The agent has finished its desktop interaction.",
]);
let sent =
crate::command_runner::output_with_timeout(command, "send completion notification", limit)
.await;
match sent {
Ok(output) if output.status.success() => CompletionOutput {
ok: true,
cue: "notification".into(),
note: "Notification submitted; desktop settings control whether it is displayed."
.into(),
},
_ => CompletionOutput {
ok: true,
cue: "skipped".into(),
note:
"Notification unavailable, failed, or timed out. The interaction is still complete."
.into(),
},
}
}
fn shell_execution_enabled_value(value: Option<&str>) -> bool {
value == Some("1")
}
fn valid_environment_name(name: &str) -> bool {
let mut chars = name.chars();
chars
.next()
.is_some_and(|first| first == '_' || first.is_ascii_alphabetic())
&& chars.all(|character| character == '_' || character.is_ascii_alphanumeric())
}
fn inherited_shell_environment_allowed(name: &str) -> bool {
matches!(
name,
"PATH"
| "HOME"
| "USER"
| "LOGNAME"
| "LANG"
| "TERM"
| "XDG_RUNTIME_DIR"
| "DISPLAY"
| "WAYLAND_DISPLAY"
| "DBUS_SESSION_BUS_ADDRESS"
) || name.starts_with("LC_")
}
fn inherited_shell_environment(
variables: impl IntoIterator<Item = (OsString, OsString)>,
) -> Vec<(String, String)> {
variables
.into_iter()
.filter_map(|(name, value)| {
let (Ok(name), Ok(value)) = (name.into_string(), value.into_string()) else {
return None;
};
inherited_shell_environment_allowed(&name).then_some((name, value))
})
.collect()
}
fn shell_command_sha256(command: &str) -> String {
Sha256::digest(command.as_bytes())
.iter()
.map(|byte| format!("{byte:02x}"))
.collect()
}
fn bounded_shell_stream(bytes: &[u8]) -> (String, bool) {
let truncated = bytes.len() > SHELL_RESPONSE_STREAM_BYTES;
let visible = if truncated {
&bytes[..SHELL_RESPONSE_STREAM_BYTES]
} else {
bytes
};
(String::from_utf8_lossy(visible).into_owned(), truncated)
}
fn shell_error_output(
command_sha256: String,
cwd: String,
timeout_seconds: u64,
error: impl Into<String>,
) -> RunShellOutput {
RunShellOutput {
ok: false,
command_sha256,
cwd,
timeout_seconds,
exit_code: None,
stdout: String::new(),
stderr: String::new(),
stdout_truncated: false,
stderr_truncated: false,
error: Some(error.into()),
}
}
async fn execute_shell(params: RunShellParams) -> RunShellOutput {
let command_sha256 = shell_command_sha256(¶ms.command);
let timeout_seconds = params.timeout_seconds.unwrap_or(SHELL_DEFAULT_TIMEOUT_SECS);
let requested_cwd = params.cwd.as_deref().unwrap_or(".").to_string();
if !shell_execution_enabled() {
return shell_error_output(
command_sha256,
requested_cwd,
timeout_seconds,
format!(
"shell execution is disabled; restart the MCP server with {SHELL_ENABLE_ENV}=1 to opt in"
),
);
}
if params.command.trim().is_empty() {
return shell_error_output(
command_sha256,
requested_cwd,
timeout_seconds,
"command must not be empty",
);
}
if params.command.len() > SHELL_MAX_COMMAND_BYTES {
return shell_error_output(
command_sha256,
requested_cwd,
timeout_seconds,
format!("command exceeds the {SHELL_MAX_COMMAND_BYTES}-byte limit"),
);
}
if requested_cwd.len() > SHELL_MAX_CWD_BYTES {
return shell_error_output(
command_sha256,
"<rejected: cwd too long>".to_string(),
timeout_seconds,
format!("cwd exceeds the {SHELL_MAX_CWD_BYTES}-byte limit"),
);
}
if !(1..=SHELL_MAX_TIMEOUT_SECS).contains(&timeout_seconds) {
return shell_error_output(
command_sha256,
requested_cwd,
timeout_seconds,
format!("timeout_seconds must be between 1 and {SHELL_MAX_TIMEOUT_SECS}"),
);
}
if params.env.len() > SHELL_MAX_ENV_ENTRIES {
return shell_error_output(
command_sha256,
requested_cwd,
timeout_seconds,
format!("env contains more than {SHELL_MAX_ENV_ENTRIES} entries"),
);
}
let env_bytes = params
.env
.iter()
.map(|(name, value)| name.len().saturating_add(value.len()))
.sum::<usize>();
if env_bytes > SHELL_MAX_ENV_BYTES {
return shell_error_output(
command_sha256,
requested_cwd,
timeout_seconds,
format!("env exceeds the {SHELL_MAX_ENV_BYTES}-byte limit"),
);
}
if let Some(invalid) = params.env.keys().find(|name| !valid_environment_name(name)) {
return shell_error_output(
command_sha256,
requested_cwd,
timeout_seconds,
format!("invalid environment variable name: {invalid}"),
);
}
let cwd = match std::fs::canonicalize(&requested_cwd) {
Ok(path) if path.is_dir() => path,
Ok(_) => {
return shell_error_output(
command_sha256,
requested_cwd,
timeout_seconds,
"cwd is not a directory",
)
}
Err(error) => {
return shell_error_output(
command_sha256,
requested_cwd,
timeout_seconds,
format!("failed to resolve cwd: {error}"),
)
}
};
let cwd_display = cwd.display().to_string();
let mut child = TokioCommand::new("/bin/sh");
// Do not use a login shell: profile scripts could reintroduce credentials
// after env_clear() and contaminate or prevent the requested command.
child.args(["-c", ¶ms.command]);
child.current_dir(&cwd);
child.env_clear();
let mut inherited_path = false;
for (name, value) in inherited_shell_environment(env::vars_os()) {
inherited_path |= name == "PATH";
child.env(name, value);
}
if !inherited_path {
child.env("PATH", "/usr/local/bin:/usr/bin:/bin");
}
child.envs(¶ms.env);
eprintln!(
"[computer-use-linux] run_shell start sha256={command_sha256} cwd={cwd_display:?} timeout_seconds={timeout_seconds}"
);
match crate::command_runner::output_with_timeout(
child,
"run approved shell command",
Duration::from_secs(timeout_seconds),
)
.await
{
Ok(output) => {
let exit_code = output.status.code();
let (stdout, stdout_truncated) = bounded_shell_stream(&output.stdout);
let (stderr, stderr_truncated) = bounded_shell_stream(&output.stderr);
eprintln!(
"[computer-use-linux] run_shell finish sha256={command_sha256} exit_code={exit_code:?} stdout_bytes={} stderr_bytes={} stdout_truncated={stdout_truncated} stderr_truncated={stderr_truncated}",
output.stdout.len(),
output.stderr.len()
);
RunShellOutput {
ok: output.status.success(),
command_sha256,
cwd: cwd_display,
timeout_seconds,
exit_code,
stdout,
stderr,
stdout_truncated,
stderr_truncated,
error: None,
}
}
Err(error) => {
let error = format!("{error:#}");
eprintln!(
"[computer-use-linux] run_shell error sha256={command_sha256} error={error:?}"
);
shell_error_output(command_sha256, cwd_display, timeout_seconds, error)
}
}
}
pub async fn serve_mcp() -> Result<()> {
let server = ComputerUseLinux::default();
let indicator = Arc::clone(&server.indicator);
let running = server.serve(rmcp::transport::stdio()).await?;
// The handshake is done, so the peer knows the client's name.
indicator.attach(running.peer().clone());
running.waiting().await?;
Ok(())
}
#[derive(Debug, Clone, Serialize, JsonSchema)]
struct ListAppsOutput {
apps: Vec<AppCandidate>,
accessible_apps: Vec<AccessibleAppSummary>,
accessibility_error: Option<String>,
note: String,
}
#[derive(Debug, Clone, Serialize, JsonSchema)]
struct ListWindowsOutput {
backend: String,
windows: Vec<WindowInfo>,
error: Option<String>,
permissions_hint: Option<String>,
note: String,
}
#[derive(Debug, Clone, Serialize, JsonSchema)]
struct FocusedWindowOutput {
backend: String,
focused_window: Option<WindowInfo>,
error: Option<String>,
permissions_hint: Option<String>,
message: String,
}
#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)]
struct ActivateWindowParams {
#[serde(default)]
window_id: Option<u64>,
#[serde(default)]
pid: Option<u32>,
#[serde(default)]
tty: Option<String>,
#[serde(default)]
terminal_pid: Option<u32>,
#[serde(default)]
terminal_command: Option<String>,
#[serde(default)]
terminal_cwd: Option<String>,
#[serde(default)]
app_id: Option<String>,
#[serde(default)]
wm_class: Option<String>,
#[serde(default)]
title: Option<String>,
}
impl ActivateWindowParams {
fn into_target(self) -> WindowTarget {
WindowTarget {
window_id: self.window_id,
pid: self.pid,
tty: self.tty,
terminal_pid: self.terminal_pid,
terminal_command: self.terminal_command,
terminal_cwd: self.terminal_cwd,
app_id: self.app_id,
wm_class: self.wm_class,
title: self.title,
}
}
}
#[derive(Debug, Clone, Serialize, JsonSchema)]
struct ActivateWindowOutput {
ok: bool,
implemented: bool,
backend: String,
focus: Option<WindowFocusResult>,
error: Option<String>,
permissions_hint: Option<String>,
// Echo of the request for debugging. `serde_json::Value` has no fixed JSON
// schema, which strict MCP clients (Claude Code) reject in `outputSchema` —
// and one invalid tool fails the whole tool list. Keep it in the runtime
// response (serde) but omit it from the generated schema.
#[schemars(skip)]
received: Option<serde_json::Value>,
}
#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)]
struct MoveWindowParams {
#[serde(flatten)]
target: ActivateWindowParams,
/// New frame-left in desktop coordinates.
x: i32,
/// New frame-top in desktop coordinates.
y: i32,
}
#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)]
struct ResizeWindowParams {
#[serde(flatten)]
target: ActivateWindowParams,
/// New frame width in desktop pixels.
width: i32,
/// New frame height in desktop pixels.
height: i32,
}
#[derive(Debug, Clone, Serialize, JsonSchema)]
struct WindowGeometryOutput {
ok: bool,
implemented: bool,
backend: String,
/// Post-operation window info (compositor-final geometry).
window: Option<WindowInfo>,
message: String,
permissions_hint: Option<String>,
#[schemars(skip)]
received: Option<serde_json::Value>,
}
#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)]
struct RunShellParams {
/// Shell program text passed to `/bin/sh -c` without loading login profiles.
command: String,
/// Existing directory in which to start the shell. Symlinks are resolved.
#[serde(default)]
cwd: Option<String>,
/// Additional environment entries. The ambient process environment is not inherited wholesale.
#[serde(default)]
env: BTreeMap<String, String>,
/// Wall-clock timeout in seconds (default 30, maximum 120).
#[serde(default)]
timeout_seconds: Option<u64>,
}
#[derive(Debug, Clone, Serialize, JsonSchema)]
struct RunShellOutput {
ok: bool,
command_sha256: String,
cwd: String,
timeout_seconds: u64,
exit_code: Option<i32>,
stdout: String,
stderr: String,
stdout_truncated: bool,
stderr_truncated: bool,
error: Option<String>,
}
#[derive(Debug, Clone, Serialize, JsonSchema)]
struct AppCandidate {
name: String,
pid: u32,
command: String,
}
#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)]
struct GetAppStateParams {
/// App name or AT-SPI id that limits the accessibility tree. Omit only when
/// you need the whole desktop tree; unscoped results can flood context.
#[serde(default)]
app_name_or_bundle_identifier: Option<String>,
/// Compositor window id. Also scopes the accessibility tree to that window's
/// application when possible.
#[serde(default)]
window_id: Option<u64>,
/// Process id. Also scopes the accessibility tree to that process when it
/// exposes AT-SPI.
#[serde(default)]
pid: Option<u32>,
/// Terminal tty device (for example /dev/pts/3). Resolves a window target
/// and scopes the tree when possible.
#[serde(default)]
tty: Option<String>,
/// Terminal emulator pid. Resolves a window target and scopes the tree when
/// possible.
#[serde(default)]
terminal_pid: Option<u32>,
/// Terminal command substring. Resolves a window target and scopes the tree
/// when possible.
#[serde(default)]
terminal_command: Option<String>,
/// Terminal working directory. Resolves a window target and scopes the tree
/// when possible.
#[serde(default)]
terminal_cwd: Option<String>,
/// Application id. Also scopes the accessibility tree when it matches an
/// AT-SPI root.
#[serde(default)]
app_id: Option<String>,
/// Window manager class. Also scopes the accessibility tree when it matches
/// an AT-SPI root.
#[serde(default)]
wm_class: Option<String>,
/// Window title substring. Also scopes the accessibility tree when it
/// matches an AT-SPI root.
#[serde(default)]
title: Option<String>,
/// Maximum raw AT-SPI nodes to inspect before compaction (default 1000, hard max 2000).
#[serde(default)]
max_nodes: Option<usize>,
/// Maximum AT-SPI traversal depth (default 32, hard max 64).
#[serde(default)]
max_depth: Option<u32>,
/// Include a size-bounded screenshot (default true). Set false when the
/// accessibility tree is enough.
#[serde(default)]
include_screenshot: Option<bool>,
/// Maximum returned screenshot width in pixels (default 1920, hard-capped).
#[serde(default)]
max_width: Option<u32>,
/// Maximum returned screenshot height in pixels (default 1920, hard-capped).
#[serde(default)]
max_height: Option<u32>,
/// Maximum returned screenshot image bytes before base64 (default 2 MiB, hard-capped).
#[serde(default)]
max_bytes: Option<usize>,
/// Additional downscale factor from 0.0 to 1.0, applied before max dimensions.
#[serde(default)]
scale: Option<f32>,
/// Output image format (default png). Use jpeg with quality to trade exact pixels for smaller payloads.
#[serde(default)]
format: Option<ScreenshotOutputFormat>,
/// JPEG quality from 1 to 95 (default 80). Ignored for png.
#[serde(default)]
#[schemars(range(min = 1, max = 95))]
quality: Option<u8>,
/// Include the full diagnostics report (large). Default false: only the
/// compact readiness block is returned.
#[serde(default)]
verbose: Option<bool>,
}
impl GetAppStateParams {
fn window_target(&self) -> WindowTarget {
WindowTarget {
window_id: self.window_id,
pid: self.pid,
tty: self.tty.clone(),
terminal_pid: self.terminal_pid,
terminal_command: self.terminal_command.clone(),
terminal_cwd: self.terminal_cwd.clone(),
app_id: self.app_id.clone(),
wm_class: self.wm_class.clone(),
title: self.title.clone(),
}
}
fn screenshot_options(&self) -> ScreenshotPayloadOptions {
ScreenshotPayloadOptions {
max_width: self.max_width,
max_height: self.max_height,
max_bytes: self.max_bytes,
scale: self.scale,
format: self.format,
quality: self.quality,
}
}
}
#[derive(Debug, Clone, Default, Deserialize, Serialize, JsonSchema)]
struct ScreenshotParams {
#[serde(default)]
window_id: Option<u64>,
#[serde(default)]
pid: Option<u32>,
#[serde(default)]
app_id: Option<String>,
#[serde(default)]
wm_class: Option<String>,
#[serde(default)]
title: Option<String>,
/// Raise the targeted window before capture (default true). Ignored without
/// a window target.
#[serde(default)]
raise_window: Option<bool>,
/// Capture the whole desktop even when a window is targeted (default false).
#[serde(default)]
full_screen: Option<bool>,
/// Maximum returned screenshot width in pixels (default 1920, hard-capped).
#[serde(default)]
max_width: Option<u32>,
/// Maximum returned screenshot height in pixels (default 1920, hard-capped).
#[serde(default)]
max_height: Option<u32>,
/// Maximum returned screenshot image bytes before base64 (default 2 MiB, hard-capped).
#[serde(default)]
max_bytes: Option<usize>,
/// Additional downscale factor from 0.0 to 1.0, applied before max dimensions.
#[serde(default)]
scale: Option<f32>,
/// Output image format (default png). Use jpeg with quality to trade exact pixels for smaller payloads.
#[serde(default)]
format: Option<ScreenshotOutputFormat>,
/// JPEG quality from 1 to 95 (default 80). Ignored for png.
#[serde(default)]
#[schemars(range(min = 1, max = 95))]
quality: Option<u8>,
}
impl ScreenshotParams {
fn window_target(&self) -> Option<WindowTarget> {
if self.window_id.is_none()
&& self.pid.is_none()
&& self.app_id.is_none()
&& self.wm_class.is_none()
&& self.title.is_none()
{
return None;
}
Some(WindowTarget {
window_id: self.window_id,
pid: self.pid,
tty: None,
terminal_pid: None,
terminal_command: None,
terminal_cwd: None,
app_id: self.app_id.clone(),
wm_class: self.wm_class.clone(),
title: self.title.clone(),
})
}
fn screenshot_options(&self) -> ScreenshotPayloadOptions {
ScreenshotPayloadOptions {
max_width: self.max_width,
max_height: self.max_height,
max_bytes: self.max_bytes,
scale: self.scale,
format: self.format,
quality: self.quality,
}
}
}
/// Screenshot metadata for `get_app_state`. The image bytes travel as a
/// separate `image` content block (issue #145), not inside this JSON.
#[derive(Debug, Clone, Serialize, JsonSchema)]
struct ScreenshotSummary {
mime_type: String,
source: String,
/// Width of the returned image payload.
width: u32,
/// Height of the returned image payload.
height: u32,
/// Coordinate-space width before payload downscaling.
coordinate_width: u32,
/// Coordinate-space height before payload downscaling.
coordinate_height: u32,
/// Returned pixels per coordinate-space pixel.
scale: f32,
resized: bool,
bytes: usize,
original_bytes: usize,
max_bytes: usize,
format: ScreenshotOutputFormat,
quality: Option<u8>,
}
impl From<&ScreenshotCapture> for ScreenshotSummary {
fn from(capture: &ScreenshotCapture) -> Self {
Self {
mime_type: capture.mime_type.clone(),
source: capture.source.clone(),
width: capture.width,
height: capture.height,
coordinate_width: capture.coordinate_width,
coordinate_height: capture.coordinate_height,
scale: capture.scale,
resized: capture.resized,
bytes: capture.bytes,
original_bytes: capture.original_bytes,
max_bytes: capture.max_bytes,
format: capture.format,
quality: capture.quality,
}
}
}
/// Assemble the `get_app_state` result: the screenshot (when present) as a
/// structured image block first, then the JSON report as text and as
/// `structured_content`. Inlining the base64 in the JSON made hosts treat a
/// 100KB+ image as plain text tokens (issue #145).
fn app_state_result(
output: GetAppStateOutput,
screenshot: Option<ScreenshotCapture>,
) -> Result<CallToolResult, ErrorData> {
let value = serde_json::to_value(&output).map_err(|error| {
ErrorData::internal_error(
format!("failed to serialize get_app_state output: {error}"),
None,
)
})?;
let mut result = CallToolResult::structured(value);
if let Some(capture) = screenshot {
result.content.insert(
0,
ContentBlock::image(data_url_payload(&capture.data_url), capture.mime_type),
);
}
Ok(result)
}
#[derive(Debug, Clone, Serialize, JsonSchema)]
struct GetAppStateOutput {
app_name_or_bundle_identifier: Option<String>,
window_context: Option<WindowInfo>,
window_error: Option<String>,
window_permissions_hint: Option<String>,
backend: String,
/// Screenshot metadata; the image itself is the first `image` content block.
screenshot: Option<ScreenshotSummary>,
screenshot_error: Option<String>,
accessibility_tree: Vec<AccessibilityNode>,
accessibility_tree_raw_count: usize,
/// True when the snapshot was limited to selected app roots rather than the
/// full desktop AT-SPI registry.
tree_scoped: bool,
/// True when max_nodes, max_depth, or the child read budget stopped
/// traversal with unread elements left, so the tree is incomplete. Recover
/// by scoping to a narrower app or window target and raising max_nodes or
/// max_depth (hard caps 2000 and 64). Failed element reads do not count.
accessibility_tree_truncated: bool,
accessibility_error: Option<String>,
/// Compact readiness summary (always present).
readiness: crate::diagnostics::ReadinessReport,
/// Full diagnostics; populated only when verbose=true.
#[serde(skip_serializing_if = "Option::is_none")]
diagnostics: Option<DoctorReport>,
message: String,
}
#[derive(Debug, Clone, Default, Deserialize, Serialize, JsonSchema)]
struct ClickParams {
#[serde(default)]
element_index: Option<u32>,
#[serde(default)]
role: Option<String>,
#[serde(default)]
name: Option<String>,
#[serde(default)]
text: Option<String>,
#[serde(default)]
states: Vec<String>,
#[serde(default)]
x: Option<i32>,
#[serde(default)]
y: Option<i32>,
#[serde(default)]
button: Option<String>,
#[serde(default)]
click_count: Option<u32>,
// Optional window target: when set, the window is raised/focused before the
// click so a coordinate click reliably lands on the intended app rather than
// whatever window happens to be stacked on top at that pixel.
#[serde(default)]
window_id: Option<u64>,
#[serde(default)]
pid: Option<u32>,
#[serde(default)]
app_id: Option<String>,
#[serde(default)]
wm_class: Option<String>,
#[serde(default)]
window_title: Option<String>,
/// Interpret `x`/`y` from the clipped window screenshot crop origin, in
/// coordinate pixels before preview resizing. Divide preview pixels by its
/// scale first. This is not a toolkit widget or raw GDK surface origin.
/// Requires a window target; missing targets are rejected.
#[serde(default)]
relative: Option<bool>,
}
impl ClickParams {
/// A window target if any window-identifying field was supplied.
fn window_target(&self) -> Option<WindowTarget> {
if self.window_id.is_none()
&& self.pid.is_none()
&& self.app_id.is_none()
&& self.wm_class.is_none()
&& self.window_title.is_none()
{
return None;
}
Some(WindowTarget {
window_id: self.window_id,
pid: self.pid,
tty: None,
terminal_pid: None,
terminal_command: None,
terminal_cwd: None,
app_id: self.app_id.clone(),
wm_class: self.wm_class.clone(),
title: self.window_title.clone(),
})
}
fn selector(&self) -> ElementSelector<'_> {
ElementSelector {
role: self.role.as_deref(),
name: self.name.as_deref(),
text: self.text.as_deref(),
states: &self.states,
}
}
}
#[derive(Debug, Clone, Default, Deserialize, Serialize, JsonSchema)]
struct ActionParams {
#[serde(default)]
element_index: Option<u32>,
#[serde(default)]
element_identifier: Option<String>,
#[serde(default)]
role: Option<String>,
#[serde(default)]
name: Option<String>,
#[serde(default)]
text: Option<String>,
#[serde(default)]
states: Vec<String>,
#[serde(default)]
action: Option<String>,
}
impl ActionParams {
fn selector(&self) -> ElementSelector<'_> {
ElementSelector {
role: self.role.as_deref(),
name: self.name.as_deref(),
text: self.text.as_deref(),
states: &self.states,
}
}
}
#[derive(Debug, Clone, Default, Deserialize, Serialize, JsonSchema)]
struct SetValueParams {
#[serde(default)]
element_index: Option<u32>,
#[serde(default)]
element_identifier: Option<String>,
#[serde(default)]
role: Option<String>,
#[serde(default)]
name: Option<String>,
#[serde(default)]
text: Option<String>,
#[serde(default)]
states: Vec<String>,
value: String,
}
impl SetValueParams {
fn selector(&self) -> ElementSelector<'_> {
ElementSelector {
role: self.role.as_deref(),
name: self.name.as_deref(),
text: self.text.as_deref(),
states: &self.states,
}
}
}
#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)]
struct ScrollParams {
#[serde(default)]
element_index: Option<u32>,
#[serde(default)]
x: Option<i32>,
#[serde(default)]
y: Option<i32>,
direction: String,
#[serde(default)]
#[schemars(range(min = -100, max = 100))]
pages: Option<f64>,
// Optional window target (parity with click): the window is raised/focused
// before scrolling so the wheel events land on the intended app.
#[serde(default)]
window_id: Option<u64>,
#[serde(default)]
pid: Option<u32>,
#[serde(default)]
app_id: Option<String>,
#[serde(default)]
wm_class: Option<String>,
#[serde(default)]
window_title: Option<String>,
/// Interpret `x`/`y` from the clipped window screenshot crop origin, in
/// coordinate pixels before preview resizing. Divide preview pixels by its
/// scale first. This is not a toolkit widget or raw GDK surface origin.
/// Requires a window target; missing targets are rejected.
#[serde(default)]
relative: Option<bool>,
}
fn scroll_units(pages: Option<f64>) -> std::result::Result<i32, String> {
let pages = pages.unwrap_or(1.0);
if !pages.is_finite() || pages.abs() > MAX_SCROLL_PAGES {
return Err(format!(
"pages must be finite and have an absolute value no greater than {MAX_SCROLL_PAGES}"
));
}
Ok(((pages.abs().max(0.1) * 5.0).round() as i32).max(1))
}
impl ScrollParams {
/// A window target if any window-identifying field was supplied.
fn window_target(&self) -> Option<WindowTarget> {
if self.window_id.is_none()
&& self.pid.is_none()
&& self.app_id.is_none()
&& self.wm_class.is_none()
&& self.window_title.is_none()
{
return None;
}
Some(WindowTarget {
window_id: self.window_id,
pid: self.pid,
tty: None,
terminal_pid: None,
terminal_command: None,
terminal_cwd: None,
app_id: self.app_id.clone(),
wm_class: self.wm_class.clone(),
title: self.window_title.clone(),
})
}
}
#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)]
struct DragParams {
start_x: i32,
start_y: i32,
end_x: i32,
end_y: i32,
}
#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)]
struct PressKeyParams {
key: String,
#[serde(default)]
window_id: Option<u64>,
#[serde(default)]
pid: Option<u32>,
#[serde(default)]
tty: Option<String>,
#[serde(default)]
terminal_pid: Option<u32>,
#[serde(default)]
terminal_command: Option<String>,
#[serde(default)]
terminal_cwd: Option<String>,
#[serde(default)]
app_id: Option<String>,
#[serde(default)]
wm_class: Option<String>,
#[serde(default)]
title: Option<String>,
}
#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)]
struct TypeTextParams {
text: String,
#[serde(default)]
window_id: Option<u64>,
#[serde(default)]
pid: Option<u32>,
#[serde(default)]
tty: Option<String>,
#[serde(default)]
terminal_pid: Option<u32>,
#[serde(default)]
terminal_command: Option<String>,
#[serde(default)]
terminal_cwd: Option<String>,
#[serde(default)]
app_id: Option<String>,
#[serde(default)]
wm_class: Option<String>,
#[serde(default)]
title: Option<String>,
}
impl PressKeyParams {
fn window_target(&self) -> WindowTarget {
WindowTarget {
window_id: self.window_id,
pid: self.pid,
tty: self.tty.clone(),
terminal_pid: self.terminal_pid,
terminal_command: self.terminal_command.clone(),
terminal_cwd: self.terminal_cwd.clone(),
app_id: self.app_id.clone(),
wm_class: self.wm_class.clone(),
title: self.title.clone(),
}
}
}
impl TypeTextParams {
fn window_target(&self) -> WindowTarget {
WindowTarget {
window_id: self.window_id,
pid: self.pid,
tty: self.tty.clone(),
terminal_pid: self.terminal_pid,
terminal_command: self.terminal_command.clone(),
terminal_cwd: self.terminal_cwd.clone(),
app_id: self.app_id.clone(),
wm_class: self.wm_class.clone(),
title: self.title.clone(),
}
}
}
#[derive(Debug, Clone, Serialize, JsonSchema)]
struct ActionOutput {
ok: bool,
implemented: bool,
action: String,
message: String,
// See ActivateWindowOutput: kept in the response, omitted from the schema
// because `serde_json::Value` produces a non-object schema strict MCP
// clients reject.
#[schemars(skip)]
received: Option<serde_json::Value>,
}
impl ComputerUseLinux {
fn is_wayland_session(&self) -> bool {
crate::diagnostics::hydrate_session_bus_env();
let session_type = env::var("XDG_SESSION_TYPE").ok();
let wayland_display = env::var("WAYLAND_DISPLAY").ok();
session_is_wayland(session_type.as_deref(), wayland_display.as_deref())
}
// The Wayland remote-desktop portal is a fallback for input: when a
// compatible ydotool CLI and working `ydotoold` socket are present we
// prefer ydotool, because it injects input without a permission prompt.
// Without `COMPUTER_USE_LINUX_PERSIST_REMOTE_DESKTOP=1` the portal asks
// again on every new process. That opt-in sends `persist_mode=2` on
// RemoteDesktop SelectDevices and reuses the single-use restore token.
// `COMPUTER_USE_LINUX_FORCE_YDOTOOL_*=1` always uses ydotool;
// `COMPUTER_USE_LINUX_FORCE_PORTAL_*=1` always uses the portal.
async fn should_prefer_portal_pointer_backend(&self) -> bool {
if env_flag_enabled("COMPUTER_USE_LINUX_FORCE_YDOTOOL_POINTER") {
return false;
}
if env_flag_enabled("COMPUTER_USE_LINUX_FORCE_PORTAL_POINTER") {
return self.is_wayland_session();
}
should_prefer_portal_backend_by_default(
self.is_wayland_session(),
ydotool_backend_available().await,
)
}
// Literal text needs a layout-aware route even when raw ydotool input is
// available. KDE uses clipboard paste; compatible compositors use wtype.
fn should_prefer_portal_text_backend(&self) -> bool {
prefer_portal_text_backend(
env_flag_enabled("COMPUTER_USE_LINUX_FORCE_YDOTOOL_KEYBOARD"),
self.should_prefer_xdotool_keyboard(),
self.is_wayland_session(),
self.is_kde_wayland_session(),
env_flag_enabled("COMPUTER_USE_LINUX_FORCE_PORTAL_KEYBOARD"),
self.should_prefer_wtype_keyboard(),
)
}
/// Portal keyboard policy for `press_key` chords. Unlike literal text
/// (where KDE prefers the clipboard paste backend), key chords have no
/// clipboard route, so the portal keyboard session is preferred on ANY
/// Wayland session — including Plasma. An already-active keyboard
/// session (e.g. established by a KDE clipboard paste) is reused even
/// when ydotool is available, so the consent the user already granted
/// keeps covering key chords; otherwise the portal is preferred only
/// when ydotool is absent or the portal is forced.
async fn should_prefer_portal_keyboard_for_chords(&self) -> bool {
if env_flag_enabled("COMPUTER_USE_LINUX_FORCE_YDOTOOL_KEYBOARD") {
return false;
}
if self.should_prefer_xdotool_keyboard() {
return false;
}
if !self.is_wayland_session() {
return false;
}
if self.cached_portal_keyboard_session().is_some()
|| env_flag_enabled("COMPUTER_USE_LINUX_FORCE_PORTAL_KEYBOARD")
{
return true;
}
!ydotool_backend_available().await
}
fn should_prefer_kde_clipboard_text_backend(&self) -> bool {
!env_flag_enabled("COMPUTER_USE_LINUX_FORCE_YDOTOOL_KEYBOARD")
&& !self.should_prefer_xdotool_keyboard()
&& self.is_kde_wayland_session()
}
/// Keyboard policy for X11 sessions: prefer `xdotool` (XTEST).
///
/// ydotool writes raw evdev scancodes to a virtual uinput device. On X11
/// the server then re-interprets them through the active XKB layout, so
/// `press_key "Return"` and chords like `ctrl+a` land as stray characters,
/// and literal text can mangle symbols/digits (issue #58). XTEST resolves
/// keysyms against the live layout instead.
///
/// `COMPUTER_USE_LINUX_FORCE_YDOTOOL_KEYBOARD=1` opts out;
/// `COMPUTER_USE_LINUX_FORCE_XDOTOOL_KEYBOARD=1` forces it on.
fn should_prefer_xdotool_keyboard(&self) -> bool {
prefer_xdotool_keyboard(
env_flag_enabled("COMPUTER_USE_LINUX_FORCE_YDOTOOL_KEYBOARD"),
env_flag_enabled("COMPUTER_USE_LINUX_FORCE_XDOTOOL_KEYBOARD"),
self.is_wayland_session(),
env_var_non_empty("DISPLAY"),
xdotool_available(),
)
}
fn should_prefer_wtype_keyboard(&self) -> bool {
prefer_wtype_keyboard(
env_flag_enabled("COMPUTER_USE_LINUX_FORCE_YDOTOOL_KEYBOARD"),
self.is_wayland_session(),
crate::diagnostics::wtype_compatible_wayland_desktop(
env::var("XDG_CURRENT_DESKTOP").ok().as_deref(),
),
wtype_available(),
)
}
fn wtype_text_mode(&self) -> Option<WtypeTextMode> {
if self.should_prefer_xdotool_keyboard()
|| self.should_prefer_kde_clipboard_text_backend()
|| self.should_prefer_portal_text_backend()
|| !self.should_prefer_wtype_keyboard()
{
return None;
}
Some(wtype_text_mode_for_desktop(
env::var("XDG_CURRENT_DESKTOP").ok().as_deref(),
))
}
fn should_prefer_xdotool_pointer(&self) -> bool {
crate::diagnostics::hydrate_session_bus_env();
prefer_xdotool_pointer(
env_flag_enabled("COMPUTER_USE_LINUX_FORCE_YDOTOOL_POINTER"),
env::var("XDG_SESSION_TYPE").ok().as_deref(),
env_var_non_empty("DISPLAY"),
env::var("WAYLAND_DISPLAY").ok().as_deref(),
xdotool_available(),
)
}
fn is_gnome_wayland_session(&self) -> bool {
self.is_wayland_session()
&& (env_contains("XDG_CURRENT_DESKTOP", "gnome")
|| env_contains("DESKTOP_SESSION", "gnome")
|| env_contains("XDG_SESSION_DESKTOP", "gnome"))
}
fn is_kde_wayland_session(&self) -> bool {
self.is_wayland_session()
&& (env_contains("XDG_CURRENT_DESKTOP", "kde")
|| env_contains("DESKTOP_SESSION", "plasma"))
}
fn cached_portal_pointer_session(&self) -> Option<PortalPointerSession> {
if env_flag_enabled("COMPUTER_USE_LINUX_FORCE_YDOTOOL_POINTER") {
return None;
}
let mut cached = self.portal_pointer_session.lock().ok()?;
if cached.as_ref().is_some_and(|session| !session.is_valid()) {
*cached = None;
}
cached.clone()
}
fn clear_portal_pointer_session(&self, failed: &PortalPointerSession) {
failed.invalidate_and_close();
if let Ok(mut cached) = self.portal_pointer_session.lock() {
if cached
.as_ref()
.is_some_and(|session| session.same_session(failed))
{
*cached = None;
}
}
}
fn cached_portal_keyboard_session(&self) -> Option<PortalKeyboardSession> {
let mut cached = self.portal_keyboard_session.lock().ok()?;
if cached.as_ref().is_some_and(|session| !session.is_valid()) {
*cached = None;
}
cached.clone()
}
fn clear_portal_keyboard_session(&self, failed: &PortalKeyboardSession) {
failed.invalidate_and_close();
if let Ok(mut cached) = self.portal_keyboard_session.lock() {
if cached
.as_ref()
.is_some_and(|session| session.same_session(failed))
{
*cached = None;
}
}
}
async fn ensure_portal_pointer_session(&self) -> Result<Option<PortalPointerSession>> {
if !self.should_prefer_portal_pointer_backend().await {
return Ok(None);
}
if let Some(session) = self.cached_portal_pointer_session() {
return Ok(Some(session));
}
let _guard = self.portal_session_init_lock.lock().await;
if let Some(session) = self.cached_portal_pointer_session() {
return Ok(Some(session));
}
let session = start_portal_pointer_session().await?;
if let Ok(mut cached) = self.portal_pointer_session.lock() {
*cached = Some(session.clone());
}
Ok(Some(session))
}
async fn ensure_portal_keyboard_session(&self) -> Result<Option<PortalKeyboardSession>> {
if env_flag_enabled("COMPUTER_USE_LINUX_FORCE_YDOTOOL_KEYBOARD")
|| !self.is_wayland_session()
{
return Ok(None);
}
if let Some(session) = self.cached_portal_keyboard_session() {
return Ok(Some(session));
}
let _guard = self.portal_session_init_lock.lock().await;
if let Some(session) = self.cached_portal_keyboard_session() {
return Ok(Some(session));
}
let session = start_portal_keyboard_session().await?;
if let Ok(mut cached) = self.portal_keyboard_session.lock() {
*cached = Some(session.clone());
}
Ok(Some(session))
}
async fn resolve_window_context(
&self,
params: &GetAppStateParams,
) -> (Option<WindowInfo>, Option<String>, Option<String>) {
let target = params.window_target();
if !target.has_target() {
return (None, None, None);
}
match list_windows().await {
Ok(windows) => match resolve_window_target(&windows, &target) {
Ok(window) => (Some(window.clone()), None, None),
Err(error) => (None, Some(format!("{error:#}")), None),
},
Err(error) => {
let error = format!("{error:#}");
let hint = window_permission_hint(&error);
(None, Some(error), hint)
}
}
}
async fn resolve_screenshot_window(
&self,
target: &WindowTarget,
raise_window: bool,
) -> Result<WindowInfo> {
let window = if raise_window {
let focus = focus_window_target(target).await?;
if !focus.exact_window_focused {
anyhow::bail!(
"the requested window could not be focused exactly; refusing to capture unrelated desktop pixels"
);
}
sleep(Duration::from_millis(250)).await;
focus
.focused_window
.filter(|window| window.window_id == focus.requested_window.window_id)
.ok_or_else(|| anyhow::anyhow!("focused-window verification returned no window"))?
} else {
let windows = list_windows().await?;
let window = resolve_window_target(&windows, target)?.clone();
if !window.focused {
anyhow::bail!(
"raise_window=false requires the requested window to already be focused"
);
}
window
};
if window.hidden {
anyhow::bail!("the requested window is hidden or minimized");
}
Ok(window)
}
async fn window_crop_rect_for_capture(
&self,
window: &WindowInfo,
raw: &RawScreenshotCapture,
) -> Result<(i32, i32, u32, u32)> {
self.window_crop_rect_for_dimensions(window, raw.width, raw.height)
.await
}
async fn window_crop_rect_for_dimensions(
&self,
window: &WindowInfo,
capture_width: u32,
capture_height: u32,
) -> Result<(i32, i32, u32, u32)> {
Ok(self
.window_coordinate_map_for_dimensions(window, capture_width, capture_height)
.await?
.capture_rect)
}
async fn window_coordinate_map_for_dimensions(
&self,
window: &WindowInfo,
capture_width: u32,
capture_height: u32,
) -> Result<WindowCoordinateMap> {
let bounds = window.bounds.as_ref().ok_or_else(|| {
anyhow::anyhow!(
"targeted screenshot requires window bounds; refusing to return the full desktop"
)
})?;
let logical_rect = window_crop_rect(bounds).ok_or_else(|| {
anyhow::anyhow!(
"targeted screenshot has unusable window bounds; refusing to return the full desktop"
)
})?;
let monitors = if window.backend == GNOME_SHELL_EXTENSION_BACKEND {
Some(
crate::windowing::backends::gnome::extension_monitor_layout()
.await
.map_err(|error| {
anyhow::anyhow!(
"GNOME targeted screenshot requires logical monitor geometry: {error:#}"
)
})?
.into_iter()
.map(|monitor| (monitor.x, monitor.y, monitor.width, monitor.height))
.collect(),
)
} else if window.backend == GNOME_SHELL_INTROSPECT_BACKEND {
crate::windowing::backends::gnome::extension_monitor_layout()
.await
.ok()
.map(|monitors| {
monitors
.into_iter()
.map(|monitor| (monitor.x, monitor.y, monitor.width, monitor.height))
.collect()
})
} else if window.backend == KWIN_BACKEND {
Some(vec![
crate::windowing::backends::kwin::logical_desktop_rect()
.await
.map_err(|error| {
anyhow::anyhow!(
"KWin targeted screenshot requires logical workspace geometry: {error:#}"
)
})?,
])
} else {
None
};
let (full_capture_rect, portal_rect) = match monitors {
Some(monitors) => (
logical_window_crop_rect(bounds, &monitors, capture_width, capture_height)?,
Some(logical_rect),
),
None => (logical_rect, None),
};
Ok(WindowCoordinateMap {
capture_rect: clip_capture_rect(full_capture_rect, capture_width, capture_height)?,
full_capture_rect,
portal_rect,
})
}
async fn focused_window_coordinate_map(
&self,
focus: &WindowFocusResult,
) -> std::result::Result<WindowCoordinateMap, String> {
let window = focus
.focused_window
.as_ref()
.unwrap_or(&focus.requested_window);
if !matches!(
window.backend.as_str(),
GNOME_SHELL_EXTENSION_BACKEND | GNOME_SHELL_INTROSPECT_BACKEND | KWIN_BACKEND
) {
let full_capture_rect = window
.bounds
.as_ref()
.and_then(window_crop_rect)
.ok_or_else(|| {
"Window-relative coordinates require usable target-window bounds.".to_string()
})?;
let capture_rect = self
.desktop_size
.lock()
.ok()
.and_then(|guard| *guard)
.map(|(width, height)| {
clip_capture_rect(full_capture_rect, width, height).map_err(|error| {
format!("Could not map target-window coordinates: {error:#}")
})
})
.transpose()?
.unwrap_or(full_capture_rect);
return Ok(WindowCoordinateMap {
capture_rect,
full_capture_rect,
portal_rect: None,
});
}
let (_, _, width, height) = self.capture_space_rect().await.ok_or_else(|| {
"Could not determine screenshot dimensions for window-relative coordinates.".to_string()
})?;
self.window_coordinate_map_for_dimensions(window, width as u32, height as u32)
.await
.map_err(|error| format!("Could not map target-window coordinates: {error:#}"))
}
async fn resolve_accessibility_app_filter(
&self,
params: &GetAppStateParams,
window_context: Option<&WindowInfo>,
) -> Option<String> {
if let Some(explicit) = trimmed_nonempty(params.app_name_or_bundle_identifier.as_deref()) {
return Some(explicit.to_string());
}
let target_pid = window_context.and_then(|window| window.pid).or(params.pid);
let candidates = accessibility_filter_candidates(window_context);
if let Some(target_pid) = target_pid {
if let Ok(apps) = list_accessible_apps(200).await {
if let Some(object_ref) =
select_accessibility_object_ref(&apps, target_pid, &candidates)
{
return Some(object_ref);
}
}
}
candidates.into_iter().next()
}
async fn focus_target_for_input(
&self,
target: &WindowTarget,
) -> std::result::Result<Option<WindowFocusResult>, String> {
if !target.has_target() {
return Ok(None);
}
let focus = focus_window_target(target).await.map_err(|error| {
let error = format!("{error:#}");
if let Some(hint) = window_permission_hint(&error) {
format!("Did not send input because the target window could not be focused: {error}. {hint}")
} else {
format!("Did not send input because the target window could not be focused: {error}")
}
})?;
if focus_satisfies_target(&focus, target) {
Ok(Some(focus))
} else {
let required = if target.requires_exact_focus() {
"exact target-window focus"
} else {
"app-level focus"
};
Err(format!(
"Did not send input because {required} verification failed after activating the target window. Focus result: requested window_id {}, focused window_id {:?}.",
focus.requested_window.window_id,
focus.focused_window.as_ref().map(|window| window.window_id)
))
}
}
fn cache_desktop_size(&self, width: u32, height: u32) {
if width == 0 || height == 0 {
return;
}
if let Ok(mut guard) = self.desktop_size.lock() {
*guard = Some((width, height));
}
}
fn logical_portal_point(
&self,
session: &PortalPointerSession,
x: i32,
y: i32,
) -> Option<(i32, i32)> {
let capture_size = self.desktop_size.lock().ok().and_then(|guard| *guard);
session.logical_point_from_capture(x, y, capture_size)
}
/// Capture size for overlay points, which are in screenshot pixels like
/// the click tools. Only a cached size is used: capturing the screen just
/// for the overlay could prompt the user.
fn indicator_space(&self) -> Option<(u32, u32)> {
*self.desktop_size.lock().ok()?
}
/// Cosmetic secrecy probe: timeout only masks the indicator, never gates input.
async fn typing_into_secret(&self, focus: Option<&WindowFocusResult>) -> bool {
if !self.indicator.enabled() {
return true;
}
timeout(Duration::from_millis(250), self.input_focus_probe(focus))
.await
.ok()
.flatten()
.as_ref()
.is_none_or(focus_requires_secret_protection)
}
/// Probe only the receiving app; the caller chooses its own time budget.
async fn input_focus_probe(&self, focus: Option<&WindowFocusResult>) -> Option<FocusProbe> {
let pid = match focus {
Some(focus) => focus
.focused_window
.as_ref()
.and_then(|window| window.pid)
.or(focus.requested_window.pid),
None => focused_window().await.ok().flatten()?.pid,
}?;
probe_focused_element(Some(pid)).await.ok()
}
/// COORDINATE SPACES: window bounds (list_windows / extension frame rects)
/// and the extension monitor layout are in LOGICAL pixels, while click/
/// scroll coordinates and screenshot captures are in PHYSICAL capture
/// pixels. On fractionally-scaled displays the two differ, so each check
/// below only ever compares values from the same space.
///
/// Logical monitor rectangles from the GNOME Shell extension, for checks
/// against logical window bounds. None when the extension is unavailable.
async fn logical_monitor_rects(&self) -> Option<Vec<(i32, i32, i32, i32)>> {
let monitors = crate::windowing::backends::gnome::extension_monitor_layout()
.await
.ok()?;
(!monitors.is_empty()).then(|| {
monitors
.iter()
.map(|m| (m.x, m.y, m.width, m.height))
.collect()
})
}
/// Physical capture-space desktop rectangle (union of monitors as captured
/// by the screenshot pipeline), for checks against click coordinates.
/// Best-effort; None disables the check.
async fn capture_space_rect(&self) -> Option<(i32, i32, i32, i32)> {
let cached = self.desktop_size.lock().ok().and_then(|guard| *guard);
if let Some((w, h)) = cached {
return Some((0, 0, w as i32, h as i32));
}
// One-time prime: a full-frame capture reveals the desktop size when
// no prior capture is available.
let raw = capture_screenshot_raw().await.ok()?;
self.cache_desktop_size(raw.width, raw.height);
(raw.width > 0 && raw.height > 0).then_some((0, 0, raw.width as i32, raw.height as i32))
}
/// Warn when a targeted window pokes outside every monitor: clicks and
/// screenshots silently truncate to visible pixels there, which reads as
/// "success" while landing nowhere.
async fn off_screen_note_for_bounds(
&self,
bounds: &crate::windowing::WindowBounds,
) -> Option<String> {
let (x, y) = bounds.x.zip(bounds.y)?;
if bounds.width == 0 || bounds.height == 0 {
return None;
}
// Window bounds are logical pixels: prefer the extension's logical
// monitor layout (same space). The physical capture rect is a safe
// fallback — on scaled displays it is at least as large as the logical
// union, so it can only under-warn, never false-positive.
let rects = match self.logical_monitor_rects().await {
Some(rects) => rects,
None => vec![self.capture_space_rect().await?],
};
let (w, h) = (bounds.width as i64, bounds.height as i64);
let window_area = w * h;
let mut visible_area = 0_i64;
for (mx, my, mw, mh) in &rects {
let ix = (x as i64).max(*mx as i64);
let iy = (y as i64).max(*my as i64);
let ix2 = (x as i64 + w).min(*mx as i64 + *mw as i64);
let iy2 = (y as i64 + h).min(*my as i64 + *mh as i64);
if ix2 > ix && iy2 > iy {
// Overlapping monitors are rare; treating them as additive keeps
// this a cheap best-effort heuristic.
visible_area += (ix2 - ix) * (iy2 - iy);
}
}
let visible_pct = (visible_area.min(window_area) * 100) / window_area.max(1);
if visible_pct >= 100 {
return None;
}
Some(format!(
"WARNING: the target window (bounds {x},{y} {w}x{h}) is only ~{visible_pct}% on-screen; off-screen regions are missing from screenshots and unreachable by coordinate input. Use move_window/resize_window to bring it fully on-screen."
))
}
/// Warn when a click/scroll coordinate is outside the captured desktop.
/// Click coordinates are physical capture-space pixels, so compare ONLY
/// against the capture rect — the extension's logical layout is a
/// different space on scaled displays and would false-positive. This is a
/// best-effort warning: never request a new capture (and possibly consent)
/// merely to decide whether to append a note to an input result.
async fn off_screen_note_for_point(&self, x: i32, y: i32) -> Option<String> {
let (mw, mh) = self.desktop_size.lock().ok().and_then(|guard| *guard)?;
let visible = x >= 0 && y >= 0 && (x as u32) < mw && (y as u32) < mh;
if visible {
return None;
}
Some(format!(
"WARNING: coordinate {x},{y} is outside the captured desktop ({mw}x{mh}); the input landed on no visible pixel."
))
}
/// Post-input feedback: which AT-SPI element holds keyboard focus in the
/// target app, and whether it is editable. Guards against the blind-typing
/// trap where verified *window* focus still sends keystrokes nowhere.
async fn focused_element_feedback(
&self,
focus: Option<&WindowFocusResult>,
expects_editable: bool,
) -> Option<String> {
let focus = focus?;
let pid = focus
.focused_window
.as_ref()
.and_then(|window| window.pid)
.or(focus.requested_window.pid);
match timeout(Duration::from_millis(1500), probe_focused_element(pid)).await {
Ok(Ok(probe)) => Some(focus_probe_feedback(
&probe,
expects_editable,
focus.exact_window_focused,
)),
Ok(Err(error)) => Some(format!(
"Focused-element feedback unavailable ({}).",
first_line(&format!("{error:#}"))
)),
Err(_) => {
Some("Focused-element feedback unavailable (AT-SPI probe timed out).".to_string())
}
}
}
/// Shared move/resize plumbing: resolve the window target, run the GNOME
/// Shell extension operation, then re-query bounds to report the result.
async fn window_geometry_op<F, Fut>(
&self,
received: Option<serde_json::Value>,
target: &WindowTarget,
op: F,
) -> Json<WindowGeometryOutput>
where
F: FnOnce(crate::windowing::WindowInfo) -> Fut,
Fut: Future<Output = Result<String>>,
{
let windows = match list_windows().await {
Ok(windows) => windows,
Err(error) => {
let error = format!("{error:#}");
return Json(WindowGeometryOutput {
ok: false,
implemented: true,
backend: "unknown".to_string(),
window: None,
message: format!("Window listing failed: {error}"),
permissions_hint: window_permission_hint(&error),
received,
});
}
};
let window = match resolve_window_target(&windows, target) {
Ok(window) => window.clone(),
Err(error) => {
return Json(WindowGeometryOutput {
ok: false,
implemented: true,
backend: "unknown".to_string(),
window: None,
message: format!("{error:#}"),
permissions_hint: None,
received,
});
}
};
let backend = window.backend.clone();
let window_id = window.window_id;
match op(window).await {
Ok(message) => {
// Re-query so the caller sees the compositor-final geometry
// (tiling constraints, minimum sizes, etc. may adjust it).
let window = list_windows().await.ok().and_then(|windows| {
windows
.into_iter()
.find(|window| window.window_id == window_id)
});
let mut message = message;
if let Some(bounds) = window.as_ref().and_then(|window| window.bounds.as_ref()) {
if let Some(note) = self.off_screen_note_for_bounds(bounds).await {
message = format!("{message} {note}");
}
}
Json(WindowGeometryOutput {
ok: true,
implemented: true,
backend,
window,
message,
permissions_hint: None,
received,
})
}
Err(error) => {
let error = format!("{error:#}");
Json(WindowGeometryOutput {
ok: false,
implemented: true,
backend,
window: None,
permissions_hint: window_permission_hint(&error),
message: error,
received,
})
}
}
}
/// Notes appended after targeted keyboard input: off-screen window warning
/// plus focused-element feedback.
async fn input_landing_notes(
&self,
focus: Option<&WindowFocusResult>,
expects_editable: bool,
) -> Vec<String> {
let mut notes = Vec::new();
if let Some(focus) = focus {
let bounds = focus
.focused_window
.as_ref()
.and_then(|window| window.bounds.as_ref())
.or(focus.requested_window.bounds.as_ref());
if let Some(bounds) = bounds {
if let Some(note) = self.off_screen_note_for_bounds(bounds).await {
notes.push(note);
}
}
}
if let Some(note) = self.focused_element_feedback(focus, expects_editable).await {
notes.push(note);
}
notes
}
#[cfg(test)]
fn cache_nodes(&self, nodes: &[AccessibilityNode]) {
self.cache_snapshot(nodes, None);
}
/// Readers may perform slow AT-SPI owner checks before resolving a click
/// or scroll again. Keep the snapshot stable for the whole input operation;
/// otherwise a checked index could resolve to a different app's node.
async fn commit_snapshot(&self, nodes: &[AccessibilityNode], target_pid: Option<u32>) {
let _input_guard = Arc::clone(&self.input_operation_lock).lock_owned().await;
self.cache_snapshot(nodes, target_pid);
}
fn cache_snapshot(&self, nodes: &[AccessibilityNode], target_pid: Option<u32>) {
if let Ok(mut cached) = self.last_snapshot.lock() {
*cached = CachedAccessibilitySnapshot {
nodes: nodes.to_vec(),
pid: target_pid,
};
}
}
#[cfg(test)]
fn clear_cached_nodes(&self) {
self.cache_snapshot(&[], None);
}
/// Reject an element-targeted action whose cached node belongs to a
/// different app than the action's target. A snapshot taken for a pid is
/// judged by that pid; an untargeted snapshot mixes apps, so its node is
/// judged by the pid that owns it on the accessibility bus. When neither is
/// known the action proceeds, since a mismatch cannot be shown.
async fn check_node_target(
&self,
node: &AccessibilityNode,
target_pid: Option<u32>,
) -> std::result::Result<(), String> {
let Some(target_pid) = target_pid else {
return Ok(());
};
let snapshot_pid = self
.last_snapshot
.lock()
.ok()
.and_then(|snapshot| snapshot.pid);
let owner_pid = match snapshot_pid {
Some(pid) => Some(pid),
None => object_ref_owner_pid(&node.object_ref).await.ok().flatten(),
};
match owner_pid {
Some(owner_pid) if owner_pid != target_pid => Err(node_target_mismatch_message(
node.index, owner_pid, target_pid,
)),
_ => Ok(()),
}
}
/// The cached node an element-targeted click or scroll would act on, when
/// the call names one. Resolution errors are left to the tool's own path.
fn cached_node_for(
&self,
element_index: Option<u32>,
selector: &ElementSelector<'_>,
purpose: ElementResolvePurpose,
) -> Option<AccessibilityNode> {
if element_index.is_none() && selector.is_empty() {
return None;
}
self.resolve_cached_node(element_index, selector, purpose)
.ok()
}
fn resolve_optional_target_point(
&self,
x: Option<i32>,
y: Option<i32>,
element_index: Option<u32>,
) -> std::result::Result<Option<(i32, i32)>, String> {
match (x.zip(y), element_index) {
(Some(point), _) => Ok(Some(point)),
(None, Some(index)) => self
.center_for_cached_node(index)
.map(Some)
.ok_or_else(|| {
format!(
"No clickable bounds cached for element_index {index}. Call get_app_state first and choose a node with positive width and height."
)
}),
(None, None) => Ok(None),
}
}
fn resolve_click_target(
&self,
params: &ClickParams,
) -> std::result::Result<ClickTarget, String> {
if let Some((x, y)) = params.x.zip(params.y) {
return Ok(ClickTarget::Coordinates(x, y));
}
let selector = params.selector();
let node = self.resolve_cached_node(
params.element_index,
&selector,
ElementResolvePurpose::Click,
)?;
// Toolkit extents are not necessarily in the pointer backend's space
// (GTK3 X11 HiDPI, GTK4 zero-origin bounds). Ordinary activation does
// not need a guessed scale when the target exposes its native action.
if is_plain_left_click(params.button.as_deref(), params.click_count) {
if let Some(action) =
primary_action(&node.actions).filter(|action| click_equivalent_action(action))
{
return Ok(ClickTarget::PrimaryAction {
object_ref: node.object_ref.clone(),
action_name: Some(action.name.clone()),
action_index: action.index,
});
}
}
if let Some((x, y)) = node.bounds.as_ref().and_then(bounds_center) {
return Ok(ClickTarget::Coordinates(x, y));
}
if !is_plain_left_click(params.button.as_deref(), params.click_count) {
return Err(format!(
"No clickable bounds cached for element_index {}. Call get_app_state first and choose a node with positive width and height.",
node.index
));
}
Err(format!(
"No clickable bounds or click-equivalent AT-SPI action for element_index {}. Use perform_action explicitly for other actions.",
node.index
))
}
fn center_for_cached_node(&self, element_index: u32) -> Option<(i32, i32)> {
let cached = self.last_snapshot.lock().ok()?;
let node = cached
.nodes
.iter()
.find(|node| node.index == element_index)?;
bounds_center(node.bounds.as_ref()?)
}
fn resolve_object_ref(
&self,
element_index: Option<u32>,
element_identifier: Option<&str>,
selector: &ElementSelector<'_>,
purpose: ElementResolvePurpose,
) -> std::result::Result<String, String> {
if let Some(element_identifier) = element_identifier
.map(str::trim)
.filter(|value| !value.is_empty())
{
return Ok(element_identifier.to_string());
}
self.resolve_cached_node(element_index, selector, purpose)
.map(|node| node.object_ref)
}
fn resolve_cached_node(
&self,
element_index: Option<u32>,
selector: &ElementSelector<'_>,
purpose: ElementResolvePurpose,
) -> std::result::Result<AccessibilityNode, String> {
let cached = self.last_snapshot.lock().map_err(|_| {
"Could not read cached accessibility nodes. Call get_app_state and retry.".to_string()
})?;
if let Some(element_index) = element_index {
return cached.nodes
.iter()
.find(|node| node.index == element_index)
.cloned()
.ok_or_else(|| {
format!(
"No cached accessibility node for element_index {element_index}. Call get_app_state first."
)
});
}
if selector.is_empty() {
return Err(
"Pass element_index, element_identifier, or a semantic selector such as role/name/text/states from the latest get_app_state result."
.to_string(),
);
}
resolve_semantic_node(cached.nodes.as_slice(), selector, purpose)
}
async fn perform_element_action(
&self,
params: &ActionParams,
requested_action: Option<&str>,
) -> Json<ActionOutput> {
let _input_guard = Arc::clone(&self.input_operation_lock).lock_owned().await;
let received = Some(serde_json::json!(params.clone()));
let object_ref = match self.resolve_object_ref(
params.element_index,
params.element_identifier.as_deref(),
¶ms.selector(),
ElementResolvePurpose::Action,
) {
Ok(object_ref) => object_ref,
Err(message) => {
return Json(ActionOutput {
ok: false,
implemented: true,
action: "perform_action".to_string(),
message,
received,
});
}
};
self.indicator.action("perform_action").await;
match invoke_accessibility_action(&object_ref, requested_action).await {
Ok(invocation) => Json(ActionOutput {
ok: invocation.ok,
implemented: true,
action: "perform_action".to_string(),
message: if invocation.ok {
format!(
"AT-SPI action {} ({}) invoked.",
invocation.action_index,
invocation
.action_name
.as_deref()
.filter(|name| !name.is_empty())
.unwrap_or("unnamed")
)
} else {
format!(
"AT-SPI action {} ({}) returned false.",
invocation.action_index,
invocation
.action_name
.as_deref()
.filter(|name| !name.is_empty())
.unwrap_or("unnamed")
)
},
received,
}),
Err(error) => Json(ActionOutput {
ok: false,
implemented: true,
action: "perform_action".to_string(),
message: error.to_string(),
received,
}),
}
}
}
#[derive(Debug)]
enum ClickTarget {
Coordinates(i32, i32),
PrimaryAction {
object_ref: String,
action_name: Option<String>,
action_index: i32,
},
}
#[derive(Debug, Clone, Copy)]
enum ElementResolvePurpose {
Click,
Action,
SetValue,
}
#[derive(Debug, Clone, Copy, Default)]
struct ElementSelector<'a> {
role: Option<&'a str>,
name: Option<&'a str>,
text: Option<&'a str>,
states: &'a [String],
}
impl ElementSelector<'_> {
fn is_empty(&self) -> bool {
[self.role, self.name, self.text]
.into_iter()
.all(|value| value.map(str::trim).is_none_or(str::is_empty))
&& self.states.iter().all(|value| value.trim().is_empty())
}
}
fn resolve_semantic_node(
nodes: &[AccessibilityNode],
selector: &ElementSelector<'_>,
purpose: ElementResolvePurpose,
) -> std::result::Result<AccessibilityNode, String> {
let mut matches = nodes
.iter()
.filter(|node| node_matches_selector(node, selector))
.collect::<Vec<_>>();
if matches.is_empty() {
return Err(format!(
"No cached accessibility node matched semantic selector {}. Call get_app_state first or pass element_index.",
describe_selector(selector)
));
}
if let Some(node) =
unique_preferred_node(&matches, |node| node_matches_resolve_purpose(node, purpose))
{
return Ok(node.clone());
}
let useful_matches = matches
.iter()
.copied()
.filter(|node| node_matches_resolve_purpose(node, purpose))
.collect::<Vec<_>>();
if !useful_matches.is_empty() {
matches = useful_matches;
}
if let Some(node) = unique_preferred_node(&matches, node_is_showing) {
return Ok(node.clone());
}
let visible_matches = matches
.iter()
.copied()
.filter(|node| node_is_showing(node))
.collect::<Vec<_>>();
if !visible_matches.is_empty() {
matches = visible_matches;
}
if matches.len() == 1 {
return Ok(matches[0].clone());
}
Err(format!(
"Semantic selector {} matched multiple cached nodes: {}. Pass element_index or add more selector fields.",
describe_selector(selector),
describe_matching_nodes(&matches),
))
}
fn unique_preferred_node<'a>(
nodes: &[&'a AccessibilityNode],
predicate: impl Fn(&AccessibilityNode) -> bool,
) -> Option<&'a AccessibilityNode> {
let mut matches = nodes.iter().copied().filter(|node| predicate(node));
let first = matches.next()?;
matches.next().is_none().then_some(first)
}
fn node_matches_selector(node: &AccessibilityNode, selector: &ElementSelector<'_>) -> bool {
selector
.role
.is_none_or(|role| normalized_contains(Some(node.role.as_str()), role))
&& selector
.name
.is_none_or(|name| normalized_contains(node.name.as_deref(), name))
&& selector.text.is_none_or(|text| {
normalized_contains(
node.text
.as_ref()
.and_then(|value| value.content.as_deref()),
text,
) || normalized_contains(node.name.as_deref(), text)
|| normalized_contains(node.description.as_deref(), text)
})
&& selector
.states
.iter()
.filter(|state| !state.trim().is_empty())
.all(|state| {
node.states
.iter()
.any(|node_state| normalized_equals(node_state, state))
})
}
fn node_matches_resolve_purpose(node: &AccessibilityNode, purpose: ElementResolvePurpose) -> bool {
match purpose {
ElementResolvePurpose::Click => {
node.bounds.as_ref().and_then(bounds_center).is_some()
|| primary_action_name(&node.actions).is_some()
}
ElementResolvePurpose::Action => !node.actions.is_empty(),
ElementResolvePurpose::SetValue => node.supports_editable_text || node.value.is_some(),
}
}
fn node_is_showing(node: &AccessibilityNode) -> bool {
node.states
.iter()
.any(|state| normalized_equals(state, "showing"))
&& node
.states
.iter()
.any(|state| normalized_equals(state, "visible"))
}
fn normalized_equals(actual: &str, expected: &str) -> bool {
normalize_text(actual) == normalize_text(expected)
}
fn normalized_contains(actual: Option<&str>, expected: &str) -> bool {
let expected = normalize_text(expected);
!expected.is_empty()
&& actual
.map(normalize_text)
.is_some_and(|actual| actual.contains(&expected))
}
fn normalize_text(value: &str) -> String {
value
.split_whitespace()
.collect::<Vec<_>>()
.join(" ")
.to_lowercase()
}
fn describe_selector(selector: &ElementSelector<'_>) -> String {
let mut parts = Vec::new();
if let Some(role) = selector.role.map(str::trim).filter(|role| !role.is_empty()) {
parts.push(format!("role={role:?}"));
}
if let Some(name) = selector.name.map(str::trim).filter(|name| !name.is_empty()) {
parts.push(format!("name={name:?}"));
}
if let Some(text) = selector.text.map(str::trim).filter(|text| !text.is_empty()) {
parts.push(format!("text={text:?}"));
}
let states = selector
.states
.iter()
.map(|state| state.trim())
.filter(|state| !state.is_empty())
.collect::<Vec<_>>();
if !states.is_empty() {
parts.push(format!("states={states:?}"));
}
if parts.is_empty() {
"<empty>".to_string()
} else {
parts.join(", ")
}
}
fn describe_matching_nodes(nodes: &[&AccessibilityNode]) -> String {
nodes
.iter()
.take(8)
.map(|node| {
format!(
"element_index {} role={:?} name={:?}",
node.index, node.role, node.name
)
})
.collect::<Vec<_>>()
.join("; ")
}
fn is_plain_left_click(button: Option<&str>, click_count: Option<u32>) -> bool {
let button = button.unwrap_or("left");
let click_count = click_count.unwrap_or(1);
matches!(button.to_ascii_lowercase().as_str(), "left" | "primary") && click_count == 1
}
fn click_equivalent_action(action: &AccessibilityAction) -> bool {
matches!(
action.name.to_ascii_lowercase().as_str(),
"click" | "press" | "toggle"
)
}
fn requested_or_primary_action(action: Option<&str>) -> &str {
match action.map(str::trim).filter(|value| !value.is_empty()) {
Some(action) => action,
None => "0",
}
}
fn primary_action(actions: &[AccessibilityAction]) -> Option<&AccessibilityAction> {
actions.first()
}
fn primary_action_name(actions: &[AccessibilityAction]) -> Option<String> {
primary_action(actions).map(|action| action.name.clone())
}
fn bounds_center(bounds: &Bounds) -> Option<(i32, i32)> {
if bounds.width <= 0 || bounds.height <= 0 {
return None;
}
if bounds.x <= i32::MIN / 2 || bounds.y <= i32::MIN / 2 {
return None;
}
Some((
bounds.x.checked_add(bounds.width / 2)?,
bounds.y.checked_add(bounds.height / 2)?,
))
}
/// Context-cost warning for an AT-SPI snapshot that no app target narrowed.
/// Silent when the tree failed (the failure message already explains) or when
/// any filter or pid match scoped the roots. When the caller did pass a target
/// and it still matched no AT-SPI root, telling them to pass a target would
/// only replay the same desktop-wide snapshot, so that case points at the
/// app's accessibility support instead.
fn unscoped_accessibility_tree_warning(
tree_scoped: bool,
tree_ok: bool,
target_requested: bool,
) -> Option<&'static str> {
if !tree_ok || tree_scoped {
return None;
}
Some(if target_requested {
"WARNING: the requested target matched no AT-SPI application root, so the accessibility tree covers the whole desktop and can flood context. The target app may expose no accessibility tree (Electron apps need --force-renderer-accessibility); check list_apps for its AT-SPI name and pass that as app_name_or_bundle_identifier, or lower max_nodes to bound the cost."
} else {
"WARNING: no app target scoped the accessibility tree, so it covers the whole desktop and can flood context. Pass app_name_or_bundle_identifier or a window target (window_id, pid, app_id, wm_class, title) to limit it."
})
}
/// Note appended when raw traversal hit max_nodes; the tree is incomplete.
fn truncated_accessibility_tree_note(truncated: bool) -> Option<&'static str> {
truncated.then_some(
"The node, depth, or read budget stopped traversal with unread elements left. To recover missing elements, scope to a narrower app or window target and raise max_nodes or max_depth (hard caps 2000 and 64); lowering max_nodes would drop more.",
)
}
fn shell_only_accessibility_tree_note(
nodes: &[AccessibilityNode],
scoped: bool,
truncated: bool,
) -> Option<&'static str> {
if !scoped || truncated || nodes.is_empty() {
return None;
}
let mut menu_nodes = std::collections::HashSet::new();
nodes.iter().all(|node| {
let in_menu = node.parent_index.is_some_and(|parent| menu_nodes.contains(&parent));
let Some(menu) = accessibility_chrome_context(&node.role, in_menu) else {
return false;
};
if menu {
menu_nodes.insert(node.index);
}
let has_content_text = !menu && node.text.as_ref().and_then(|text| text.content.as_deref())
.is_some_and(|text| text.chars().any(|ch| ch != '\u{fffc}' && !ch.is_whitespace()));
!node.supports_editable_text && node.value.is_none() && !has_content_text
})
.then_some(
"AT-SPI exposes only window/menu chrome; no content controls were observed. If this is an Electron/Chromium app, relaunch it with --force-renderer-accessibility to expose renderer content. Screenshots can still be used for observation; this tree does not establish that the UI is empty.",
)
}
fn compact_accessibility_tree_with_note(
nodes: Vec<AccessibilityNode>,
scoped: bool,
truncated: bool,
) -> (Vec<AccessibilityNode>, Option<&'static str>) {
// Menu ancestry is observation metadata; compaction can remove its nodes.
let note = shell_only_accessibility_tree_note(&nodes, scoped, truncated);
(compact_accessibility_tree(nodes), note)
}
fn compact_accessibility_tree(nodes: Vec<AccessibilityNode>) -> Vec<AccessibilityNode> {
if nodes.is_empty() {
return nodes;
}
let keep = nodes
.iter()
.map(should_keep_accessibility_node)
.collect::<Vec<_>>();
let mut old_to_new = vec![None; nodes.len()];
let mut compacted = Vec::new();
for (old_index, node) in nodes.iter().enumerate() {
if !keep[old_index] {
continue;
}
let mut compacted_node = node.clone();
compacted_node.index = compacted.len() as u32;
compacted_node.parent_index = nearest_kept_parent(&keep, &nodes, old_index);
old_to_new[old_index] = Some(compacted_node.index);
compacted.push(compacted_node);
}
for node in &mut compacted {
node.parent_index = node
.parent_index
.and_then(|old_parent| old_to_new.get(old_parent as usize).copied().flatten());
}
let child_counts = compacted.iter().filter_map(|node| node.parent_index).fold(
vec![0_i32; compacted.len()],
|mut counts, parent_index| {
counts[parent_index as usize] += 1;
counts
},
);
for (index, node) in compacted.iter_mut().enumerate() {
node.child_count = child_counts[index];
}
compacted
}
fn nearest_kept_parent(
keep: &[bool],
nodes: &[AccessibilityNode],
old_index: usize,
) -> Option<u32> {
let mut parent = nodes[old_index].parent_index;
while let Some(parent_index) = parent {
let parent_usize = parent_index as usize;
if keep.get(parent_usize).copied().unwrap_or(false) {
return Some(parent_index);
}
parent = nodes.get(parent_usize).and_then(|node| node.parent_index);
}
None
}
fn should_keep_accessibility_node(node: &AccessibilityNode) -> bool {
if node.depth <= 1 {
return true;
}
if is_actionable_accessibility_node(node) || has_meaningful_node_copy(node) {
return true;
}
matches!(
node.role.as_str(),
"page tab" | "menu item" | "menu" | "list item" | "tree item"
) && !is_sentinel_or_missing_bounds(node.bounds.as_ref())
}
fn is_actionable_accessibility_node(node: &AccessibilityNode) -> bool {
!node.actions.is_empty() || node.supports_editable_text || node.value.is_some()
}
fn has_meaningful_node_copy(node: &AccessibilityNode) -> bool {
has_non_empty_text(node.name.as_deref())
|| has_non_empty_text(node.description.as_deref())
|| has_non_empty_text(node.text.as_ref().and_then(|text| text.content.as_deref()))
}
fn has_non_empty_text(value: Option<&str>) -> bool {
value.map(str::trim).is_some_and(|value| !value.is_empty())
}
fn is_sentinel_or_missing_bounds(bounds: Option<&Bounds>) -> bool {
bounds.is_none()
}
fn select_accessibility_object_ref(
apps: &[AccessibleAppSummary],
target_pid: u32,
candidates: &[String],
) -> Option<String> {
let mut pid_matches = apps.iter().filter(|app| app.pid == Some(target_pid));
let first = pid_matches.next()?;
let second = pid_matches.next();
if second.is_none() {
return Some(first.object_ref.clone());
}
let lowered_candidates = candidates
.iter()
.map(|candidate| candidate.to_ascii_lowercase())
.collect::<Vec<_>>();
apps.iter()
.filter(|app| app.pid == Some(target_pid))
.find(|app| {
let name = app.name.as_deref().unwrap_or_default().to_ascii_lowercase();
lowered_candidates
.iter()
.any(|candidate| !candidate.is_empty() && name.contains(candidate))
})
.map(|app| app.object_ref.clone())
.or_else(|| Some(first.object_ref.clone()))
}
fn accessibility_filter_candidates(window_context: Option<&WindowInfo>) -> Vec<String> {
let Some(window) = window_context else {
return Vec::new();
};
let mut candidates = Vec::new();
push_candidate(&mut candidates, window.title.as_deref());
push_candidate(&mut candidates, window.wm_class.as_deref());
if let Some(app_id) = trimmed_nonempty(window.app_id.as_deref()) {
if !app_id.starts_with("window:") {
push_candidate(&mut candidates, Some(app_id));
if let Some(stripped) = app_id.strip_suffix(".desktop") {
push_candidate(&mut candidates, Some(stripped));
let normalized = stripped.replace(['-', '_', '.'], " ");
push_candidate(&mut candidates, Some(normalized.as_str()));
} else {
let normalized = app_id.replace(['-', '_', '.'], " ");
push_candidate(&mut candidates, Some(normalized.as_str()));
}
}
}
candidates
}
fn push_candidate(candidates: &mut Vec<String>, value: Option<&str>) {
let Some(value) = trimmed_nonempty(value) else {
return;
};
if !candidates.iter().any(|candidate| candidate == value) {
candidates.push(value.to_string());
}
}
fn trimmed_nonempty(value: Option<&str>) -> Option<&str> {
value.map(str::trim).filter(|value| !value.is_empty())
}
fn env_contains(key: &str, needle: &str) -> bool {
env::var(key)
.ok()
.is_some_and(|value| value.to_ascii_lowercase().contains(needle))
}
/// True when an environment variable is set to `"1"` (an explicit on switch).
fn env_flag_enabled(key: &str) -> bool {
env::var(key).ok().as_deref() == Some("1")
}
fn env_var_non_empty(key: &str) -> bool {
env::var(key)
.map(|value| !value.trim().is_empty())
.unwrap_or(false)
}
/// Return the base64 payload of a `data:` URL (or the original string if bare).
fn data_url_payload(data_url: &str) -> String {
data_url
.split_once(',')
.map(|(_, payload)| payload)
.unwrap_or(data_url)
.to_string()
}
fn session_is_wayland(session_type: Option<&str>, wayland_display: Option<&str>) -> bool {
match session_type
.map(str::trim)
.filter(|value| !value.is_empty())
{
Some(value) => value.eq_ignore_ascii_case("wayland"),
None => wayland_display.is_some_and(|value| !value.trim().is_empty()),
}
}
fn native_x11_xdotool_pointer_session(
session_type: Option<&str>,
wayland_display: Option<&str>,
) -> bool {
session_type.is_some_and(|value| value.trim().eq_ignore_ascii_case("x11"))
&& wayland_display.is_none_or(|value| value.trim().is_empty())
}
fn prefer_xdotool_pointer(
force_ydotool: bool,
session_type: Option<&str>,
display_available: bool,
wayland_display: Option<&str>,
xdotool_available: bool,
) -> bool {
!force_ydotool
&& native_x11_xdotool_pointer_session(session_type, wayland_display)
&& display_available
&& xdotool_available
}
fn prefer_xdotool_keyboard(
force_ydotool: bool,
force_xdotool: bool,
is_wayland: bool,
display_available: bool,
xdotool_available: bool,
) -> bool {
!force_ydotool && display_available && xdotool_available && (force_xdotool || !is_wayland)
}
fn prepare_app_state_screenshot(
mut raw: RawScreenshotCapture,
crop: Option<(i32, i32, u32, u32)>,
target_requested: bool,
options: ScreenshotPayloadOptions,
) -> Result<ScreenshotCapture> {
if target_requested && crop.is_none() {
anyhow::bail!(
"targeted screenshot requires a resolved window; refusing to return the full desktop"
);
}
if let Some(rect) = crop {
let (x, y, width, height) = clip_capture_rect(rect, raw.width, raw.height)?;
let (bytes, width, height) = crop_png(&raw.bytes, x, y, width, height)
.map_err(|error| anyhow::anyhow!("targeted screenshot crop failed: {error}"))?;
raw = RawScreenshotCapture {
mime_type: raw.mime_type,
bytes,
source: raw.source,
width,
height,
};
}
prepare_screenshot_payload(raw, options)
}
fn ensure_readonly_screenshot_target_is_visible(window: &WindowInfo) -> Result<()> {
if window.hidden {
anyhow::bail!("targeted get_app_state screenshot requires a visible, unminimized window");
}
if !window.focused {
anyhow::bail!(
"targeted get_app_state screenshot requires the window to already be focused; use the screenshot tool to raise it before capture"
);
}
Ok(())
}
#[derive(Debug, Clone, Copy)]
struct WindowCoordinateMap {
capture_rect: (i32, i32, u32, u32),
full_capture_rect: (i32, i32, u32, u32),
portal_rect: Option<(i32, i32, u32, u32)>,
}
impl WindowCoordinateMap {
fn portal_point(&self, capture_x: i32, capture_y: i32) -> Option<(i32, i32)> {
let (portal_x, portal_y, portal_width, portal_height) = self.portal_rect?;
let (full_x, full_y, full_width, full_height) = self.full_capture_rect;
Some((
map_coordinate_between_rects(capture_x, full_x, full_width, portal_x, portal_width),
map_coordinate_between_rects(capture_y, full_y, full_height, portal_y, portal_height),
))
}
}
fn map_coordinate_between_rects(
value: i32,
source_origin: i32,
source_size: u32,
target_origin: i32,
target_size: u32,
) -> i32 {
let offset = i64::from(value) - i64::from(source_origin);
let scaled = offset.saturating_mul(i64::from(target_size)) / i64::from(source_size.max(1));
(i64::from(target_origin) + scaled).clamp(i64::from(i32::MIN), i64::from(i32::MAX)) as i32
}
fn logical_window_crop_rect(
bounds: &crate::windowing::WindowBounds,
monitors: &[(i32, i32, i32, i32)],
capture_width: u32,
capture_height: u32,
) -> Result<(i32, i32, u32, u32)> {
let mut monitors = monitors
.iter()
.filter(|(_, _, width, height)| *width > 0 && *height > 0);
let first = monitors
.next()
.ok_or_else(|| anyhow::anyhow!("desktop returned no usable monitor geometry"))?;
let (mut min_x, mut min_y) = (i64::from(first.0), i64::from(first.1));
let (mut max_x, mut max_y) = (
i64::from(first.0) + i64::from(first.2),
i64::from(first.1) + i64::from(first.3),
);
for (x, y, width, height) in monitors {
min_x = min_x.min(i64::from(*x));
min_y = min_y.min(i64::from(*y));
max_x = max_x.max(i64::from(*x) + i64::from(*width));
max_y = max_y.max(i64::from(*y) + i64::from(*height));
}
let logical_width = max_x - min_x;
let logical_height = max_y - min_y;
if logical_width <= 0 || logical_height <= 0 || capture_width == 0 || capture_height == 0 {
anyhow::bail!("screenshot or monitor geometry is empty");
}
let scale_x = f64::from(capture_width) / logical_width as f64;
let scale_y = f64::from(capture_height) / logical_height as f64;
if !scale_x.is_finite() || !scale_y.is_finite() || (scale_x - scale_y).abs() > 0.01 {
anyhow::bail!(
"captured desktop {}x{} does not have a uniform scale relative to the logical monitor layout {}x{}",
capture_width,
capture_height,
logical_width,
logical_height
);
}
let x = i64::from(
bounds
.x
.ok_or_else(|| anyhow::anyhow!("window x is unavailable"))?,
);
let y = i64::from(
bounds
.y
.ok_or_else(|| anyhow::anyhow!("window y is unavailable"))?,
);
if bounds.width == 0 || bounds.height == 0 {
anyhow::bail!("window bounds are empty");
}
let left = (((x - min_x) as f64) * scale_x).floor() as i64;
let top = (((y - min_y) as f64) * scale_y).floor() as i64;
let right = (((x + i64::from(bounds.width) - min_x) as f64) * scale_x).ceil() as i64;
let bottom = (((y + i64::from(bounds.height) - min_y) as f64) * scale_y).ceil() as i64;
let width = u32::try_from(right - left)
.map_err(|_| anyhow::anyhow!("scaled window width is invalid"))?;
let height = u32::try_from(bottom - top)
.map_err(|_| anyhow::anyhow!("scaled window height is invalid"))?;
let left = i32::try_from(left).map_err(|_| anyhow::anyhow!("scaled window x is invalid"))?;
let top = i32::try_from(top).map_err(|_| anyhow::anyhow!("scaled window y is invalid"))?;
Ok((left, top, width, height))
}
fn clip_capture_rect(
(x, y, width, height): (i32, i32, u32, u32),
capture_width: u32,
capture_height: u32,
) -> Result<(i32, i32, u32, u32)> {
let left = i64::from(x).max(0);
let top = i64::from(y).max(0);
let right = (i64::from(x) + i64::from(width)).min(i64::from(capture_width));
let bottom = (i64::from(y) + i64::from(height)).min(i64::from(capture_height));
if right <= left || bottom <= top {
anyhow::bail!(
"targeted screenshot window is outside the captured desktop; refusing to return the full desktop"
);
}
Ok((
i32::try_from(left).map_err(|_| anyhow::anyhow!("capture crop x is invalid"))?,
i32::try_from(top).map_err(|_| anyhow::anyhow!("capture crop y is invalid"))?,
u32::try_from(right - left)
.map_err(|_| anyhow::anyhow!("capture crop width is invalid"))?,
u32::try_from(bottom - top)
.map_err(|_| anyhow::anyhow!("capture crop height is invalid"))?,
))
}
/// Convert a window's bounds into a crop rectangle, if it has a usable origin
/// and non-zero size.
fn window_crop_rect(bounds: &crate::windowing::WindowBounds) -> Option<(i32, i32, u32, u32)> {
let x = bounds.x?;
let y = bounds.y?;
if bounds.width == 0 || bounds.height == 0 {
return None;
}
Some((x, y, bounds.width, bounds.height))
}
fn apply_window_relative_click_coordinates(
params: &mut ClickParams,
capture_rect: (i32, i32, u32, u32),
) -> std::result::Result<(), String> {
let (relative_x, relative_y) = params
.x
.zip(params.y)
.ok_or_else(|| "Relative coordinate clicks require both x and y.".to_string())?;
let (origin_x, origin_y, width, height) = capture_rect;
if width == 0 || height == 0 {
return Err(
"Relative coordinate clicks require non-empty target-window bounds.".to_string(),
);
}
if relative_x < 0 || relative_y < 0 {
return Err("Relative click coordinates must be inside target-window bounds.".to_string());
}
if relative_x as u32 >= width || relative_y as u32 >= height {
return Err("Relative click coordinates must be inside target-window bounds.".to_string());
}
let x = origin_x
.checked_add(relative_x)
.ok_or_else(|| "Relative click x coordinate overflowed.".to_string())?;
let y = origin_y
.checked_add(relative_y)
.ok_or_else(|| "Relative click y coordinate overflowed.".to_string())?;
params.x = Some(x);
params.y = Some(y);
Ok(())
}
/// Point a window-targeted scroll at the centre of the resolved window when
/// the caller supplied no coordinates. Without this the wheel events land on
/// whatever is under the current pointer position.
fn apply_window_center_scroll_point(
params: &mut ScrollParams,
capture_rect: (i32, i32, u32, u32),
) -> std::result::Result<(), String> {
let (origin_x, origin_y, width, height) = capture_rect;
if width == 0 || height == 0 {
return Err(
"Window-targeted scroll requires non-empty target-window bounds; pass x/y explicitly."
.to_string(),
);
}
params.x = Some(origin_x.saturating_add((width / 2) as i32));
params.y = Some(origin_y.saturating_add((height / 2) as i32));
Ok(())
}
fn apply_window_relative_scroll_coordinates(
params: &mut ScrollParams,
capture_rect: (i32, i32, u32, u32),
) -> std::result::Result<(), String> {
let (relative_x, relative_y) = params
.x
.zip(params.y)
.ok_or_else(|| "Relative scroll coordinates require both x and y.".to_string())?;
let (origin_x, origin_y, width, height) = capture_rect;
if width == 0 || height == 0 {
return Err(
"Relative scroll coordinates require non-empty target-window bounds.".to_string(),
);
}
if relative_x < 0 || relative_y < 0 || relative_x as u32 >= width || relative_y as u32 >= height
{
return Err("Relative scroll coordinates must be inside target-window bounds.".to_string());
}
params.x = Some(origin_x.saturating_add(relative_x));
params.y = Some(origin_y.saturating_add(relative_y));
Ok(())
}
/// Crop a PNG image to `(x, y, w, h)` (clamped to the image), returning the
/// re-encoded PNG and the actual cropped dimensions.
fn crop_png(
raw: &[u8],
x: i32,
y: i32,
w: u32,
h: u32,
) -> std::result::Result<(Vec<u8>, u32, u32), String> {
use std::io::Cursor;
let img = image::load_from_memory_with_format(raw, image::ImageFormat::Png)
.map_err(|e| format!("decode png: {e}"))?;
let (iw, ih) = (img.width(), img.height());
let x = x.max(0) as u32;
let y = y.max(0) as u32;
if x >= iw || y >= ih {
return Err("crop origin outside image".into());
}
let w = w.min(iw - x);
let h = h.min(ih - y);
let sub = img.crop_imm(x, y, w, h);
let mut out = Vec::new();
sub.write_to(&mut Cursor::new(&mut out), image::ImageFormat::Png)
.map_err(|e| format!("encode png: {e}"))?;
Ok((out, w, h))
}
fn action_result(
action: &str,
result: std::result::Result<Vec<Output>, String>,
received: Option<serde_json::Value>,
) -> ActionOutput {
match result {
Ok(_) => ActionOutput {
ok: true,
implemented: true,
action: action.to_string(),
message: if action == "type_text" {
"Action sent through ydotool using US physical key positions. The active keyboard layout can change the resulting text; verify the field contents."
} else {
"Action sent through ydotool."
}
.to_string(),
received,
},
Err(message) => ActionOutput {
ok: false,
implemented: true,
action: action.to_string(),
message,
received,
},
}
}
fn unsafe_pointer_fallback(action: &str, received: Option<serde_json::Value>) -> ActionOutput {
ActionOutput {
ok: false,
implemented: true,
action: action.to_string(),
message: "Coordinate input was not sent: ydotool cannot position the pointer accurately on Wayland. Enable the uinput absolute pointer with working screenshot capture, or use COMPUTER_USE_LINUX_FORCE_PORTAL_POINTER=1 with an approved desktop portal session. Remove COMPUTER_USE_LINUX_FORCE_YDOTOOL_POINTER if set.".to_string(),
received,
}
}
fn portal_start_error(
action: &str,
error: anyhow::Error,
received: Option<serde_json::Value>,
) -> ActionOutput {
ActionOutput {
ok: false,
implemented: true,
action: action.to_string(),
message: format!("Input was not sent because the remote desktop portal could not start; input was not replayed through ydotool: {error:#}"),
received,
}
}
fn abs_pointer_error(
action: &str,
error: String,
received: Option<serde_json::Value>,
) -> ActionOutput {
ActionOutput {
ok: false,
implemented: true,
action: action.to_string(),
message: format!("Absolute pointer {action} may have started before it failed; input was not replayed through another backend: {error}"),
received,
}
}
fn abs_pointer_result(
action: &str,
result: std::result::Result<crate::abs_pointer::PointerLanding, String>,
received: Option<serde_json::Value>,
) -> ActionOutput {
match result {
Ok(landing) => with_notes(
ActionOutput {
ok: true,
implemented: true,
action: action.to_string(),
message: if action == "scroll" {
"Pointer positioned through uinput; wheel input sent through ydotool."
.to_string()
} else {
"Action sent through the uinput absolute pointer.".to_string()
},
received,
},
abs_pointer_clamp_note(landing),
),
Err(error) => abs_pointer_error(action, error, received),
}
}
fn portal_action_error(
action: &str,
error: anyhow::Error,
received: Option<serde_json::Value>,
) -> ActionOutput {
ActionOutput {
ok: false,
implemented: true,
action: action.to_string(),
message: format!(
"Remote desktop portal {action} may have started before it failed; input was not replayed through another backend: {error:#}"
),
received,
}
}
fn portal_coordinate_error(action: &str, received: Option<serde_json::Value>) -> ActionOutput {
ActionOutput {
ok: false,
implemented: true,
action: action.to_string(),
message: format!(
"Remote desktop portal {action} was not sent because the coordinate could not be mapped safely to the complete shared desktop; input was not replayed through another backend."
),
received,
}
}
fn action_result_with_focus(
action: &str,
result: std::result::Result<Vec<Output>, String>,
received: Option<serde_json::Value>,
focus: Option<WindowFocusResult>,
) -> ActionOutput {
with_focus_context(action_result(action, result, received), focus)
}
fn successful_action_with_focus(
action: &str,
message: &str,
received: Option<serde_json::Value>,
focus: Option<WindowFocusResult>,
) -> ActionOutput {
with_focus_context(
ActionOutput {
ok: true,
implemented: true,
action: action.to_string(),
message: message.to_string(),
received,
},
focus,
)
}
fn with_focus_context(mut output: ActionOutput, focus: Option<WindowFocusResult>) -> ActionOutput {
if output.ok {
if let Some(focus) = focus {
let verification = if focus.exact_window_focused {
"exact window-focus"
} else {
"app-level focus"
};
output.message = format!(
"{} Target window_id {} was focused with {verification} verification before input.",
output.message, focus.requested_window.window_id,
);
}
}
output
}
/// Exact window focus alone cannot rule out lost input. Only a complete,
/// scoped shell-only tree explains the missing focus as unavailable content.
fn focus_probe_feedback(
probe: &FocusProbe,
expects_editable: bool,
exact_window_focused: bool,
) -> String {
match probe {
FocusProbe::Found(element) => describe_focused_element(element, expects_editable),
FocusProbe::ShellOnly if exact_window_focused => "Focused-element feedback unavailable: window focus was verified before input, but AT-SPI exposes only window/menu chrome in the target app. Field contents were not verified. Electron/Chromium apps may need --force-renderer-accessibility.".to_string(),
FocusProbe::ShellOnly => "WARNING: AT-SPI exposes only window/menu chrome and no focused element in the target app; the input may have landed nowhere.".to_string(),
FocusProbe::NoneFocused => "WARNING: AT-SPI reports no focused element in the target app; the input may have landed nowhere.".to_string(),
FocusProbe::Incomplete => "Focused-element feedback unavailable: the focused element was not reached within the AT-SPI search limits, so the input could not be verified. Its absence here does not mean the input was lost.".to_string(),
FocusProbe::NoAccessibleApp => "Focused-element feedback unavailable: the target app exposes no AT-SPI tree, so the input could not be verified. Electron apps need --force-renderer-accessibility.".to_string(),
}
}
fn node_target_mismatch_message(element_index: u32, owner_pid: u32, target_pid: u32) -> String {
format!(
"element_index {element_index} comes from the latest get_app_state snapshot of pid {owner_pid}, not the target pid {target_pid}. Call get_app_state for the target app and use an index from that result."
)
}
fn focus_target_pid(focus: Option<&WindowFocusResult>) -> Option<u32> {
let focus = focus?;
focus
.focused_window
.as_ref()
.and_then(|window| window.pid)
.or(focus.requested_window.pid)
}
fn describe_focused_element(element: &FocusedElementSummary, expects_editable: bool) -> String {
let name = element
.name
.as_deref()
.filter(|name| !name.is_empty())
.map(|name| format!(" \"{name}\""))
.unwrap_or_default();
if focused_element_is_editable(element) {
format!("Focused element: {}{name} (editable).", element.role)
} else if element.is_terminal {
format!(
"Focused element: {}{name} (terminal keyboard input).",
element.role
)
} else if expects_editable {
format!(
"WARNING: focused element is {}{name}, which is not editable — the typed text likely went nowhere. Click the intended input first or use set_value.",
element.role
)
} else {
format!("Focused element: {}{name} (not editable).", element.role)
}
}
fn first_line(text: &str) -> &str {
text.lines().next().unwrap_or(text)
}
/// Append supplemental notes (off-screen or focused-element feedback) to an
/// action result message without changing ok/implemented semantics.
fn with_notes(mut output: ActionOutput, notes: impl IntoIterator<Item = String>) -> ActionOutput {
for note in notes {
output.message = format!("{} {note}", output.message);
}
output
}
fn abs_pointer_clamp_note(landing: crate::abs_pointer::PointerLanding) -> Option<String> {
(landing.requested != landing.emitted).then(|| {
format!(
"Requested coordinate {},{} was clamped to {},{} by the uinput absolute pointer.",
landing.requested.0, landing.requested.1, landing.emitted.0, landing.emitted.1
)
})
}
fn focus_satisfies_target(focus: &WindowFocusResult, target: &WindowTarget) -> bool {
if target.requires_exact_focus() {
focus.exact_window_focused
} else {
focus.exact_window_focused || focus.app_focused
}
}
async fn window_list_output() -> ListWindowsOutput {
match list_windows().await {
Ok(windows) => {
let backend = window_backend(windows.iter());
let note = registry::list_note(&backend);
ListWindowsOutput {
backend,
windows,
error: None,
permissions_hint: None,
note: note.to_string(),
}
}
Err(error) => {
let error = format!("{error:#}");
ListWindowsOutput {
backend: GNOME_SHELL_INTROSPECT_BACKEND.to_string(),
windows: Vec::new(),
permissions_hint: window_permission_hint(&error),
error: Some(error),
note: "Window listing failed, so targeted keyboard input cannot safely focus or verify a target window."
.to_string(),
}
}
}
}
fn window_backend<'a>(windows: impl Iterator<Item = &'a WindowInfo>) -> String {
windows
.map(|window| window.backend.clone())
.next()
.unwrap_or_else(|| GNOME_SHELL_INTROSPECT_BACKEND.to_string())
}
fn absolute_mousemove_args(x: i32, y: i32) -> Vec<String> {
vec![
"mousemove".to_string(),
"--absolute".to_string(),
"--".to_string(),
x.to_string(),
y.to_string(),
]
}
fn xdotool_pointer_click_args(
x: i32,
y: i32,
count: u32,
button: Option<&str>,
) -> Option<Vec<String>> {
let button = xdotool_pointer_button_code(button)?;
Some(vec![
"mousemove".to_string(),
"--".to_string(),
x.to_string(),
y.to_string(),
"click".to_string(),
"--repeat".to_string(),
count.to_string(),
button.to_string(),
])
}
fn xdotool_pointer_button_code(button: Option<&str>) -> Option<&'static str> {
match button.unwrap_or("left").to_ascii_lowercase().as_str() {
"left" => Some("1"),
"middle" => Some("2"),
"right" => Some("3"),
_ => None,
}
}
/// Gap between XTEST wheel-button clicks, in milliseconds.
///
/// xdotool's own default is 100ms. One page is five clicks and valid requests
/// can ask for up to 100 pages, so 12ms keeps the bounded operation responsive
/// while matching the pacing used by `xdotool type`.
const XDOTOOL_SCROLL_CLICK_DELAY_MS: u64 = 12;
/// X11 wheel buttons: 4 up, 5 down, 6 left, 7 right. `repeat` is the notch
/// count, the same number ydotool would pass to `mousemove --wheel`.
fn xdotool_pointer_scroll_args(
point: Option<(i32, i32)>,
repeat: u32,
direction: ScrollDirection,
) -> Vec<String> {
let button = match direction {
ScrollDirection::Up => "4",
ScrollDirection::Down => "5",
ScrollDirection::Left => "6",
ScrollDirection::Right => "7",
};
let mut args = Vec::new();
if let Some((x, y)) = point {
args.extend([
"mousemove".to_string(),
"--".to_string(),
x.to_string(),
y.to_string(),
]);
}
args.extend([
"click".to_string(),
"--delay".to_string(),
XDOTOOL_SCROLL_CLICK_DELAY_MS.to_string(),
"--repeat".to_string(),
repeat.max(1).to_string(),
button.to_string(),
]);
args
}
/// `xdotool click --repeat` sleeps `--delay` after every click, including
/// the last, so the command timeout has to cover that gap.
fn xdotool_scroll_timeout(repeat: u32, delay_ms: u64) -> Duration {
INPUT_COMMAND_TIMEOUT.saturating_add(Duration::from_millis(
u64::from(repeat.max(1)).saturating_mul(delay_ms),
))
}
#[derive(Debug)]
struct PointerCommandResult {
outputs: Vec<Output>,
backend: KeyboardCommandBackend,
}
async fn run_xdotool_pointer_or_fallback<F, Fut>(
program: &Path,
args: &[String],
fallback: F,
) -> std::result::Result<PointerCommandResult, String>
where
F: FnOnce() -> Fut,
Fut: Future<Output = std::result::Result<Vec<Output>, String>>,
{
match run_xdotool(program, args).await {
XdotoolAttempt::Unavailable => fallback().await.map(|outputs| PointerCommandResult {
outputs,
backend: KeyboardCommandBackend::Ydotool,
}),
XdotoolAttempt::Finished(result) => result.map(|output| PointerCommandResult {
outputs: vec![output],
backend: KeyboardCommandBackend::Xdotool,
}),
}
}
async fn run_xdotool_pointer_or_fallback_with_timeout<F, Fut>(
program: &Path,
args: &[String],
command_timeout: Duration,
fallback: F,
) -> std::result::Result<PointerCommandResult, String>
where
F: FnOnce() -> Fut,
Fut: Future<Output = std::result::Result<Vec<Output>, String>>,
{
match run_xdotool_with_timeout(program, args, command_timeout).await {
XdotoolAttempt::Unavailable => fallback().await.map(|outputs| PointerCommandResult {
outputs,
backend: KeyboardCommandBackend::Ydotool,
}),
XdotoolAttempt::Finished(result) => result.map(|output| PointerCommandResult {
outputs: vec![output],
backend: KeyboardCommandBackend::Xdotool,
}),
}
}
fn wheel_mousemove_args(dx: i32, dy: i32) -> Vec<String> {
vec![
"mousemove".to_string(),
"--wheel".to_string(),
"--".to_string(),
dx.to_string(),
dy.to_string(),
]
}
async fn run_ydotool_sequence(
commands: &[Vec<String>],
) -> std::result::Result<Vec<Output>, String> {
let mut outputs = Vec::new();
for (index, args) in commands.iter().enumerate() {
outputs.push(run_ydotool(args).await?);
if index + 1 < commands.len() {
sleep(Duration::from_millis(35)).await;
}
}
Ok(outputs)
}
async fn run_ydotool_drag(
start_x: i32,
start_y: i32,
end_x: i32,
end_y: i32,
) -> std::result::Result<Vec<Output>, String> {
let mut outputs = vec![run_ydotool(&absolute_mousemove_args(start_x, start_y)).await?];
sleep(Duration::from_millis(35)).await;
let mut first_error = match run_ydotool(&["click".to_string(), "0x40".to_string()]).await {
Ok(output) => {
outputs.push(output);
None
}
Err(error) => Some(error),
};
sleep(Duration::from_millis(35)).await;
if first_error.is_none() {
match run_ydotool(&absolute_mousemove_args(end_x, end_y)).await {
Ok(output) => outputs.push(output),
Err(error) => first_error = Some(error),
}
sleep(Duration::from_millis(35)).await;
}
let release = run_ydotool(&["click".to_string(), "0x80".to_string()]).await;
match (first_error, release) {
(None, Ok(output)) => {
outputs.push(output);
Ok(outputs)
}
(Some(error), Ok(_)) => Err(error),
(None, Err(error)) => Err(error),
(Some(error), Err(release_error)) => Err(format!(
"{error}; ydotool button release also failed: {release_error}"
)),
}
}
async fn run_abs_pointer<T, F>(
pointer: Arc<Mutex<Option<crate::abs_pointer::AbsPointer>>>,
operation: F,
) -> std::result::Result<T, String>
where
T: Send + 'static,
F: FnOnce(&mut crate::abs_pointer::AbsPointer) -> Result<T> + Send + 'static,
{
tokio::task::spawn_blocking(move || {
let mut guard = pointer
.lock()
.map_err(|_| "absolute pointer lock was poisoned".to_string())?;
let pointer = guard
.as_mut()
.ok_or_else(|| "absolute pointer is unavailable".to_string())?;
operation(pointer).map_err(|error| format!("{error:#}"))
})
.await
.map_err(|error| format!("absolute pointer task failed: {error}"))?
}
async fn run_cancellation_safe_input<T, F>(
input_guard: tokio::sync::OwnedMutexGuard<()>,
operation: F,
) -> (
Option<tokio::sync::OwnedMutexGuard<()>>,
std::result::Result<T, String>,
)
where
T: Send + 'static,
F: Future<Output = std::result::Result<T, String>> + Send + 'static,
{
// Dropping a JoinHandle detaches its task, retaining the guard until the
// stateful input operation has completed even if the caller is cancelled.
match tokio::spawn(async move { (input_guard, operation.await) }).await {
Ok((input_guard, result)) => (Some(input_guard), result),
Err(error) => (None, Err(format!("stateful input task failed: {error}"))),
}
}
async fn run_ydotool(args: &[String]) -> std::result::Result<Output, String> {
let support = ydotool::ensure_supported_async().await?;
let socket = ydotool::socket_path_for_command()?;
let mut command = TokioCommand::new(&support.executable);
command.args(args);
command.env("YDOTOOL_SOCKET", socket);
let output =
crate::command_runner::output_with_timeout(command, "run ydotool", INPUT_COMMAND_TIMEOUT)
.await
.map_err(|error| format!("{error:#}"))?;
if output.status.success() {
if let Some(error) = ydotool::cli_error(&output.stderr) {
Err(error)
} else {
Ok(output)
}
} else {
Err(ydotool_output_error(output))
}
}
async fn run_ydotool_type_text(text: &str) -> std::result::Result<Output, String> {
validate_ydotool_text(text)?;
let support = ydotool::ensure_supported_async().await?;
let socket = ydotool::socket_path_for_command()?;
let mut command = TokioCommand::new(&support.executable);
command.args(["type", "--file", "-"]);
command.env("YDOTOOL_SOCKET", socket);
let output = crate::command_runner::output_with_stdin(
command,
"run ydotool type",
ydotool_type_timeout(text),
text.as_bytes().to_vec(),
)
.await
.map_err(|error| format!("{error:#}"))?;
if output.status.success() {
if let Some(error) = ydotool::cli_error(&output.stderr) {
Err(error)
} else {
Ok(output)
}
} else {
Err(ydotool_output_error(output))
}
}
fn validate_ydotool_text(text: &str) -> std::result::Result<(), String> {
if let Some(ch) = text
.chars()
.find(|ch| !matches!(ch, '\t' | '\n' | ' '..='~'))
{
return Err(format!(
"ydotool cannot type U+{:04X}; no text was sent. Use a layout-aware text backend or set_value on an editable field.",
ch as u32
));
}
Ok(())
}
fn ydotool_type_timeout(text: &str) -> Duration {
let text_seconds = (text.chars().count() as u64).div_ceil(YDOTOOL_TYPE_CHARS_PER_SECOND);
Duration::from_secs(INPUT_COMMAND_TIMEOUT.as_secs().saturating_add(text_seconds))
}
const EVDEV_KEY_LEFTCTRL: i32 = 29;
const EVDEV_KEY_LEFTSHIFT: i32 = 42;
const EVDEV_KEY_V: i32 = 47;
const EVDEV_KEY_INSERT: i32 = 110;
const KDE_CLIPBOARD_RESTORE_MIN_DELAY_MS: u64 = 1_500;
const KDE_CLIPBOARD_RESTORE_MAX_DELAY_MS: u64 = 5_000;
const KDE_CLIPBOARD_RESTORE_CHARS_PER_SECOND: u64 = 250;
fn kde_clipboard_restore_delay(text: &str) -> Duration {
let text_delay_ms = (text.chars().count() as u64)
.saturating_mul(1_000)
.div_ceil(KDE_CLIPBOARD_RESTORE_CHARS_PER_SECOND);
Duration::from_millis(text_delay_ms.clamp(
KDE_CLIPBOARD_RESTORE_MIN_DELAY_MS,
KDE_CLIPBOARD_RESTORE_MAX_DELAY_MS,
))
}
#[derive(Debug)]
struct KdeClipboardPasteError {
message: String,
can_fallback_to_ydotool: bool,
clear_portal_keyboard_session: bool,
}
impl KdeClipboardPasteError {
fn before_text_input(message: String) -> Self {
Self {
message,
can_fallback_to_ydotool: true,
clear_portal_keyboard_session: false,
}
}
fn after_portal_input(message: String) -> Self {
Self {
message,
can_fallback_to_ydotool: false,
clear_portal_keyboard_session: true,
}
}
}
async fn run_kde_clipboard_paste_text(
session: &PortalKeyboardSession,
text: &str,
paste_shortcut: KdeClipboardPasteShortcut,
) -> std::result::Result<String, KdeClipboardPasteError> {
let previous = kde_clipboard_contents()
.await
.map_err(KdeClipboardPasteError::before_text_input)?;
kde_set_clipboard_contents(text)
.await
.map_err(KdeClipboardPasteError::before_text_input)?;
let (modifiers, keycode) = kde_clipboard_paste_chord(paste_shortcut);
let paste_result = press_keycode_chord(session, modifiers, keycode)
.await
.map_err(|error| format!("{error:#}"));
sleep(kde_clipboard_restore_delay(text)).await;
let restore_result = kde_set_clipboard_contents(&previous).await;
match (paste_result, restore_result) {
(Ok(_), Ok(_)) => Ok("Action pasted through KDE clipboard integration.".to_string()),
(Err(error), Ok(_)) => Err(KdeClipboardPasteError::after_portal_input(error)),
(Ok(_), Err(restore_error)) => Ok(format!(
"Action pasted through KDE clipboard integration. Warning: previous KDE clipboard contents could not be restored: {restore_error}"
)),
(Err(error), Err(restore_error)) => Err(KdeClipboardPasteError::after_portal_input(
format!("{error}; previous KDE clipboard contents could not be restored: {restore_error}"),
)),
}
}
/// Paste chord for KDE clipboard text input.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum KdeClipboardPasteShortcut {
/// Ctrl+V, for ordinary widgets.
Standard,
/// Ctrl+Shift+V, for most terminal emulators.
CtrlShiftV,
/// Shift+Insert, for the xterm and rxvt families.
ShiftInsert,
}
async fn kde_clipboard_paste_shortcut(
target: &WindowTarget,
focus: Option<&WindowFocusResult>,
) -> KdeClipboardPasteShortcut {
let current = match focus {
Some(_) => None,
None => focused_window().await.ok().flatten(),
};
let window = focus
.map(|focus| {
focus
.focused_window
.as_ref()
.unwrap_or(&focus.requested_window)
})
.or(current.as_ref());
let terminal = kde_clipboard_terminal_shortcut(target, window);
if terminal.is_none() {
return KdeClipboardPasteShortcut::Standard;
}
// A terminal window can hold focus on a non-terminal widget (a search
// field, a tab rename box) that pastes with plain Ctrl+V. Only an answer
// from the terminal's own AT-SPI app may override the chord: without a pid,
// or when the terminal exposes no AT-SPI root, keep the terminal chord.
let Some(pid) = window.and_then(|window| window.pid) else {
return kde_clipboard_shortcut_for_focus(terminal, None);
};
let focused_element = timeout(
Duration::from_millis(1_500),
focused_element_summary_in_app(pid),
)
.await
.ok()
.and_then(Result::ok)
.flatten();
kde_clipboard_shortcut_for_focus(terminal, focused_element.as_ref())
}
/// The terminal chord for the resolved window, or for an explicit terminal
/// selector when no window resolved. A resolved window always wins over a
/// possibly stale selector.
fn kde_clipboard_terminal_shortcut(
target: &WindowTarget,
window: Option<&WindowInfo>,
) -> Option<TerminalPasteShortcut> {
match window {
Some(window) => terminal_paste_shortcut(window),
None => target
.has_terminal_target()
.then_some(TerminalPasteShortcut::CtrlShiftV),
}
}
fn kde_clipboard_shortcut_for_focus(
terminal: Option<TerminalPasteShortcut>,
focused_element: Option<&FocusedElementSummary>,
) -> KdeClipboardPasteShortcut {
// No AT-SPI answer (xterm exposes none) keeps the terminal chord; a
// concrete non-terminal focused element switches back to Ctrl+V.
if focused_element.is_some_and(|element| !element.is_terminal) {
return KdeClipboardPasteShortcut::Standard;
}
match terminal {
Some(TerminalPasteShortcut::CtrlShiftV) => KdeClipboardPasteShortcut::CtrlShiftV,
Some(TerminalPasteShortcut::ShiftInsert) => KdeClipboardPasteShortcut::ShiftInsert,
None => KdeClipboardPasteShortcut::Standard,
}
}
fn kde_clipboard_paste_chord(shortcut: KdeClipboardPasteShortcut) -> (&'static [i32], i32) {
match shortcut {
KdeClipboardPasteShortcut::Standard => (&[EVDEV_KEY_LEFTCTRL], EVDEV_KEY_V),
KdeClipboardPasteShortcut::CtrlShiftV => {
(&[EVDEV_KEY_LEFTCTRL, EVDEV_KEY_LEFTSHIFT], EVDEV_KEY_V)
}
KdeClipboardPasteShortcut::ShiftInsert => (&[EVDEV_KEY_LEFTSHIFT], EVDEV_KEY_INSERT),
}
}
async fn kde_clipboard_contents() -> std::result::Result<String, String> {
let connection = kde_clipboard_connection().await?;
let proxy = kde_clipboard_proxy(&connection).await?;
let output: String = kde_clipboard_dbus_operation(
"getClipboardContents",
proxy.call("getClipboardContents", &()),
)
.await?;
Ok(output)
}
async fn kde_set_clipboard_contents(text: &str) -> std::result::Result<(), String> {
let connection = kde_clipboard_connection().await?;
let proxy = kde_clipboard_proxy(&connection).await?;
let _: () = kde_clipboard_dbus_operation(
"setClipboardContents",
proxy.call("setClipboardContents", &(text)),
)
.await?;
Ok(())
}
async fn kde_clipboard_connection() -> std::result::Result<ZbusConnection, String> {
ZbusConnection::session()
.await
.map_err(|error| format!("failed to connect to session bus for KDE clipboard: {error}"))
}
async fn kde_clipboard_proxy(
connection: &ZbusConnection,
) -> std::result::Result<ZbusProxy<'_>, String> {
kde_clipboard_dbus_operation(
"proxy creation",
ZbusProxy::new(
connection,
KDE_KLIPPER_SERVICE,
KDE_KLIPPER_PATH,
KDE_KLIPPER_INTERFACE,
),
)
.await
}
async fn kde_clipboard_dbus_operation<T, F>(
operation: &'static str,
future: F,
) -> std::result::Result<T, String>
where
F: Future<Output = zbus::Result<T>>,
{
kde_clipboard_dbus_operation_with_timeout(operation, future, KDE_CLIPBOARD_DBUS_TIMEOUT).await
}
async fn kde_clipboard_dbus_operation_with_timeout<T, F>(
operation: &'static str,
future: F,
timeout_duration: Duration,
) -> std::result::Result<T, String>
where
F: Future<Output = zbus::Result<T>>,
{
timeout(timeout_duration, future)
.await
.map_err(|_| format!("KDE clipboard {operation} timed out"))?
.map_err(|error| format!("KDE clipboard {operation} failed: {error}"))
}
fn ydotool_output_error(output: Output) -> String {
command_output_error("ydotool", output)
}
/// X11 keyboard input runs through `xdotool` (XTEST) instead of ydotool.
///
/// ydotool injects raw evdev keycodes into a virtual uinput device. Under X11
/// the server re-interprets those scancodes through the active XKB layout, so
/// named keys and chords land as unrelated glyphs and literal text can mangle
/// symbols/digits (`_` → `%`, `1` → `+`). XTEST resolves keysyms against the
/// live layout, which is what X11 clients actually expect. See issue #58.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum KeyboardCommandBackend {
Wtype,
Xdotool,
Ydotool,
}
struct KeyboardCommandResult {
output: Output,
backend: KeyboardCommandBackend,
}
enum XdotoolAttempt {
Unavailable,
Finished(std::result::Result<Output, String>),
}
async fn run_xdotool(program: &Path, args: &[String]) -> XdotoolAttempt {
run_xdotool_with_timeout(program, args, INPUT_COMMAND_TIMEOUT).await
}
async fn run_xdotool_with_timeout(
program: &Path,
args: &[String],
command_timeout: Duration,
) -> XdotoolAttempt {
let mut command = TokioCommand::new(program);
command.args(args);
command.stdout(Stdio::piped());
command.stderr(Stdio::piped());
command.kill_on_drop(true);
command.process_group(0);
// A busy-file spawn error is transient, not "xdotool is unavailable"; do
// not let it route input to the ydotool fallback.
match crate::command_runner::spawn_retrying_busy(&mut command).await {
Ok(child) => XdotoolAttempt::Finished(
match crate::command_runner::output_child(child, "run xdotool", command_timeout)
.await
.map_err(|error| format!("{error:#}"))
{
Ok(output) if output.status.success() => Ok(output),
Ok(output) => Err(command_output_error("xdotool", output)),
Err(error) => Err(error),
},
),
Err(_) => XdotoolAttempt::Unavailable,
}
}
async fn run_xdotool_or_fallback<F, Fut>(
program: &Path,
args: &[String],
fallback: F,
) -> std::result::Result<KeyboardCommandResult, String>
where
F: FnOnce() -> Fut,
Fut: Future<Output = std::result::Result<Output, String>>,
{
run_xdotool_or_fallback_with_timeout(program, args, INPUT_COMMAND_TIMEOUT, fallback).await
}
async fn run_xdotool_or_fallback_with_timeout<F, Fut>(
program: &Path,
args: &[String],
command_timeout: Duration,
fallback: F,
) -> std::result::Result<KeyboardCommandResult, String>
where
F: FnOnce() -> Fut,
Fut: Future<Output = std::result::Result<Output, String>>,
{
match run_xdotool_with_timeout(program, args, command_timeout).await {
XdotoolAttempt::Unavailable => fallback().await.map(|output| KeyboardCommandResult {
output,
backend: KeyboardCommandBackend::Ydotool,
}),
XdotoolAttempt::Finished(result) => result.map(|output| KeyboardCommandResult {
output,
backend: KeyboardCommandBackend::Xdotool,
}),
}
}
async fn run_wtype_type_text_or_fallback<F, Fut>(
program: &Path,
text: &str,
mode: WtypeTextMode,
secret: bool,
fallback: F,
) -> std::result::Result<KeyboardCommandResult, String>
where
F: FnOnce() -> Fut,
Fut: Future<Output = std::result::Result<Output, String>>,
{
// Defense in depth: no helper caller may put a secret in argv or bypass
// validation through the missing-program fallback.
validate_wtype_text(text)?;
validate_wtype_privacy(mode, secret)?;
let available = if program.components().count() > 1 {
std::fs::metadata(program)
.map(|meta| meta.is_file())
.unwrap_or(false)
} else {
program.to_str().is_some_and(which_in_path)
};
if !available {
return fallback().await.map(|output| KeyboardCommandResult {
output,
backend: KeyboardCommandBackend::Ydotool,
});
}
let mut command = TokioCommand::new(program);
// wtype's stdin mode uploads a new keymap every 100 characters. On
// Hyprland/Electron those changes can turn literal text into modifier
// keys and shortcuts while wtype still exits successfully. Passing one
// literal argument builds the complete keymap before the first key event.
// `--` prevents option-looking text from becoming wtype commands.
let output = match mode {
WtypeTextMode::Argument => {
command.args(["--", text]);
crate::command_runner::output_with_timeout(
command,
"run wtype",
ydotool_type_timeout(text),
)
.await
}
WtypeTextMode::Stdin => {
command.arg("-");
crate::command_runner::output_with_stdin(
command,
"run wtype",
ydotool_type_timeout(text),
text.as_bytes().to_vec(),
)
.await
}
}
.map_err(|error| format!("{error:#}"))?;
if !output.status.success() {
return Err(command_output_error("wtype", output));
}
Ok(KeyboardCommandResult {
output,
backend: KeyboardCommandBackend::Wtype,
})
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum WtypeTextMode {
Argument,
Stdin,
}
enum WtypeFieldKind {
Password,
NonSecretText,
Ambiguous,
}
fn classify_wtype_field(element: &FocusedElementSummary) -> WtypeFieldKind {
// The overlay's mask predicate includes uncertainty. Only an explicit
// password role confirms a password; terminals can be password prompts.
if element.role.to_ascii_lowercase().contains("password") {
WtypeFieldKind::Password
} else if !element.is_terminal
&& !crate::indicator::is_secret_role(&element.role)
&& focused_element_is_editable(element)
{
WtypeFieldKind::NonSecretText
} else {
WtypeFieldKind::Ambiguous
}
}
fn wtype_text_mode_for_desktop(desktop: Option<&str>) -> WtypeTextMode {
if desktop.is_some_and(|value| {
value
.split(':')
.any(|part| part.eq_ignore_ascii_case("hyprland"))
}) {
WtypeTextMode::Argument
} else {
WtypeTextMode::Stdin
}
}
fn preflight_type_text(text: &str, mode: Option<WtypeTextMode>) -> std::result::Result<(), String> {
if mode.is_some() {
validate_wtype_text(text)?;
} else if text.contains('\0') {
return Err("Literal text was not sent: NUL characters are unsupported.".to_string());
}
Ok(())
}
async fn hyprland_wtype_delivery(
text: &str,
probe: impl Future<Output = Option<FocusProbe>>,
) -> std::result::Result<WtypeTextMode, String> {
let probe = timeout(WTYPE_FOCUS_TIMEOUT, probe).await.ok().flatten();
if let Some(FocusProbe::Found(element)) = probe.as_ref() {
match classify_wtype_field(element) {
WtypeFieldKind::Password => return Err("Literal text was not sent: wtype argument mode is disabled for password fields. Use set_value on an editable field.".to_string()),
WtypeFieldKind::NonSecretText => return Ok(WtypeTextMode::Argument),
WtypeFieldKind::Ambiguous => {}
}
}
// Terminals and generic text roles remain ambiguous even with AT-SPI.
// Keep their stdin route, bounded to a single wtype batch so no mid-text
// keymap refresh is needed. Ambiguous text must never be exposed in argv.
if text.chars().count() <= WTYPE_STDIN_MAX_CHARS {
Ok(WtypeTextMode::Stdin)
} else {
Err(format!("Literal text was not sent: the focused field could not be classified. Hyprland's private stdin fallback accepts at most {WTYPE_STDIN_MAX_CHARS} Unicode characters per call to avoid keymap-refresh corruption. Send smaller calls and verify each one, or enable the target's accessibility support."))
}
}
fn validate_wtype_privacy(mode: WtypeTextMode, secret: bool) -> std::result::Result<(), String> {
if mode == WtypeTextMode::Argument && secret {
return Err("Literal text was not sent: wtype argument mode is disabled for password fields and fields whose secrecy could not be determined. Enable the target application's accessibility support or use set_value on an editable field.".to_string());
}
Ok(())
}
fn focus_requires_secret_protection(probe: &FocusProbe) -> bool {
match probe {
FocusProbe::Found(element) => {
// Electron exposes the Editable state on normal entries without
// implementing EditableText. The interface is not a secrecy test.
let accepts_text = focused_element_is_editable(element) || element.is_terminal;
crate::indicator::is_secret_role(&element.role) || !accepts_text
}
_ => true,
}
}
fn focused_element_is_editable(element: &FocusedElementSummary) -> bool {
element.editable || element.states.iter().any(|state| state == "editable")
}
fn validate_wtype_text(text: &str) -> std::result::Result<(), String> {
if text.contains('\0') {
return Err(
"Literal text was not sent: wtype cannot represent NUL characters.".to_string(),
);
}
if text.len() > WTYPE_MAX_TEXT_BYTES {
return Err(format!(
"Literal text was not sent: wtype accepts at most {WTYPE_MAX_TEXT_BYTES} UTF-8 bytes per call. Use set_value on an editable field or send smaller text segments."
));
}
Ok(())
}
/// True when `xdotool` can drive this session: an X11 session with `DISPLAY`
/// set and the binary present. `COMPUTER_USE_LINUX_FORCE_YDOTOOL_KEYBOARD=1`
/// opts out; `COMPUTER_USE_LINUX_FORCE_XDOTOOL_KEYBOARD=1` forces it on.
fn xdotool_available() -> bool {
which_in_path("xdotool")
}
fn wtype_available() -> bool {
which_in_path("wtype")
}
fn prefer_wtype_keyboard(
force_ydotool: bool,
is_wayland: bool,
compatible_desktop: bool,
available: bool,
) -> bool {
!force_ydotool && is_wayland && compatible_desktop && available
}
/// Default per-character delay for `xdotool type`, in milliseconds (xdotool's
/// own default). `--delay 0` lets XTEST key events race each other on some X
/// servers (Cinnamon on Mint 22, issue #147): every character arrives, but in
/// a random order. A small delay keeps them ordered.
/// `COMPUTER_USE_LINUX_XDOTOOL_TYPE_DELAY_MS` overrides it.
const XDOTOOL_TYPE_DELAY_MS: u64 = 12;
const XDOTOOL_TYPE_DELAY_ENV: &str = "COMPUTER_USE_LINUX_XDOTOOL_TYPE_DELAY_MS";
fn xdotool_type_delay_ms() -> u64 {
env::var(XDOTOOL_TYPE_DELAY_ENV)
.ok()
.and_then(|value| value.trim().parse::<u64>().ok())
.unwrap_or(XDOTOOL_TYPE_DELAY_MS)
}
fn xdotool_type_args_with_delay(text: &str, delay_ms: u64) -> Vec<String> {
// XTEST maps LF to Linefeed, which editors ignore. CR maps to Return.
// Fold CRLF first so a Windows line ending still inserts one newline.
let text = text.replace("\r\n", "\n").replace('\n', "\r");
vec![
"type".to_string(),
"--clearmodifiers".to_string(),
"--delay".to_string(),
delay_ms.to_string(),
"--".to_string(),
text,
]
}
/// `xdotool type` spends about `delay_ms` per character, so long inputs need
/// more than the flat input timeout.
fn xdotool_type_timeout(text: &str, delay_ms: u64) -> Duration {
let chars = text.chars().count() as u64;
INPUT_COMMAND_TIMEOUT.saturating_add(Duration::from_millis(chars.saturating_mul(delay_ms)))
}
fn which_in_path(binary: &str) -> bool {
let Ok(path) = env::var("PATH") else {
return false;
};
env::split_paths(&path).any(|dir| {
let candidate = dir.join(binary);
std::fs::metadata(&candidate)
.map(|meta| meta.is_file())
.unwrap_or(false)
})
}
/// Map our key grammar onto an `xdotool key` spec such as `ctrl+a`, `Return`,
/// or `shift+F5`. Returns `None` for keys the grammar does not accept, so the
/// caller keeps its existing "never silently dropped" error.
fn xdotool_key_spec(key: &str) -> Option<String> {
let parts = key
.split('+')
.map(str::trim)
.filter(|part| !part.is_empty())
.collect::<Vec<_>>();
let (key_part, modifier_parts) = parts.split_last()?;
// Validate through the same evdev grammar so both backends accept exactly
// the same input set.
key_chord(key)?;
let mut spec = Vec::new();
for part in modifier_parts {
spec.push(xdotool_modifier_name(part)?.to_string());
}
if modifier_parts.is_empty() {
if let Some(bare) = xdotool_modifier_keysym(key_part) {
return Some(bare.to_string());
}
}
spec.push(xdotool_keysym_name(key_part)?);
Some(spec.join("+"))
}
fn xdotool_modifier_name(key: &str) -> Option<&'static str> {
match normalize_key(key).as_str() {
"ctrl" | "control" => Some("ctrl"),
"alt" | "option" => Some("alt"),
"shift" => Some("shift"),
"meta" | "super" | "cmd" | "command" => Some("super"),
_ => None,
}
}
/// Standalone keysym for a bare modifier press (`press_key "Super"`).
fn xdotool_modifier_keysym(key: &str) -> Option<&'static str> {
match normalize_key(key).as_str() {
"ctrl" | "control" => Some("ctrl"),
"alt" | "option" => Some("alt"),
"shift" => Some("shift"),
"meta" | "super" | "cmd" | "command" => Some("super"),
_ => None,
}
}
fn xdotool_keysym_name(key: &str) -> Option<String> {
let normalized = normalize_key(key);
let named = match normalized.as_str() {
"enter" | "return" => "Return",
"escape" | "esc" => "Escape",
"tab" => "Tab",
"backspace" => "BackSpace",
"delete" | "del" => "Delete",
"space" => "space",
"home" => "Home",
"end" => "End",
"pageup" | "page_up" => "Page_Up",
"pagedown" | "page_down" => "Page_Down",
"arrowleft" | "left" => "Left",
"arrowright" | "right" => "Right",
"arrowup" | "up" => "Up",
"arrowdown" | "down" => "Down",
"f1" => "F1",
"f2" => "F2",
"f3" => "F3",
"f4" => "F4",
"f5" => "F5",
"f6" => "F6",
"f7" => "F7",
"f8" => "F8",
"f9" => "F9",
"f10" => "F10",
"f11" => "F11",
"f12" => "F12",
value if value.len() == 1 && value.as_bytes()[0].is_ascii_alphanumeric() => {
return Some(value.to_string());
}
_ => return None,
};
Some(named.to_string())
}
fn command_output_error(command: &str, output: Output) -> String {
let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string();
let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string();
let detail = if stderr.is_empty() { stdout } else { stderr };
if detail.is_empty() {
format!("{command} exited with {}", output.status)
} else {
detail
}
}
async fn ydotool_backend_available() -> bool {
ydotool_backend_available_from(
ydotool::connectable_socket_path().is_ok(),
ydotool::ensure_supported_async().await.is_ok(),
)
}
fn ydotool_backend_available_from(socket_available: bool, cli_supported: bool) -> bool {
socket_available && cli_supported
}
fn should_prefer_portal_backend_by_default(is_wayland: bool, ydotool_available: bool) -> bool {
is_wayland && !ydotool_available
}
fn prefer_portal_text_backend(
force_ydotool: bool,
prefer_xdotool: bool,
is_wayland: bool,
is_kde: bool,
force_portal: bool,
prefer_wtype: bool,
) -> bool {
!force_ydotool && !prefer_xdotool && is_wayland && !is_kde && (force_portal || !prefer_wtype)
}
fn mouse_button_code(button: Option<&str>) -> String {
match button.unwrap_or("left").to_ascii_lowercase().as_str() {
"right" => "0xC1",
"middle" => "0xC2",
"side" => "0xC3",
"extra" => "0xC4",
"forward" => "0xC5",
"back" => "0xC6",
_ => "0xC0",
}
.to_string()
}
/// Parse a chord like `Ctrl+Shift+P` into raw evdev codes: the held
/// modifiers plus the final key. A bare modifier (`Super`) parses as a
/// chord with no held modifiers whose key is the modifier itself.
fn key_chord(key: &str) -> Option<(Vec<u16>, u16)> {
let parts = key
.split('+')
.map(str::trim)
.filter(|part| !part.is_empty())
.collect::<Vec<_>>();
let (key_part, modifier_parts) = parts.split_last()?;
if modifier_parts.is_empty() {
if let Some(modifier) = modifier_keycode(key_part) {
return Some((Vec::new(), modifier));
}
}
let mut modifiers = Vec::new();
for part in modifier_parts {
modifiers.push(modifier_keycode(part)?);
}
let keycode = keycode(key_part)?;
Some((modifiers, keycode))
}
/// Portal form of [`key_chord`]. Modifiers and named keys go out as keysyms
/// so the compositor resolves them against the active keymap: with Caps Lock
/// and Control swapped, `Ctrl` presses whichever key produces `Control_L`
/// (issue #191). Letters and digits stay physical keycodes: mutter drops a
/// keysym missing from the current layout group, so `Ctrl+A` as a keysym
/// would vanish under a non-Latin layout, while the US keycode is what GTK and
/// Qt match shortcuts against there.
fn portal_key_chord(key: &str) -> Option<(Vec<PortalKey>, PortalKey)> {
let parts = key
.split('+')
.map(str::trim)
.filter(|part| !part.is_empty())
.collect::<Vec<_>>();
let (key_part, modifier_parts) = parts.split_last()?;
let (_, keycode) = key_chord(key)?;
let mut modifiers = Vec::new();
for part in modifier_parts {
modifiers.push(PortalKey::Keysym(modifier_keysym(part)?));
}
let key = modifier_keysym(key_part)
.filter(|_| modifier_parts.is_empty())
.or_else(|| named_keysym(key_part))
.map_or(PortalKey::Keycode(i32::from(keycode)), PortalKey::Keysym);
Some((modifiers, key))
}
fn modifier_keysym(key: &str) -> Option<i32> {
let keysym = match normalize_key(key).as_str() {
"ctrl" | "control" => xkey::Control_L,
"alt" | "option" => xkey::Alt_L,
"shift" => xkey::Shift_L,
"meta" | "super" | "cmd" | "command" => xkey::Super_L,
_ => return None,
};
Some(keysym as i32)
}
fn named_keysym(key: &str) -> Option<i32> {
let keysym = match normalize_key(key).as_str() {
"enter" | "return" => xkey::Return,
"escape" | "esc" => xkey::Escape,
"tab" => xkey::Tab,
"backspace" => xkey::BackSpace,
"delete" | "del" => xkey::Delete,
"space" => xkey::space,
"home" => xkey::Home,
"end" => xkey::End,
"pageup" | "page_up" => xkey::Page_Up,
"pagedown" | "page_down" => xkey::Page_Down,
"arrowleft" | "left" => xkey::Left,
"arrowright" | "right" => xkey::Right,
"arrowup" | "up" => xkey::Up,
"arrowdown" | "down" => xkey::Down,
"f1" => xkey::F1,
"f2" => xkey::F2,
"f3" => xkey::F3,
"f4" => xkey::F4,
"f5" => xkey::F5,
"f6" => xkey::F6,
"f7" => xkey::F7,
"f8" => xkey::F8,
"f9" => xkey::F9,
"f10" => xkey::F10,
"f11" => xkey::F11,
"f12" => xkey::F12,
_ => return None,
};
Some(keysym as i32)
}
fn key_sequence(key: &str) -> Option<Vec<String>> {
let (modifiers, keycode) = key_chord(key)?;
let mut events = Vec::new();
for modifier in &modifiers {
events.push(format!("{modifier}:1"));
}
events.push(format!("{keycode}:1"));
events.push(format!("{keycode}:0"));
for modifier in modifiers.iter().rev() {
events.push(format!("{modifier}:0"));
}
Some(events)
}
fn ydotool_key_args(key_events: Vec<String>, has_modifiers: bool) -> Vec<String> {
let mut args = vec!["key".to_string()];
if has_modifiers {
args.extend(["-d".to_string(), "100".to_string()]);
}
args.extend(key_events);
args
}
fn modifier_keycode(key: &str) -> Option<u16> {
match normalize_key(key).as_str() {
"ctrl" | "control" => Some(29),
"alt" | "option" => Some(56),
"shift" => Some(42),
"meta" | "super" | "cmd" | "command" => Some(125),
_ => None,
}
}
fn keycode(key: &str) -> Option<u16> {
match normalize_key(key).as_str() {
"enter" | "return" => Some(28),
"escape" | "esc" => Some(1),
"tab" => Some(15),
"backspace" => Some(14),
"delete" | "del" => Some(111),
"space" => Some(57),
"home" => Some(102),
"end" => Some(107),
"pageup" | "page_up" => Some(104),
"pagedown" | "page_down" => Some(109),
"arrowleft" | "left" => Some(105),
"arrowright" | "right" => Some(106),
"arrowup" | "up" => Some(103),
"arrowdown" | "down" => Some(108),
"f1" => Some(59),
"f2" => Some(60),
"f3" => Some(61),
"f4" => Some(62),
"f5" => Some(63),
"f6" => Some(64),
"f7" => Some(65),
"f8" => Some(66),
"f9" => Some(67),
"f10" => Some(68),
"f11" => Some(87),
"f12" => Some(88),
value if value.len() == 1 => keycode_for_ascii(value.as_bytes()[0] as char),
_ => None,
}
}
fn normalize_key(key: &str) -> String {
key.trim().to_ascii_lowercase().replace(['-', ' '], "")
}
fn keycode_for_ascii(value: char) -> Option<u16> {
match value {
'a' => Some(30),
'b' => Some(48),
'c' => Some(46),
'd' => Some(32),
'e' => Some(18),
'f' => Some(33),
'g' => Some(34),
'h' => Some(35),
'i' => Some(23),
'j' => Some(36),
'k' => Some(37),
'l' => Some(38),
'm' => Some(50),
'n' => Some(49),
'o' => Some(24),
'p' => Some(25),
'q' => Some(16),
'r' => Some(19),
's' => Some(31),
't' => Some(20),
'u' => Some(22),
'v' => Some(47),
'w' => Some(17),
'x' => Some(45),
'y' => Some(21),
'z' => Some(44),
'1' => Some(2),
'2' => Some(3),
'3' => Some(4),
'4' => Some(5),
'5' => Some(6),
'6' => Some(7),
'7' => Some(8),
'8' => Some(9),
'9' => Some(10),
'0' => Some(11),
_ => None,
}
}
fn list_process_apps() -> Vec<AppCandidate> {
let output = Command::new("ps")
.args(["-eo", "pid=,comm=,args="])
.output();
let Ok(output) = output else {
return Vec::new();
};
if !output.status.success() {
return Vec::new();
}
String::from_utf8_lossy(&output.stdout)
.lines()
.filter_map(parse_process_line)
.filter(|app| looks_like_desktop_app(&app.name, &app.command))
.take(50)
.collect()
}
fn parse_process_line(line: &str) -> Option<AppCandidate> {
let trimmed = line.trim();
let mut parts = trimmed.splitn(3, char::is_whitespace);
let pid = parts.next()?.parse().ok()?;
let name = parts.next()?.to_string();
let command = parts.next().unwrap_or("").trim().to_string();
Some(AppCandidate { name, pid, command })
}
fn looks_like_desktop_app(name: &str, command: &str) -> bool {
let haystack = format!("{name} {command}").to_ascii_lowercase();
[
"codex",
"electron",
"chrome",
"chromium",
"firefox",
"brave",
"code",
"gnome-terminal",
"ptyxis",
"kgx",
"nautilus",
"slack",
"discord",
"spotify",
"obsidian",
]
.iter()
.any(|needle| haystack.contains(needle))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::atspi_tree::{AccessibilityAction, Bounds};
use crate::windows::{WindowBounds, GNOME_SHELL_EXTENSION_BACKEND};
use std::os::unix::fs::PermissionsExt;
/// Restores an environment variable when a test ends, including on panic.
struct EnvVarGuard {
key: &'static str,
original: Option<std::ffi::OsString>,
}
impl EnvVarGuard {
fn set(key: &'static str, value: &str) -> Self {
let original = std::env::var_os(key);
std::env::set_var(key, value);
Self { key, original }
}
fn unset(key: &'static str) -> Self {
let original = std::env::var_os(key);
std::env::remove_var(key);
Self { key, original }
}
}
impl Drop for EnvVarGuard {
fn drop(&mut self) {
match &self.original {
Some(value) => std::env::set_var(self.key, value),
None => std::env::remove_var(self.key),
}
}
}
/// The run_shell tests change the same process-wide variable, so they
/// take turns instead of racing each other under the parallel test runner.
static SHELL_ENV_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
/// Exercise the real handlers in a separate process so backend overrides
/// cannot race other tests. All command programs are harmless stubs; the
/// private datagram socket and invalid D-Bus address cannot reach a desktop.
#[test]
fn pointer_coordinate_backend_dispatch_is_safe() {
let dir = std::env::temp_dir().join(format!(
"cul-pointer-safety-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir(&dir).unwrap();
std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).unwrap();
struct Cleanup(std::path::PathBuf);
impl Drop for Cleanup {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.0);
}
}
let _cleanup = Cleanup(dir.clone());
let socket_path = dir.join("input.sock");
let _socket = std::os::unix::net::UnixDatagram::bind(&socket_path).unwrap();
for program in ["ydotool", "xdotool", "systemctl", "gnome-screenshot"] {
let path = dir.join(program);
std::fs::write(
&path,
r#"#!/bin/sh
case "${0##*/}" in
systemctl) exit 0 ;;
gnome-screenshot) printf 'capture\n' >> "$CUL_POINTER_TEST_CAPTURE_LOG"; exit 1 ;;
ydotool)
case "$1" in help|--help) printf '%s\n' click mousemove type key debug; exit 0 ;; esac
# Capability probes use their own socket. Record only dispatched input.
[ "$YDOTOOL_SOCKET" = "$CUL_POINTER_TEST_SOCKET" ] || exit 0 ;;
esac
printf '%s %s\n' "${0##*/}" "$*" >> "$CUL_POINTER_TEST_LOG"
"#,
)
.unwrap();
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o700)).unwrap();
}
for mode in [
"wayland-default",
"wayland-uncached",
"wayland-ydotool",
"wayland-portal",
"x11",
] {
let log = dir.join(format!("{mode}.log"));
let capture_log = dir.join(format!("{mode}-capture.log"));
let mut command = Command::new(std::env::current_exe().unwrap());
command
.args([
"--exact",
"server::tests::pointer_coordinate_backend_child",
"--nocapture",
])
.env_clear()
.env("PATH", &dir)
.env("HOME", &dir)
.env("CUL_POINTER_TEST_MODE", mode)
.env("CUL_POINTER_TEST_LOG", &log)
.env("CUL_POINTER_TEST_CAPTURE_LOG", &capture_log)
.env("COMPUTER_USE_LINUX_SCREENSHOT_BACKEND", "gnome-screenshot")
.env("CUL_POINTER_TEST_SOCKET", &socket_path)
.env("YDOTOOL_SOCKET", &socket_path)
.env("XDG_RUNTIME_DIR", &dir)
.env(
"DBUS_SESSION_BUS_ADDRESS",
format!("unix:path={}/no-bus", dir.display()),
)
.env("DISPLAY", ":9876")
.env(
"WAYLAND_DISPLAY",
if mode == "x11" { "" } else { "no-wayland" },
)
.env(
"XDG_SESSION_TYPE",
if mode == "x11" { "x11" } else { "wayland" },
)
.env("CU_DISABLE_ABS_POINTER", "1")
.env(
"COMPUTER_USE_LINUX_FORCE_YDOTOOL_POINTER",
if mode == "wayland-ydotool" { "1" } else { "0" },
)
.env(
"COMPUTER_USE_LINUX_FORCE_PORTAL_POINTER",
if mode == "wayland-portal" { "1" } else { "0" },
);
for key in [
"DESKTOP_SESSION",
"HYPRLAND_INSTANCE_SIGNATURE",
"XAUTHORITY",
"XDG_SESSION_DESKTOP",
"XDG_CURRENT_DESKTOP",
] {
command.env(key, "pointer-test");
}
let output = crate::command_runner::output_blocking_with_timeout(
&mut command,
"pointer backend regression",
Duration::from_secs(15),
)
.unwrap();
assert!(
output.status.success(),
"{mode}: {}{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
assert!(
!capture_log.exists(),
"{mode}: a pointer warning triggered screenshot capture"
);
let commands = std::fs::read_to_string(&log).unwrap_or_default();
match mode {
"wayland-portal" => assert!(commands.is_empty(), "{commands}"),
"x11" => {
assert!(
commands.contains("xdotool mousemove -- 5536 700 click"),
"{commands}"
);
assert!(
commands.contains("ydotool mousemove --absolute -- 100 100"),
"{commands}"
);
}
_ => assert_eq!(commands, "ydotool mousemove --wheel -- 0 -5\n"),
}
}
}
#[tokio::test]
async fn pointer_coordinate_backend_child() {
let Ok(mode) = std::env::var("CUL_POINTER_TEST_MODE") else {
return;
};
let backend = ComputerUseLinux::default();
if mode != "wayland-uncached" {
backend.cache_desktop_size(9376, 1600);
assert!(backend.off_screen_note_for_point(9376, 700).await.is_some());
}
assert!(backend.off_screen_note_for_point(5536, 700).await.is_none());
let click: ClickParams =
serde_json::from_value(serde_json::json!({"x":5536,"y":700})).unwrap();
let scroll: ScrollParams =
serde_json::from_value(serde_json::json!({"x":5536,"y":700,"direction":"down"}))
.unwrap();
let drag = DragParams {
start_x: 100,
start_y: 100,
end_x: 200,
end_y: 200,
};
let results = [
backend.click(Parameters(click)).await.0,
backend.scroll(Parameters(scroll)).await.0,
backend.drag(Parameters(drag)).await.0,
];
for result in results {
if mode == "x11" {
assert!(result.ok, "{}", result.message);
} else {
assert!(!result.ok, "{}", result.message);
let expected = if mode == "wayland-portal" {
"portal could not start"
} else {
"cannot position the pointer accurately on Wayland"
};
assert!(result.message.contains(expected), "{}", result.message);
}
}
let scroll: ScrollParams =
serde_json::from_value(serde_json::json!({"direction":"down"})).unwrap();
let result = backend.scroll(Parameters(scroll)).await.0;
assert_eq!(result.ok, mode != "wayland-portal", "{}", result.message);
}
#[test]
fn completion_tool_is_explicitly_opt_in_and_has_side_effect_annotations() {
let server = ComputerUseLinux::default();
assert!(!server
.router_with_completion(false)
.list_all()
.iter()
.any(|t| t.name == "complete_interaction"));
let tools = server.router_with_completion(true).list_all();
let tool = tools
.iter()
.find(|t| t.name == "complete_interaction")
.unwrap();
let value = serde_json::to_value(tool).unwrap();
assert_eq!(value["annotations"]["readOnlyHint"], false);
assert_eq!(value["annotations"]["destructiveHint"], false);
assert_eq!(value["annotations"]["idempotentHint"], false);
assert_eq!(value["annotations"]["openWorldHint"], true);
}
#[tokio::test]
async fn completion_notification_handles_missing_and_failed_backends() {
for path in ["/nonexistent/computer-use-notify", "/bin/false"] {
let result =
send_completion_notification(Path::new(path), Duration::from_secs(2)).await;
assert!(result.ok);
assert_eq!(result.cue, "skipped");
}
let result =
send_completion_notification(Path::new("/bin/true"), Duration::from_secs(2)).await;
assert_eq!(result.cue, "notification");
}
#[tokio::test]
async fn completion_notification_timeout_is_bounded() {
let dir = std::env::temp_dir().join(format!(
"computer-use-notify-test-{}-{:?}",
std::process::id(),
std::time::SystemTime::now()
));
std::fs::create_dir_all(&dir).unwrap();
let program = dir.join("notify");
std::fs::write(&program, "#!/bin/sh\nexec sleep 30\n").unwrap();
std::fs::set_permissions(&program, std::fs::Permissions::from_mode(0o700)).unwrap();
let started = std::time::Instant::now();
let output = send_completion_notification(&program, Duration::from_millis(30)).await;
assert_eq!(output.cue, "skipped");
assert!(started.elapsed() < Duration::from_secs(3));
std::fs::remove_dir_all(dir).unwrap();
}
#[test]
fn exported_tool_schemas_omit_unsigned_integer_formats() {
let tools = ComputerUseLinux::default().mcp_tool_router().list_all();
let value = serde_json::to_value(tools).unwrap();
let mut unsupported = Vec::new();
collect_unsigned_integer_formats(&value, "$", &mut unsupported);
assert!(
unsupported.is_empty(),
"unsupported unsigned integer formats: {unsupported:?}"
);
}
fn collect_unsigned_integer_formats(
value: &serde_json::Value,
path: &str,
unsupported: &mut Vec<String>,
) {
match value {
serde_json::Value::Object(object) => {
if matches!(
object.get("format").and_then(serde_json::Value::as_str),
Some("uint" | "uint8" | "uint16" | "uint32" | "uint64" | "usize")
) {
unsupported.push(path.to_string());
}
for (key, nested) in object {
collect_unsigned_integer_formats(nested, &format!("{path}/{key}"), unsupported);
}
}
serde_json::Value::Array(items) => {
for (index, nested) in items.iter().enumerate() {
collect_unsigned_integer_formats(
nested,
&format!("{path}/{index}"),
unsupported,
);
}
}
_ => {}
}
}
fn node(index: u32, bounds: Option<Bounds>) -> AccessibilityNode {
node_with_actions(index, bounds, Vec::new())
}
#[test]
fn shell_only_tree_explains_missing_renderer_content() {
let mut frame = node(0, None);
frame.role = "frame".to_string();
let nodes = [frame];
let note = shell_only_accessibility_tree_note(&nodes, true, false).unwrap();
assert!(note.contains("--force-renderer-accessibility"));
assert!(note.contains("If this is an Electron/Chromium app"));
assert!(shell_only_accessibility_tree_note(&nodes, false, false).is_none());
assert!(shell_only_accessibility_tree_note(&nodes, true, true).is_none());
assert!(shell_only_accessibility_tree_note(&[], true, false).is_none());
}
#[test]
fn content_and_editable_trees_do_not_get_renderer_hint() {
let mut content = node(0, None);
assert!(shell_only_accessibility_tree_note(&[content.clone()], true, false).is_none());
content.role = "frame".to_string();
content.supports_editable_text = true;
assert!(shell_only_accessibility_tree_note(&[content], true, false).is_none());
}
#[test]
fn native_menu_chrome_does_not_hide_the_renderer_hint() {
let mut frame = node(0, None);
frame.role = "frame".to_string();
let mut menu = node(1, None);
menu.role = "menu item".to_string();
menu.actions.push(AccessibilityAction {
index: 0,
name: "click".to_string(),
description: String::new(),
keybinding: String::new(),
});
assert!(
shell_only_accessibility_tree_note(&[frame.clone(), menu.clone()], true, false)
.is_some()
);
menu.supports_editable_text = true;
assert!(shell_only_accessibility_tree_note(&[frame, menu], true, false).is_none());
}
#[test]
fn native_menu_buttons_are_chrome_only_beneath_a_menu() {
let mut frame = node(0, None);
frame.role = "frame".to_string();
let mut panel = node(1, None);
panel.role = "panel".to_string();
panel.parent_index = Some(0);
let mut menu = node(2, None);
menu.role = "menu bar".to_string();
menu.parent_index = Some(1);
let mut button = node(3, None);
button.role = "button".to_string();
button.parent_index = Some(2);
assert!(shell_only_accessibility_tree_note(
&[frame.clone(), panel.clone(), menu.clone(), button.clone()],
true,
false
)
.is_some());
button.parent_index = Some(1);
assert!(
shell_only_accessibility_tree_note(&[frame, panel, menu, button], true, false)
.is_none()
);
}
#[test]
fn review_hint_keeps_raw_menu_ancestry_after_compaction() {
let mut frame = node(0, None);
frame.role = "frame".to_string();
frame.depth = 1;
let mut panel = node(1, None);
panel.role = "panel".to_string();
panel.name = None;
panel.depth = 2;
panel.parent_index = Some(0);
let mut menu = node(2, None);
menu.role = "menu bar".to_string();
menu.name = None;
menu.depth = 3;
menu.parent_index = Some(1);
let mut button = node(3, None);
button.role = "button".to_string();
button.depth = 4;
button.parent_index = Some(2);
let nodes = vec![frame, panel, menu, button];
let (compacted, note) = compact_accessibility_tree_with_note(nodes.clone(), true, false);
assert_eq!(
compacted.len(),
2,
"unnamed panel and menu bar should be compacted away"
);
assert_eq!(compacted[1].parent_index, Some(0));
assert!(
shell_only_accessibility_tree_note(&compacted, true, false).is_none(),
"compacted data cannot recover the removed menu context"
);
assert!(
note.is_some(),
"raw menu ancestry must determine the observation hint"
);
assert!(
compact_accessibility_tree_with_note(nodes.clone(), false, false)
.1
.is_none()
);
assert!(compact_accessibility_tree_with_note(nodes, true, true)
.1
.is_none());
}
fn node_with_actions(
index: u32,
bounds: Option<Bounds>,
actions: Vec<AccessibilityAction>,
) -> AccessibilityNode {
AccessibilityNode {
index,
parent_index: None,
depth: 0,
object_ref: format!(":1.{index}/org/a11y/atspi/accessible/{index}"),
role: "push button".to_string(),
name: Some(format!("Button {index}")),
description: None,
child_count: 0,
bounds,
states: Vec::new(),
actions,
value: None,
text: None,
supports_editable_text: false,
}
}
fn click_action() -> AccessibilityAction {
AccessibilityAction {
index: 0,
name: "Click".to_string(),
description: "Clicks the element".to_string(),
keybinding: String::new(),
}
}
fn solid_png(width: u32, height: u32) -> Vec<u8> {
let img = image::RgbaImage::from_pixel(width, height, image::Rgba([32, 128, 192, 255]));
let mut out = Vec::new();
image::DynamicImage::ImageRgba8(img)
.write_to(&mut std::io::Cursor::new(&mut out), image::ImageFormat::Png)
.unwrap();
out
}
#[test]
fn targeted_app_state_crops_before_screenshot_payload_resize() {
let raw = RawScreenshotCapture {
mime_type: "image/png".to_string(),
bytes: solid_png(400, 200),
source: "test".to_string(),
width: 400,
height: 200,
};
let capture = prepare_app_state_screenshot(
raw,
Some((50, 20, 200, 100)),
true,
ScreenshotPayloadOptions {
max_width: Some(100),
max_height: Some(100),
max_bytes: Some(1024 * 1024),
..Default::default()
},
)
.unwrap();
assert_eq!(
(capture.coordinate_width, capture.coordinate_height),
(200, 100)
);
assert_eq!((capture.width, capture.height), (100, 50));
}
#[test]
fn unresolved_app_state_target_refuses_full_desktop_screenshot() {
let raw = RawScreenshotCapture {
mime_type: "image/png".to_string(),
bytes: solid_png(400, 200),
source: "test".to_string(),
width: 400,
height: 200,
};
let error =
prepare_app_state_screenshot(raw, None, true, ScreenshotPayloadOptions::default())
.unwrap_err();
assert!(error.to_string().contains("requires a resolved window"));
}
#[test]
fn targeted_app_state_crops_only_visible_part_of_offscreen_window() {
let raw = RawScreenshotCapture {
mime_type: "image/png".to_string(),
bytes: solid_png(400, 200),
source: "test".to_string(),
width: 400,
height: 200,
};
let capture = prepare_app_state_screenshot(
raw,
Some((-50, -40, 100, 100)),
true,
ScreenshotPayloadOptions::default(),
)
.unwrap();
assert_eq!(
(capture.coordinate_width, capture.coordinate_height),
(50, 60)
);
}
fn sample_app_state_output(screenshot: Option<&ScreenshotCapture>) -> GetAppStateOutput {
GetAppStateOutput {
app_name_or_bundle_identifier: None,
window_context: None,
window_error: None,
window_permissions_hint: None,
backend: "linux-atspi".to_string(),
screenshot: screenshot.map(ScreenshotSummary::from),
screenshot_error: None,
accessibility_tree: Vec::new(),
accessibility_tree_raw_count: 0,
tree_scoped: true,
accessibility_tree_truncated: false,
accessibility_error: None,
readiness: crate::diagnostics::ReadinessReport {
can_register_mcp_tools: true,
can_build_accessibility_tree: true,
can_query_windows: true,
can_focus_apps: true,
can_focus_windows: true,
can_send_development_input: true,
can_capture_screenshots: true,
screenshot_capture_status: crate::diagnostics::ScreenshotCaptureStatus::Verified,
recommended_next_step: String::new(),
blockers: Vec::new(),
warnings: Vec::new(),
},
diagnostics: None,
message: "ok".to_string(),
}
}
#[test]
fn app_state_result_emits_screenshot_as_image_block_not_inline_base64() {
// Issue #145: the base64 payload must ride a structured image block so
// hosts count it as vision tokens instead of a 100KB text blob.
let raw = RawScreenshotCapture {
mime_type: "image/png".to_string(),
bytes: solid_png(64, 32),
source: "test".to_string(),
width: 64,
height: 32,
};
let capture =
prepare_app_state_screenshot(raw, None, false, ScreenshotPayloadOptions::default())
.unwrap();
let payload = data_url_payload(&capture.data_url);
let output = sample_app_state_output(Some(&capture));
let result = app_state_result(output, Some(capture)).unwrap();
assert_eq!(result.content.len(), 2);
let image = result.content[0]
.as_image()
.expect("first block is an image");
assert_eq!(image.mime_type, "image/png");
assert_eq!(image.data, payload);
let text = &result.content[1]
.as_text()
.expect("second block is text")
.text;
assert!(
!text.contains(&payload),
"base64 leaked into the text block"
);
assert!(!text.contains("data_url"));
let structured = result.structured_content.expect("structured content");
assert_eq!(structured["screenshot"]["width"], 64);
assert_eq!(structured["screenshot"]["mime_type"], "image/png");
assert!(structured["screenshot"].get("data_url").is_none());
}
#[test]
fn app_state_result_without_screenshot_is_text_only() {
let result = app_state_result(sample_app_state_output(None), None).unwrap();
assert_eq!(result.content.len(), 1);
assert!(result.content[0].as_text().is_some());
assert_eq!(
result.structured_content.unwrap()["screenshot"],
serde_json::Value::Null
);
}
#[test]
fn gnome_window_crop_scales_logical_bounds_to_capture_pixels() {
let bounds = WindowBounds {
x: Some(6),
y: Some(36),
width: 1357,
height: 1144,
};
let monitors = [(0, 0, 1920, 1200)];
assert_eq!(
logical_window_crop_rect(&bounds, &monitors, 2560, 1600).unwrap(),
(8, 48, 1810, 1526)
);
}
#[test]
fn portal_points_map_capture_pixels_back_to_logical_window_space() {
let scaled = WindowCoordinateMap {
capture_rect: (8, 48, 1810, 1526),
full_capture_rect: (8, 48, 1810, 1526),
portal_rect: Some((6, 36, 1357, 1144)),
};
assert_eq!(scaled.portal_point(913, 811), Some((684, 608)));
let clipped = WindowCoordinateMap {
capture_rect: (0, 0, 50, 60),
full_capture_rect: (-50, -40, 100, 100),
portal_rect: Some((-50, -40, 100, 100)),
};
assert_eq!(clipped.portal_point(0, 0), Some((0, 0)));
}
#[test]
fn scaled_kwin_bounds_keep_capture_and_portal_spaces_distinct() {
let bounds = WindowBounds {
x: Some(1000),
y: Some(100),
width: 800,
height: 600,
};
let logical_rect = window_crop_rect(&bounds).unwrap();
let full_capture_rect =
logical_window_crop_rect(&bounds, &[(0, 0, 1920, 1080)], 3840, 2160).unwrap();
let mapping = WindowCoordinateMap {
capture_rect: full_capture_rect,
full_capture_rect,
portal_rect: Some(logical_rect),
};
assert_eq!(mapping.capture_rect, (2000, 200, 1600, 1200));
assert_eq!(mapping.portal_point(2400, 500), Some((1200, 250)));
}
#[test]
fn kwin_mapping_uses_the_workspace_geometry_origin() {
let bounds = WindowBounds {
x: Some(1100),
y: Some(50),
width: 800,
height: 600,
};
let logical_rect = window_crop_rect(&bounds).unwrap();
let full_capture_rect =
logical_window_crop_rect(&bounds, &[(100, -50, 1920, 1080)], 3840, 2160).unwrap();
let mapping = WindowCoordinateMap {
capture_rect: full_capture_rect,
full_capture_rect,
portal_rect: Some(logical_rect),
};
assert_eq!(mapping.capture_rect, (2000, 200, 1600, 1200));
assert_eq!(mapping.portal_point(2400, 500), Some((1300, 200)));
}
#[test]
fn gnome_window_crop_accounts_for_negative_monitor_origins() {
let bounds = WindowBounds {
x: Some(-900),
y: Some(100),
width: 400,
height: 300,
};
let monitors = [(-1000, 0, 1000, 800), (0, 0, 1200, 800)];
assert_eq!(
logical_window_crop_rect(&bounds, &monitors, 2200, 800).unwrap(),
(100, 100, 400, 300)
);
}
#[test]
fn readonly_targeted_screenshot_requires_focused_visible_window() {
let mut window = window_info(1, Some("Target"), None, None, None);
assert!(ensure_readonly_screenshot_target_is_visible(&window).is_err());
window.focused = true;
assert!(ensure_readonly_screenshot_target_is_visible(&window).is_ok());
window.hidden = true;
assert!(ensure_readonly_screenshot_target_is_visible(&window).is_err());
}
#[test]
fn wayland_display_is_enough_to_select_portal_fallback() {
assert!(session_is_wayland(None, Some("wayland-1")));
assert!(session_is_wayland(Some(" "), Some("wayland-1")));
assert!(session_is_wayland(Some("wayland"), None));
assert!(!session_is_wayland(Some("x11"), None));
assert!(!session_is_wayland(None, Some(" ")));
}
#[test]
fn incompatible_ydotool_socket_does_not_suppress_portal_fallback() {
let incompatible_ydotool = ydotool_backend_available_from(true, false);
let compatible_ydotool = ydotool_backend_available_from(true, true);
assert!(should_prefer_portal_backend_by_default(
true,
incompatible_ydotool
));
assert!(!should_prefer_portal_backend_by_default(
true,
compatible_ydotool
));
assert!(!should_prefer_portal_backend_by_default(
false,
incompatible_ydotool
));
}
#[test]
fn xdotool_keyboard_override_policy_matches_documented_precedence() {
assert!(prefer_xdotool_keyboard(false, true, true, true, true));
assert!(!prefer_xdotool_keyboard(true, true, true, true, true));
assert!(!prefer_xdotool_keyboard(false, true, true, false, true));
assert!(!prefer_xdotool_keyboard(false, false, true, true, true));
assert!(prefer_xdotool_keyboard(false, false, false, true, true));
}
#[test]
fn wayland_prefers_wtype_unless_ydotool_is_forced() {
assert!(prefer_wtype_keyboard(false, true, true, true));
assert!(!prefer_wtype_keyboard(true, true, true, true));
assert!(!prefer_wtype_keyboard(false, false, true, true));
assert!(!prefer_wtype_keyboard(false, true, false, true));
assert!(!prefer_wtype_keyboard(false, true, true, false));
}
#[test]
fn literal_text_portal_policy_preserves_layout_aware_routes_and_overrides() {
// GNOME has no wtype route. ydotool availability must not suppress
// literal portal text, unlike pointer and physical chord policies.
assert!(prefer_portal_text_backend(
false, false, true, false, false, false
));
// A compatible compositor with wtype needs no portal prompt.
assert!(!prefer_portal_text_backend(
false, false, true, false, false, true
));
assert!(prefer_portal_text_backend(
false, false, true, false, true, true
));
// Explicit raw/XTEST overrides, KDE clipboard, and X11 keep precedence.
assert!(!prefer_portal_text_backend(
true, false, true, false, true, false
));
assert!(!prefer_portal_text_backend(
false, true, true, false, true, false
));
assert!(!prefer_portal_text_backend(
false, false, true, true, true, false
));
assert!(!prefer_portal_text_backend(
false, false, false, false, true, false
));
}
#[tokio::test]
async fn raw_text_rejects_unsupported_characters_before_backend_access() {
assert!(validate_ydotool_text("probe-0710_a/b\t\nXYZ").is_ok());
for text in [
"prefixä",
"prefix中",
"prefix\r",
"prefix\0",
"prefix\u{7f}",
] {
// This exits before CLI probing, socket discovery or input, even
// though a supported prefix appears before the invalid character.
let error = run_ydotool_type_text(text).await.unwrap_err();
assert!(error.contains("no text was sent"), "{error}");
}
}
#[test]
fn raw_text_success_explains_layout_dependence() {
let result = action_result("type_text", Ok(Vec::new()), None);
assert!(result.ok);
assert!(result.message.contains("US physical key positions"));
assert!(result.message.contains("verify the field contents"));
}
#[test]
fn window_crop_happens_before_screenshot_payload_resize() {
let (cropped, width, height) = crop_png(&solid_png(400, 200), 50, 20, 200, 100).unwrap();
let capture = prepare_screenshot_payload(
RawScreenshotCapture {
mime_type: "image/png".to_string(),
bytes: cropped,
source: "test".to_string(),
width,
height,
},
ScreenshotPayloadOptions {
max_width: Some(100),
max_height: Some(100),
max_bytes: Some(1024 * 1024),
..Default::default()
},
)
.unwrap();
assert_eq!(
(capture.coordinate_width, capture.coordinate_height),
(200, 100)
);
assert_eq!((capture.width, capture.height), (100, 50));
assert!(capture.resized);
}
fn window_info(
window_id: u64,
title: Option<&str>,
app_id: Option<&str>,
wm_class: Option<&str>,
pid: Option<u32>,
) -> WindowInfo {
WindowInfo {
window_id,
title: title.map(str::to_string),
app_id: app_id.map(str::to_string),
wm_class: wm_class.map(str::to_string),
pid,
bounds: Some(WindowBounds {
x: Some(10),
y: Some(20),
width: 800,
height: 600,
}),
workspace: Some(0),
focused: false,
hidden: false,
client_type: Some("wayland".to_string()),
backend: GNOME_SHELL_EXTENSION_BACKEND.to_string(),
terminal: None,
}
}
#[test]
fn relative_click_coordinates_use_capture_space_rect() {
let mut params = ClickParams {
x: Some(7),
y: Some(9),
relative: Some(true),
..Default::default()
};
apply_window_relative_click_coordinates(&mut params, (133, 267, 1067, 800)).unwrap();
assert_eq!((params.x, params.y), (Some(140), Some(276)));
}
#[test]
fn relative_click_origin_is_screenshot_crop_not_raw_surface_buffer() {
// GNOME Wayland CSD probe: frame [760,406,400,300], buffer
// [746,394,428,329]. The screenshot's green target is at [850,543].
// Crop-local [90,137] and desktop [850,543] must denote the same point;
// substituting the buffer/shadow origin would miss by [14,12].
let mut params = ClickParams {
x: Some(90),
y: Some(137),
relative: Some(true),
..Default::default()
};
apply_window_relative_click_coordinates(&mut params, (760, 406, 400, 300)).unwrap();
assert_eq!((params.x, params.y), (Some(850), Some(543)));
}
#[test]
fn relative_click_coordinates_require_xy() {
let mut params = ClickParams {
x: Some(7),
relative: Some(true),
..Default::default()
};
let error =
apply_window_relative_click_coordinates(&mut params, (100, 200, 800, 600)).unwrap_err();
assert!(error.contains("both x and y"));
assert_eq!((params.x, params.y), (Some(7), None));
}
#[test]
fn relative_click_coordinates_must_stay_inside_bounds() {
for (x, y) in [(-1, 9), (7, -1), (800, 9), (7, 600)] {
let mut params = ClickParams {
x: Some(x),
y: Some(y),
relative: Some(true),
..Default::default()
};
let error = apply_window_relative_click_coordinates(&mut params, (100, 200, 800, 600))
.unwrap_err();
assert!(error.contains("inside target-window bounds"));
assert_eq!((params.x, params.y), (Some(x), Some(y)));
}
}
#[test]
fn absolute_pointer_note_reports_the_emitted_coordinate() {
assert_eq!(
abs_pointer_clamp_note(crate::abs_pointer::PointerLanding {
requested: (1920, 1080),
emitted: (1919, 1079),
}),
Some(
"Requested coordinate 1920,1080 was clamped to 1919,1079 by the uinput absolute pointer."
.to_string()
)
);
assert_eq!(
abs_pointer_clamp_note(crate::abs_pointer::PointerLanding {
requested: (640, 480),
emitted: (640, 480),
}),
None
);
}
#[test]
fn accessibility_filter_candidates_prefer_title_and_skip_synthetic_app_id() {
let window = window_info(
42,
Some("CU ATSPI GTK Test"),
Some("window:46"),
Some("cu_atspi_gtk_test.py"),
Some(2914326),
);
let candidates = accessibility_filter_candidates(Some(&window));
assert_eq!(
candidates,
vec![
"CU ATSPI GTK Test".to_string(),
"cu_atspi_gtk_test.py".to_string(),
]
);
}
#[test]
fn select_accessibility_object_ref_prefers_exact_pid_match() {
let apps = vec![
AccessibleAppSummary {
object_ref: ":1.31/org/a11y/atspi/accessible/root".to_string(),
name: Some("electron".to_string()),
pid: Some(2774076),
role: "application".to_string(),
child_count: 1,
bounds: None,
},
AccessibleAppSummary {
object_ref: ":1.64/org/a11y/atspi/accessible/root".to_string(),
name: Some("cu_atspi_gtk_test.py".to_string()),
pid: Some(2914326),
role: "application".to_string(),
child_count: 1,
bounds: None,
},
];
let object_ref = select_accessibility_object_ref(
&apps,
2914326,
&[
"CU ATSPI GTK Test".to_string(),
"cu_atspi_gtk_test.py".to_string(),
],
)
.unwrap();
assert_eq!(object_ref, ":1.64/org/a11y/atspi/accessible/root");
}
#[test]
fn compact_accessibility_tree_reparents_actionable_descendants() {
let nodes = vec![
AccessibilityNode {
index: 0,
parent_index: None,
depth: 0,
object_ref: ":1.0/root".to_string(),
role: "application".to_string(),
name: Some("demo-app".to_string()),
description: None,
child_count: 1,
bounds: None,
states: Vec::new(),
actions: Vec::new(),
value: None,
text: None,
supports_editable_text: false,
},
AccessibilityNode {
index: 1,
parent_index: Some(0),
depth: 1,
object_ref: ":1.1/frame".to_string(),
role: "frame".to_string(),
name: Some("Demo Frame".to_string()),
description: None,
child_count: 1,
bounds: None,
states: Vec::new(),
actions: Vec::new(),
value: None,
text: None,
supports_editable_text: false,
},
AccessibilityNode {
index: 2,
parent_index: Some(1),
depth: 2,
object_ref: ":1.2/filler".to_string(),
role: "filler".to_string(),
name: None,
description: None,
child_count: 1,
bounds: None,
states: Vec::new(),
actions: Vec::new(),
value: None,
text: None,
supports_editable_text: false,
},
AccessibilityNode {
index: 3,
parent_index: Some(2),
depth: 3,
object_ref: ":1.3/button".to_string(),
role: "button".to_string(),
name: Some("Run".to_string()),
description: None,
child_count: 0,
bounds: Some(Bounds {
x: 10,
y: 20,
width: 100,
height: 40,
}),
states: Vec::new(),
actions: vec![AccessibilityAction {
index: 0,
name: "Click".to_string(),
description: "Clicks the button".to_string(),
keybinding: String::new(),
}],
value: None,
text: None,
supports_editable_text: false,
},
];
let compacted = compact_accessibility_tree(nodes);
assert_eq!(compacted.len(), 3);
assert_eq!(compacted[0].role, "application");
assert_eq!(compacted[1].role, "frame");
assert_eq!(compacted[2].role, "button");
assert_eq!(compacted[2].parent_index, Some(1));
assert_eq!(compacted[1].child_count, 1);
}
#[test]
fn compact_accessibility_tree_drops_structural_noise() {
let nodes = vec![
AccessibilityNode {
index: 0,
parent_index: None,
depth: 0,
object_ref: ":1.0/root".to_string(),
role: "application".to_string(),
name: Some("demo-app".to_string()),
description: None,
child_count: 2,
bounds: None,
states: Vec::new(),
actions: Vec::new(),
value: None,
text: None,
supports_editable_text: false,
},
AccessibilityNode {
index: 1,
parent_index: Some(0),
depth: 1,
object_ref: ":1.1/frame".to_string(),
role: "frame".to_string(),
name: Some("Demo Frame".to_string()),
description: None,
child_count: 2,
bounds: None,
states: Vec::new(),
actions: Vec::new(),
value: None,
text: None,
supports_editable_text: false,
},
AccessibilityNode {
index: 2,
parent_index: Some(1),
depth: 2,
object_ref: ":1.2/tab".to_string(),
role: "page tab".to_string(),
name: Some("Hidden".to_string()),
description: None,
child_count: 0,
bounds: None,
states: Vec::new(),
actions: Vec::new(),
value: None,
text: None,
supports_editable_text: false,
},
AccessibilityNode {
index: 3,
parent_index: Some(1),
depth: 2,
object_ref: ":1.3/separator".to_string(),
role: "separator".to_string(),
name: None,
description: None,
child_count: 0,
bounds: None,
states: Vec::new(),
actions: Vec::new(),
value: None,
text: None,
supports_editable_text: false,
},
];
let compacted = compact_accessibility_tree(nodes);
assert_eq!(compacted.len(), 3);
assert_eq!(compacted[2].role, "page tab");
assert_eq!(compacted[2].name.as_deref(), Some("Hidden"));
}
#[test]
fn kde_clipboard_restore_delay_uses_minimum_for_short_text() {
assert_eq!(
kde_clipboard_restore_delay("short"),
Duration::from_millis(KDE_CLIPBOARD_RESTORE_MIN_DELAY_MS)
);
}
#[test]
fn kde_clipboard_restore_delay_scales_and_caps_long_text() {
let scaled_text = "x".repeat(1_000);
assert_eq!(
kde_clipboard_restore_delay(&scaled_text),
Duration::from_millis(4_000)
);
let capped_text = "x".repeat(10_000);
assert_eq!(
kde_clipboard_restore_delay(&capped_text),
Duration::from_millis(KDE_CLIPBOARD_RESTORE_MAX_DELAY_MS)
);
}
#[test]
fn kde_clipboard_maps_each_paste_shortcut_to_its_chord() {
assert_eq!(
kde_clipboard_paste_chord(KdeClipboardPasteShortcut::Standard),
(&[EVDEV_KEY_LEFTCTRL][..], EVDEV_KEY_V)
);
assert_eq!(
kde_clipboard_paste_chord(KdeClipboardPasteShortcut::CtrlShiftV),
(&[EVDEV_KEY_LEFTCTRL, EVDEV_KEY_LEFTSHIFT][..], EVDEV_KEY_V)
);
assert_eq!(
kde_clipboard_paste_chord(KdeClipboardPasteShortcut::ShiftInsert),
(&[EVDEV_KEY_LEFTSHIFT][..], EVDEV_KEY_INSERT)
);
}
fn focused(role: &str, is_terminal: bool) -> FocusedElementSummary {
FocusedElementSummary {
role: role.to_string(),
name: None,
editable: false,
states: vec!["focused".to_string()],
is_terminal,
}
}
#[test]
fn kde_clipboard_focus_fallback_uses_the_role_enum_not_the_name() {
let ctrl_shift_v = Some(TerminalPasteShortcut::CtrlShiftV);
let shift_insert = Some(TerminalPasteShortcut::ShiftInsert);
// No AT-SPI answer (xterm exposes none) keeps the terminal chord.
assert_eq!(
kde_clipboard_shortcut_for_focus(shift_insert, None),
KdeClipboardPasteShortcut::ShiftInsert
);
// A terminal widget keeps it, whatever its localized role name.
assert_eq!(
kde_clipboard_shortcut_for_focus(ctrl_shift_v, Some(&focused("терминал", true))),
KdeClipboardPasteShortcut::CtrlShiftV
);
// A focused search field inside the terminal window pastes with Ctrl+V.
assert_eq!(
kde_clipboard_shortcut_for_focus(ctrl_shift_v, Some(&focused("text", false))),
KdeClipboardPasteShortcut::Standard
);
assert_eq!(
kde_clipboard_shortcut_for_focus(None, None),
KdeClipboardPasteShortcut::Standard
);
}
#[test]
fn kde_clipboard_routes_xterm_family_to_shift_insert() {
for wm_class in ["xterm", "UXTerm", "urxvt", "rxvt-unicode", "koi8rxterm"] {
let window = window_info(1, Some("user@host"), None, Some(wm_class), Some(100));
assert_eq!(
kde_clipboard_terminal_shortcut(&WindowTarget::default(), Some(&window)),
Some(TerminalPasteShortcut::ShiftInsert),
"{wm_class} should paste with Shift+Insert"
);
}
}
#[test]
fn kde_clipboard_routes_explicit_tty_targets_to_terminal_paste() {
let target = WindowTarget {
tty: Some("/dev/pts/11".to_string()),
..Default::default()
};
assert_eq!(
kde_clipboard_terminal_shortcut(&target, None),
Some(TerminalPasteShortcut::CtrlShiftV)
);
}
#[test]
fn kde_clipboard_prefers_resolved_window_over_stale_terminal_selector() {
let target = WindowTarget {
tty: Some("/dev/pts/11".to_string()),
..Default::default()
};
let window = window_info(
1,
Some("Browser"),
Some("firefox"),
Some("firefox"),
Some(100),
);
assert_eq!(
kde_clipboard_terminal_shortcut(&target, Some(&window)),
None
);
}
#[tokio::test]
async fn kde_clipboard_async_routing_prefers_focus_over_stale_terminal_selector() {
let target = WindowTarget {
tty: Some("/dev/pts/11".to_string()),
..Default::default()
};
let window = window_info(
1,
Some("Browser"),
Some("firefox"),
Some("firefox"),
Some(100),
);
let focus = WindowFocusResult {
requested_window: window.clone(),
focused_window: Some(window),
exact_window_focused: true,
app_focused: true,
backend: KWIN_BACKEND.to_string(),
note: "test".to_string(),
};
assert_eq!(
kde_clipboard_paste_shortcut(&target, Some(&focus)).await,
KdeClipboardPasteShortcut::Standard
);
}
#[test]
fn kde_clipboard_routes_konsole_identity_to_terminal_paste() {
let window = window_info(
1,
Some("unflappable-donkey"),
Some("org.kde.konsole"),
Some("konsole"),
Some(100),
);
assert_eq!(
kde_clipboard_terminal_shortcut(&WindowTarget::default(), Some(&window)),
Some(TerminalPasteShortcut::CtrlShiftV)
);
}
#[test]
fn kde_clipboard_does_not_classify_terminal_words_in_browser_titles() {
let window = window_info(
1,
Some("xterm documentation"),
Some("firefox"),
Some("firefox"),
Some(100),
);
assert_eq!(
kde_clipboard_terminal_shortcut(&WindowTarget::default(), Some(&window)),
None
);
}
#[test]
fn kde_clipboard_pty_metadata_does_not_change_editor_paste() {
let mut window = window_info(
1,
Some("xterm integration test"),
Some("example-ide"),
Some("ExampleIde"),
Some(100),
);
window.terminal = Some(crate::terminal::TerminalWindowContext {
tty: "/dev/pts/11".to_string(),
root_process: crate::terminal::TerminalProcess {
pid: 200,
command_name: "bash".to_string(),
command_line: "bash".to_string(),
cwd: None,
},
active_process: None,
process_count: 1,
confidence: "high".to_string(),
match_reason: "one child PTY".to_string(),
});
let terminal = kde_clipboard_terminal_shortcut(&WindowTarget::default(), Some(&window));
assert_eq!(terminal, None);
assert_eq!(
kde_clipboard_shortcut_for_focus(terminal, None),
KdeClipboardPasteShortcut::Standard
);
}
#[tokio::test]
async fn kde_clipboard_dbus_operation_times_out_when_pending() {
let error = kde_clipboard_dbus_operation_with_timeout(
"proxy creation",
std::future::pending::<zbus::Result<()>>(),
Duration::from_millis(1),
)
.await
.unwrap_err();
assert_eq!(error, "KDE clipboard proxy creation timed out");
}
#[test]
fn cached_element_index_resolves_to_bounds_center() {
let backend = ComputerUseLinux::default();
backend.cache_nodes(&[node(
7,
Some(Bounds {
x: 10,
y: 20,
width: 100,
height: 40,
}),
)]);
let point = backend
.resolve_optional_target_point(None, None, Some(7))
.unwrap()
.unwrap();
assert_eq!(point, (60, 40));
}
#[test]
fn coordinate_target_overrides_cached_element_index() {
let backend = ComputerUseLinux::default();
backend.cache_nodes(&[node(
7,
Some(Bounds {
x: 10,
y: 20,
width: 100,
height: 40,
}),
)]);
let point = backend
.resolve_optional_target_point(Some(200), Some(300), Some(7))
.unwrap()
.unwrap();
assert_eq!(point, (200, 300));
}
#[test]
fn cached_element_index_requires_positive_bounds() {
let backend = ComputerUseLinux::default();
backend.cache_nodes(&[node(
7,
Some(Bounds {
x: 10,
y: 20,
width: 0,
height: 40,
}),
)]);
let error = backend
.resolve_optional_target_point(None, None, Some(7))
.unwrap_err();
assert!(error.contains("No clickable bounds cached for element_index 7"));
}
#[test]
fn cached_element_index_ignores_sentinel_bounds() {
let backend = ComputerUseLinux::default();
backend.cache_nodes(&[node(
7,
Some(Bounds {
x: i32::MIN,
y: i32::MIN,
width: 1,
height: 1,
}),
)]);
let error = backend
.resolve_optional_target_point(None, None, Some(7))
.unwrap_err();
assert!(error.contains("No clickable bounds cached for element_index 7"));
}
#[test]
fn empty_node_cache_clears_stale_element_index() {
let backend = ComputerUseLinux::default();
backend.cache_nodes(&[node(
7,
Some(Bounds {
x: 10,
y: 20,
width: 100,
height: 40,
}),
)]);
backend.cache_nodes(&[]);
let error = backend
.resolve_optional_target_point(None, None, Some(7))
.unwrap_err();
assert!(error.contains("No clickable bounds cached for element_index 7"));
}
#[test]
fn click_target_falls_back_to_primary_action_without_bounds() {
let backend = ComputerUseLinux::default();
backend.cache_nodes(&[node_with_actions(
7,
None,
vec![AccessibilityAction {
index: 0,
name: "Click".to_string(),
description: "Clicks the button".to_string(),
keybinding: String::new(),
}],
)]);
let target = backend
.resolve_click_target(&ClickParams {
element_index: Some(7),
..Default::default()
})
.unwrap();
match target {
ClickTarget::PrimaryAction {
object_ref,
action_name,
action_index,
} => {
assert_eq!(object_ref, ":1.7/org/a11y/atspi/accessible/7");
assert_eq!(action_name.as_deref(), Some("Click"));
assert_eq!(action_index, 0);
}
ClickTarget::Coordinates(_, _) => {
panic!("expected AT-SPI primary-action fallback")
}
}
}
#[test]
fn element_activation_ignores_hidpi_and_zero_origin_bounds() {
for (x, y) in [(75, 144), (0, 0)] {
let backend = ComputerUseLinux::default();
backend.cache_nodes(&[node_with_actions(
7,
Some(Bounds {
x,
y,
width: 320,
height: 120,
}),
vec![AccessibilityAction {
index: 0,
name: "click".into(),
description: String::new(),
keybinding: String::new(),
}],
)]);
assert!(matches!(
backend
.resolve_click_target(&ClickParams {
element_index: Some(7),
..Default::default()
})
.unwrap(),
ClickTarget::PrimaryAction {
action_index: 0,
..
}
));
// Explicit coordinates and gestures retain pointer semantics.
for params in [
ClickParams {
x: Some(470),
y: Some(408),
element_index: Some(7),
..Default::default()
},
ClickParams {
button: Some("right".into()),
element_index: Some(7),
..Default::default()
},
ClickParams {
click_count: Some(2),
element_index: Some(7),
..Default::default()
},
] {
assert!(matches!(
backend.resolve_click_target(¶ms).unwrap(),
ClickTarget::Coordinates(_, _)
));
}
}
}
#[test]
fn element_click_does_not_replace_pointer_with_non_activation_action() {
let backend = ComputerUseLinux::default();
backend.cache_nodes(&[node_with_actions(
7,
Some(Bounds {
x: 10,
y: 20,
width: 20,
height: 20,
}),
vec![AccessibilityAction {
index: 0,
name: "show-menu".into(),
description: String::new(),
keybinding: String::new(),
}],
)]);
assert!(matches!(
backend
.resolve_click_target(&ClickParams {
element_index: Some(7),
..Default::default()
})
.unwrap(),
ClickTarget::Coordinates(20, 30)
));
}
#[test]
fn entry_submit_and_slider_jump_are_not_pointer_clicks() {
for (role, action_name) in [
("entry", "activate"),
("spin button", "activate"),
("slider", "jump"),
] {
let backend = ComputerUseLinux::default();
let mut target = node_with_actions(
7,
Some(Bounds {
x: 10,
y: 20,
width: 100,
height: 40,
}),
vec![AccessibilityAction {
index: 0,
name: action_name.into(),
description: String::new(),
keybinding: String::new(),
}],
);
target.role = role.into();
backend.cache_nodes(&[target]);
assert!(matches!(
backend
.resolve_click_target(&ClickParams {
element_index: Some(7),
..Default::default()
})
.unwrap(),
ClickTarget::Coordinates(60, 40)
));
let mut node = backend.last_snapshot.lock().unwrap().nodes[0].clone();
for bounds in [
None,
Some(Bounds {
x: i32::MIN,
y: i32::MIN,
width: 1,
height: 1,
}),
Some(Bounds {
x: 0,
y: 0,
width: 0,
height: 0,
}),
] {
node.bounds = bounds;
backend.cache_nodes(&[node.clone()]);
assert!(backend
.resolve_click_target(&ClickParams {
element_index: Some(7),
..Default::default()
})
.is_err());
}
}
}
#[test]
fn click_target_falls_back_to_primary_action_with_sentinel_bounds() {
let backend = ComputerUseLinux::default();
backend.cache_nodes(&[node_with_actions(
7,
Some(Bounds {
x: i32::MIN,
y: i32::MIN,
width: 1,
height: 1,
}),
vec![AccessibilityAction {
index: 0,
name: "Click".to_string(),
description: "Clicks the button".to_string(),
keybinding: String::new(),
}],
)]);
let target = backend
.resolve_click_target(&ClickParams {
element_index: Some(7),
..Default::default()
})
.unwrap();
match target {
ClickTarget::PrimaryAction {
object_ref,
action_name,
action_index,
} => {
assert_eq!(object_ref, ":1.7/org/a11y/atspi/accessible/7");
assert_eq!(action_name.as_deref(), Some("Click"));
assert_eq!(action_index, 0);
}
ClickTarget::Coordinates(_, _) => {
panic!("expected AT-SPI primary-action fallback")
}
}
}
#[test]
fn click_target_requires_bounds_for_non_plain_clicks() {
let backend = ComputerUseLinux::default();
backend.cache_nodes(&[node_with_actions(
7,
None,
vec![AccessibilityAction {
index: 0,
name: "Click".to_string(),
description: "Clicks the button".to_string(),
keybinding: String::new(),
}],
)]);
let error = backend
.resolve_click_target(&ClickParams {
element_index: Some(7),
button: Some("right".to_string()),
..Default::default()
})
.unwrap_err();
assert!(error.contains("No clickable bounds cached for element_index 7"));
}
#[test]
fn absolute_mousemove_uses_coordinate_separator() {
assert_eq!(
absolute_mousemove_args(200, 300),
vec![
"mousemove".to_string(),
"--absolute".to_string(),
"--".to_string(),
"200".to_string(),
"300".to_string(),
]
);
}
#[test]
fn native_x11_pointer_policy_requires_explicit_x11_without_wayland_display() {
assert!(native_x11_xdotool_pointer_session(Some("x11"), None));
assert!(!native_x11_xdotool_pointer_session(
Some("wayland"),
Some("wayland-0")
));
assert!(!native_x11_xdotool_pointer_session(
Some("x11"),
Some("wayland-0")
));
}
#[test]
fn xdotool_pointer_command_is_single_no_sync_move_and_click() {
assert_eq!(
xdotool_pointer_click_args(1550, 930, 3, Some("right")),
Some(vec![
"mousemove".to_string(),
"--".to_string(),
"1550".to_string(),
"930".to_string(),
"click".to_string(),
"--repeat".to_string(),
"3".to_string(),
"3".to_string(),
])
);
}
#[test]
fn xdotool_pointer_policy_requires_all_pure_gating_conditions() {
let eligible = (false, Some("x11"), true, None, true);
assert!(prefer_xdotool_pointer(
eligible.0, eligible.1, eligible.2, eligible.3, eligible.4
));
assert!(!prefer_xdotool_pointer(true, Some("x11"), true, None, true));
assert!(!prefer_xdotool_pointer(
false,
Some("wayland"),
true,
None,
true
));
assert!(!prefer_xdotool_pointer(
false,
None,
true,
Some("wayland-0"),
true
));
assert!(!prefer_xdotool_pointer(
false,
Some("x11"),
false,
None,
true
));
assert!(!prefer_xdotool_pointer(
false,
Some("x11"),
true,
None,
false
));
}
#[test]
fn xdotool_pointer_supports_only_standard_buttons() {
assert!(xdotool_pointer_click_args(10, 20, 1, None).is_some());
assert!(xdotool_pointer_click_args(10, 20, 1, Some("middle")).is_some());
assert!(xdotool_pointer_click_args(10, 20, 1, Some("right")).is_some());
}
#[test]
fn extended_pointer_buttons_do_not_construct_xdotool_commands() {
for button in ["side", "extra", "forward", "back"] {
assert_eq!(xdotool_pointer_click_args(10, 20, 1, Some(button)), None);
}
}
#[tokio::test]
async fn pointer_xdotool_spawn_failure_uses_ydotool_fallback() {
let result = run_xdotool_pointer_or_fallback(
Path::new("/definitely/missing/xdotool"),
&[],
|| async { Ok::<_, String>(Vec::new()) },
)
.await
.expect("spawn failure should use fallback");
assert_eq!(result.backend, KeyboardCommandBackend::Ydotool);
}
#[tokio::test]
async fn pointer_xdotool_nonzero_exit_does_not_use_ydotool_fallback() {
let result = run_xdotool_pointer_or_fallback(
Path::new("/bin/sh"),
&["-c".to_string(), "exit 9".to_string()],
|| async { Err::<Vec<Output>, _>("fallback called".to_string()) },
)
.await;
let error = result.expect_err("launched nonzero xdotool must be terminal");
assert!(!error.contains("fallback called"));
}
#[test]
fn xdotool_scroll_uses_wheel_buttons_instead_of_a_valuator_event() {
assert_eq!(
xdotool_pointer_scroll_args(Some((1550, 930)), 5, ScrollDirection::Up),
vec![
"mousemove".to_string(),
"--".to_string(),
"1550".to_string(),
"930".to_string(),
"click".to_string(),
"--delay".to_string(),
"12".to_string(),
"--repeat".to_string(),
"5".to_string(),
"4".to_string(),
]
);
assert_eq!(
xdotool_pointer_scroll_args(Some((-20, 15)), 1, ScrollDirection::Down),
vec![
"mousemove".to_string(),
"--".to_string(),
"-20".to_string(),
"15".to_string(),
"click".to_string(),
"--delay".to_string(),
"12".to_string(),
"--repeat".to_string(),
"1".to_string(),
"5".to_string(),
]
);
assert_eq!(
xdotool_pointer_scroll_args(None, 3, ScrollDirection::Left),
vec![
"click".to_string(),
"--delay".to_string(),
"12".to_string(),
"--repeat".to_string(),
"3".to_string(),
"6".to_string(),
]
);
assert_eq!(
xdotool_pointer_scroll_args(None, 0, ScrollDirection::Right),
vec![
"click".to_string(),
"--delay".to_string(),
"12".to_string(),
"--repeat".to_string(),
"1".to_string(),
"7".to_string(),
]
);
}
#[test]
fn scroll_pages_are_bounded_before_unit_conversion() {
assert_eq!(scroll_units(None).unwrap(), 5);
assert_eq!(scroll_units(Some(1.0)).unwrap(), 5);
assert_eq!(scroll_units(Some(-1.0)).unwrap(), 5);
assert_eq!(scroll_units(Some(0.0)).unwrap(), 1);
assert_eq!(scroll_units(Some(0.1)).unwrap(), 1);
assert_eq!(scroll_units(Some(100.0)).unwrap(), 500);
assert_eq!(scroll_units(Some(-100.0)).unwrap(), 500);
for pages in [
100.000_001,
-100.000_001,
f64::NAN,
f64::INFINITY,
f64::NEG_INFINITY,
] {
assert!(scroll_units(Some(pages)).is_err(), "accepted {pages}");
}
}
#[test]
fn maximum_scroll_timeout_is_bounded() {
let repeat = u32::try_from(scroll_units(Some(MAX_SCROLL_PAGES)).unwrap()).unwrap();
assert_eq!(repeat, 500);
assert_eq!(
xdotool_scroll_timeout(repeat, XDOTOOL_SCROLL_CLICK_DELAY_MS),
Duration::from_secs(16)
);
}
#[test]
fn xdotool_scroll_timeout_covers_the_gap_after_every_click() {
assert_eq!(
xdotool_scroll_timeout(1, 12),
INPUT_COMMAND_TIMEOUT + Duration::from_millis(12)
);
assert_eq!(
xdotool_scroll_timeout(5, 100),
INPUT_COMMAND_TIMEOUT + Duration::from_millis(500)
);
assert_eq!(
xdotool_scroll_timeout(0, 12),
INPUT_COMMAND_TIMEOUT + Duration::from_millis(12)
);
}
#[test]
fn wheel_mousemove_uses_coordinate_separator_for_negative_values() {
assert_eq!(
wheel_mousemove_args(0, -3),
vec![
"mousemove".to_string(),
"--wheel".to_string(),
"--".to_string(),
"0".to_string(),
"-3".to_string(),
]
);
}
#[test]
fn pointer_actions_keep_pixel_coordinates_for_ydotool_absolute_moves() {
assert_eq!(
absolute_mousemove_args(1550, 930),
vec![
"mousemove".to_string(),
"--absolute".to_string(),
"--".to_string(),
"1550".to_string(),
"930".to_string(),
]
);
}
#[test]
fn key_chord_splits_modifiers_and_key() {
assert_eq!(key_chord("Ctrl+Shift+P"), Some((vec![29, 42], 25)));
assert_eq!(key_chord("Ctrl+S"), Some((vec![29], 31)));
assert_eq!(key_chord("Enter"), Some((vec![], 28)));
// A bare modifier is a chord with no held modifiers.
assert_eq!(key_chord("Super"), Some((vec![], 125)));
assert_eq!(key_chord("NotAKey"), None);
}
#[test]
fn portal_key_chord_sends_modifiers_and_named_keys_as_keysyms() {
use PortalKey::{Keycode, Keysym};
// Modifiers resolve through the live keymap (#191); letters stay
// physical so non-Latin layout groups still see Ctrl+A.
assert_eq!(
portal_key_chord("Ctrl+A"),
Some((vec![Keysym(xkey::Control_L as i32)], Keycode(30)))
);
assert_eq!(
portal_key_chord("ctrl+shift+t"),
Some((
vec![Keysym(xkey::Control_L as i32), Keysym(xkey::Shift_L as i32)],
Keycode(20)
))
);
assert_eq!(
portal_key_chord("Alt+F4"),
Some((vec![Keysym(xkey::Alt_L as i32)], Keysym(xkey::F4 as i32)))
);
assert_eq!(
portal_key_chord("Meta+Return"),
Some((
vec![Keysym(xkey::Super_L as i32)],
Keysym(xkey::Return as i32)
))
);
assert_eq!(
portal_key_chord("Super"),
Some((vec![], Keysym(xkey::Super_L as i32)))
);
assert_eq!(
portal_key_chord("page-down"),
Some((vec![], Keysym(xkey::Page_Down as i32)))
);
assert_eq!(
portal_key_chord("Ctrl+1"),
Some((vec![Keysym(xkey::Control_L as i32)], Keycode(2)))
);
assert_eq!(portal_key_chord("Ctrl+NotAKey"), None);
assert_eq!(portal_key_chord("Hyper+A"), None);
}
#[test]
fn portal_key_chord_accepts_exactly_the_evdev_grammar() {
for key in [
"enter",
"Escape",
"tab",
"backspace",
"del",
"space",
"home",
"end",
"pageup",
"left",
"right",
"up",
"down",
"f1",
"f12",
"a",
"z",
"0",
"9",
"ctrl",
"alt",
"shift",
"cmd",
"Ctrl+Alt+Delete",
"Shift+Tab",
"ctrl+shift",
"?",
"F13",
"Ctrl+",
"",
] {
assert_eq!(
portal_key_chord(key).is_some(),
key_chord(key).is_some(),
"{key}"
);
}
}
#[test]
fn key_sequence_presses_modifiers_around_key() {
assert_eq!(
key_sequence("Ctrl+Shift+P"),
Some(vec![
"29:1".to_string(),
"42:1".to_string(),
"25:1".to_string(),
"25:0".to_string(),
"42:0".to_string(),
"29:0".to_string(),
])
);
}
#[test]
fn ydotool_modifier_chords_include_an_inter_event_delay() {
let args = ydotool_key_args(key_sequence("Ctrl+T").unwrap(), true);
assert_eq!(args, ["key", "-d", "100", "29:1", "20:1", "20:0", "29:0"]);
}
#[test]
fn key_sequence_presses_bare_modifier() {
assert_eq!(
key_sequence("Super"),
Some(vec!["125:1".to_string(), "125:0".to_string()])
);
}
#[test]
fn xdotool_key_spec_maps_named_keys_to_x11_keysyms() {
assert_eq!(xdotool_key_spec("Return"), Some("Return".to_string()));
assert_eq!(xdotool_key_spec("enter"), Some("Return".to_string()));
assert_eq!(xdotool_key_spec("Escape"), Some("Escape".to_string()));
assert_eq!(xdotool_key_spec("backspace"), Some("BackSpace".to_string()));
assert_eq!(xdotool_key_spec("PageUp"), Some("Page_Up".to_string()));
assert_eq!(xdotool_key_spec("ArrowLeft"), Some("Left".to_string()));
assert_eq!(xdotool_key_spec("f5"), Some("F5".to_string()));
assert_eq!(xdotool_key_spec("space"), Some("space".to_string()));
}
#[test]
fn xdotool_key_spec_maps_chords_with_modifier_prefixes() {
assert_eq!(xdotool_key_spec("ctrl+a"), Some("ctrl+a".to_string()));
assert_eq!(xdotool_key_spec("Ctrl+S"), Some("ctrl+s".to_string()));
assert_eq!(
xdotool_key_spec("Ctrl+Shift+P"),
Some("ctrl+shift+p".to_string())
);
assert_eq!(
xdotool_key_spec("Meta+Return"),
Some("super+Return".to_string())
);
assert_eq!(xdotool_key_spec("Alt+F4"), Some("alt+F4".to_string()));
}
#[test]
fn xdotool_key_spec_maps_bare_modifier_to_single_keysym() {
assert_eq!(xdotool_key_spec("Super"), Some("super".to_string()));
assert_eq!(xdotool_key_spec("ctrl"), Some("ctrl".to_string()));
}
#[test]
fn xdotool_type_keeps_a_per_character_delay_so_xtest_events_stay_ordered() {
// Issue #147: `--delay 0` delivers characters out of order on some X
// servers. The default must stay non-zero.
const { assert!(XDOTOOL_TYPE_DELAY_MS > 0) };
let text = "x".repeat(10_000);
let args = xdotool_type_args_with_delay(&text, XDOTOOL_TYPE_DELAY_MS);
assert_eq!(
&args[..5],
["type", "--clearmodifiers", "--delay", "12", "--"]
);
assert_eq!(args[3], XDOTOOL_TYPE_DELAY_MS.to_string());
assert_eq!(args[5], text);
}
#[test]
fn xdotool_type_uses_return_for_lf_and_crlf() {
let args = xdotool_type_args_with_delay("L1\nL2\r\nL3\r", 12);
assert_eq!(args[5], "L1\rL2\rL3\r");
assert_eq!(
xdotool_type_args_with_delay("quotes '\" and literal \\n: שלום", 12)[5],
"quotes '\" and literal \\n: שלום"
);
}
#[test]
fn xdotool_type_timeout_grows_with_text_length() {
assert_eq!(xdotool_type_timeout("", 12), INPUT_COMMAND_TIMEOUT);
assert_eq!(
xdotool_type_timeout(&"x".repeat(1_000), 12),
INPUT_COMMAND_TIMEOUT + Duration::from_secs(12)
);
assert_eq!(
xdotool_type_timeout(&"x".repeat(1_000), 0),
INPUT_COMMAND_TIMEOUT
);
}
#[tokio::test]
async fn launched_xdotool_failure_does_not_replay_through_ydotool() {
let dir = std::env::temp_dir().join(format!(
"computer-use-linux-xdotool-fallback-{}-{:?}",
std::process::id(),
std::time::SystemTime::now()
));
std::fs::create_dir_all(&dir).expect("create command test directory");
let ydotool = dir.join("ydotool");
let xdotool_marker = dir.join("xdotool-ran");
let ydotool_marker = dir.join("ydotool-ran");
std::fs::write(
&ydotool,
format!("#!/bin/sh\ntouch '{}'\n", ydotool_marker.display()),
)
.expect("write fake ydotool");
std::fs::set_permissions(&ydotool, std::fs::Permissions::from_mode(0o700))
.expect("make fake ydotool executable");
let xdotool_args = vec![
"-c".to_string(),
format!("touch '{}'; exit 9", xdotool_marker.display()),
];
let result = run_xdotool_or_fallback(Path::new("/bin/sh"), &xdotool_args, || async {
TokioCommand::new(&ydotool)
.output()
.await
.map_err(|error| error.to_string())
})
.await;
assert!(result.is_err());
assert!(xdotool_marker.exists(), "fake xdotool did not execute");
assert!(
!ydotool_marker.exists(),
"ydotool replayed input after xdotool started"
);
let _ = std::fs::remove_dir_all(dir);
}
#[tokio::test]
async fn unavailable_xdotool_uses_ydotool_fallback() {
let result = run_xdotool_or_fallback(
Path::new("/definitely/missing/xdotool"),
&xdotool_type_args_with_delay("text", XDOTOOL_TYPE_DELAY_MS),
|| async {
TokioCommand::new("sh")
.args(["-c", "exit 0"])
.output()
.await
.map_err(|error| error.to_string())
},
)
.await
.expect("spawn failure should use fallback");
assert_eq!(result.backend, KeyboardCommandBackend::Ydotool);
assert!(result.output.status.success());
}
#[tokio::test]
async fn wtype_passes_long_unicode_as_one_literal_argument() {
let dir = std::env::temp_dir().join(format!(
"computer-use-linux-wtype-unicode-{}-{:?}",
std::process::id(),
std::time::SystemTime::now()
));
std::fs::create_dir_all(&dir).expect("create command test directory");
let wtype = dir.join("wtype");
let captured = dir.join("captured");
std::fs::write(
&wtype,
format!(
"#!/bin/sh\n[ \"$#\" -eq 2 ] && [ \"$1\" = -- ] || exit 42\nprintf '%s' \"$2\" > '{}'\n",
captured.display()
),
)
.expect("write fake wtype");
std::fs::set_permissions(&wtype, std::fs::Permissions::from_mode(0o700))
.expect("make fake wtype executable");
let text =
"--Türkçe PostgreSQL (W, g, m, v, f, ş, ü) / 0123456789 - ; 'İstanbul'\n".repeat(50);
let result = run_wtype_type_text_or_fallback(
&wtype,
&text,
WtypeTextMode::Argument,
false,
|| async { panic!("available wtype must not fall back") },
)
.await
.expect("wtype should succeed");
assert_eq!(result.backend, KeyboardCommandBackend::Wtype);
assert_eq!(std::fs::read_to_string(&captured).unwrap(), text);
for text in ["--", "-", ""] {
run_wtype_type_text_or_fallback(
&wtype,
text,
WtypeTextMode::Argument,
false,
|| async { panic!("available wtype must not fall back") },
)
.await
.unwrap();
assert_eq!(std::fs::read_to_string(&captured).unwrap(), text);
}
let _ = std::fs::remove_dir_all(dir);
}
#[tokio::test]
async fn review_argument_mode_never_starts_wtype_for_protected_text() {
// Even /bin/false and the fallback must never run. An exit-status
// error would prove that validation happened too late.
let error = run_wtype_type_text_or_fallback(
Path::new("/bin/false"),
"synthetic-password",
WtypeTextMode::Argument,
true,
|| async { panic!("secret input must not fall back") },
)
.await
.err()
.expect("secret text must be rejected");
assert!(error.contains("password fields"), "{error}");
let error = run_wtype_type_text_or_fallback(
Path::new("/definitely/missing/wtype"),
"synthetic-password",
WtypeTextMode::Argument,
true,
|| async { panic!("secret input must not fall back even if wtype disappears") },
)
.await
.err()
.expect("secret text must be rejected");
assert!(error.contains("password fields"), "{error}");
}
#[tokio::test]
async fn review_slow_focus_probe_has_its_own_input_budget() {
let mode = hyprland_wtype_delivery("long text".repeat(100).as_str(), async {
tokio::time::sleep(Duration::from_millis(300)).await;
Some(FocusProbe::Found(FocusedElementSummary {
role: "entry".to_string(),
name: None,
editable: false,
states: vec!["focused".to_string(), "editable".to_string()],
is_terminal: false,
}))
})
.await
.unwrap();
assert_eq!(
mode,
WtypeTextMode::Argument,
"the indicator's 250 ms budget must not gate text dispatch"
);
}
#[tokio::test]
async fn review_inaccessible_fields_keep_short_text_on_private_stdin() {
let text = "ş".repeat(WTYPE_STDIN_MAX_CHARS);
for probe in [
None,
Some(FocusProbe::NoAccessibleApp),
Some(FocusProbe::Incomplete),
Some(FocusProbe::NoneFocused),
Some(FocusProbe::ShellOnly),
] {
let mode = hyprland_wtype_delivery(&text, async { probe })
.await
.unwrap();
assert_eq!(mode, WtypeTextMode::Stdin);
// cat accepts only '-' here; an argv text argument would instead
// be opened as a filename and fail. No desktop input is generated.
let result = run_wtype_type_text_or_fallback(
Path::new("/bin/cat"),
&text,
mode,
true,
|| async { panic!("available stdin helper must not fall back") },
)
.await
.unwrap();
assert_eq!(result.output.stdout, text.as_bytes());
}
}
#[tokio::test]
async fn review_ambiguous_focused_roles_use_stdin_instead_of_password_rejection() {
let short = "ş".repeat(WTYPE_STDIN_MAX_CHARS);
let long = "ş".repeat(WTYPE_STDIN_MAX_CHARS + 1);
for (role, terminal) in [
("terminal", true),
("Terminal", true),
("entry", true),
("Entry", false),
("Text", false),
("text box", false),
("TextBox", false),
("textbox", false),
("Unknown", false),
("invalid", false),
("", false),
] {
// Editable state/interface must not make an ambiguous role public.
for editable in [false, true] {
let element = FocusedElementSummary {
role: role.to_string(),
name: None,
editable,
states: vec!["focused".to_string(), "editable".to_string()],
is_terminal: terminal,
};
let mode = hyprland_wtype_delivery(&short, async {
Some(FocusProbe::Found(element.clone()))
})
.await;
assert!(
matches!(mode, Ok(WtypeTextMode::Stdin)),
"{role:?}, editable={editable}: {mode:?}"
);
let delivered = run_wtype_type_text_or_fallback(
Path::new("/bin/cat"),
&short,
mode.unwrap(),
true,
|| async { panic!("stdin helper must not fall back") },
)
.await
.unwrap();
assert_eq!(delivered.output.stdout, short.as_bytes());
let error =
hyprland_wtype_delivery(&long, async { Some(FocusProbe::Found(element)) })
.await
.unwrap_err();
assert!(
error.contains("100 Unicode characters"),
"{role:?}: {error}"
);
assert!(!error.contains("password fields"), "{role:?}: {error}");
}
}
}
#[tokio::test]
async fn only_confirmed_password_roles_are_hard_rejected() {
for role in [
"password text",
"PasswordText",
"PASSWORD TEXT",
"password entry",
] {
let error = hyprland_wtype_delivery("short", async {
Some(FocusProbe::Found(FocusedElementSummary {
role: role.to_string(),
name: None,
editable: true,
states: vec!["focused".to_string(), "editable".to_string()],
is_terminal: false,
}))
})
.await
.unwrap_err();
assert!(error.contains("password fields"), "{role}: {error}");
}
for role in ["entry", "text"] {
let mode = hyprland_wtype_delivery(&"x".repeat(3000), async {
Some(FocusProbe::Found(FocusedElementSummary {
role: role.to_string(),
name: None,
editable: false,
states: vec!["focused".to_string(), "editable".to_string()],
is_terminal: false,
}))
})
.await
.unwrap();
assert_eq!(mode, WtypeTextMode::Argument, "{role}");
}
}
#[tokio::test]
async fn unknown_stdin_limit_and_password_rejection_remain_explicit() {
let error = hyprland_wtype_delivery(&"ş".repeat(WTYPE_STDIN_MAX_CHARS + 1), async {
Some(FocusProbe::NoAccessibleApp)
})
.await
.unwrap_err();
assert!(error.contains("100 Unicode characters"), "{error}");
let error = hyprland_wtype_delivery("short", async {
Some(FocusProbe::Found(FocusedElementSummary {
role: "password text".to_string(),
name: None,
editable: true,
states: vec!["focused".to_string(), "editable".to_string()],
is_terminal: false,
}))
})
.await
.unwrap_err();
assert!(error.contains("password fields"), "{error}");
let mode = hyprland_wtype_delivery("short", std::future::pending())
.await
.unwrap();
assert_eq!(
mode,
WtypeTextMode::Stdin,
"timeout must not reject all input"
);
}
#[test]
fn electron_editable_state_does_not_require_the_editable_text_interface() {
let mut element = FocusedElementSummary {
role: "entry".to_string(),
name: None,
editable: false,
states: vec!["focused".to_string(), "editable".to_string()],
is_terminal: false,
};
assert!(!focus_requires_secret_protection(&FocusProbe::Found(
element.clone()
)));
assert!(!describe_focused_element(&element, true).starts_with("WARNING:"));
element.role = "password text".to_string();
assert!(focus_requires_secret_protection(&FocusProbe::Found(
element.clone()
)));
element.role = "frame".to_string();
element.states.clear();
assert!(focus_requires_secret_protection(&FocusProbe::Found(
element
)));
for probe in [
FocusProbe::ShellOnly,
FocusProbe::NoneFocused,
FocusProbe::Incomplete,
FocusProbe::NoAccessibleApp,
] {
assert!(focus_requires_secret_protection(&probe));
}
}
#[tokio::test]
async fn other_compositors_keep_wtype_text_on_stdin() {
for desktop in [None, Some("sway"), Some("wlroots"), Some("river")] {
let mode = wtype_text_mode_for_desktop(desktop);
assert_eq!(mode, WtypeTextMode::Stdin);
let text = "synthetic secret: şü / -- -";
let result = run_wtype_type_text_or_fallback(
Path::new("/bin/cat"),
text,
mode,
true,
|| async { panic!("available stdin helper must not fall back") },
)
.await
.unwrap();
assert_eq!(result.output.stdout, text.as_bytes());
}
assert_eq!(
wtype_text_mode_for_desktop(Some("Hyprland")),
WtypeTextMode::Argument
);
}
#[test]
fn review_invalid_text_is_rejected_before_window_targeting() {
let dir = std::env::temp_dir().join(format!("cul-text-preflight-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let helper = dir.join("wtype");
std::fs::write(&helper, "#!/bin/sh\nexit 99\n").unwrap();
std::fs::set_permissions(&helper, std::fs::Permissions::from_mode(0o700)).unwrap();
let mut command = Command::new(std::env::current_exe().unwrap());
command
.args([
"--exact",
"server::tests::invalid_text_preflight_child",
"--ignored",
"--nocapture",
])
.env("CUL_PREFLIGHT_CHILD", "1")
.env("PATH", &dir)
.env("XDG_SESSION_TYPE", "wayland")
.env("XDG_CURRENT_DESKTOP", "Hyprland")
.env_remove("DISPLAY");
for key in [
"COMPUTER_USE_LINUX_FORCE_YDOTOOL_KEYBOARD",
"COMPUTER_USE_LINUX_FORCE_XDOTOOL_KEYBOARD",
"COMPUTER_USE_LINUX_FORCE_PORTAL_KEYBOARD",
] {
command.env_remove(key);
}
let result = crate::command_runner::output_blocking_with_timeout(
&mut command,
"text preflight child",
Duration::from_secs(10),
)
.unwrap();
let _ = std::fs::remove_dir_all(dir);
assert!(
result.status.success(),
"{}{}",
String::from_utf8_lossy(&result.stdout),
String::from_utf8_lossy(&result.stderr)
);
}
#[tokio::test]
#[ignore = "isolated environment subprocess; invoked by review_invalid_text_is_rejected_before_window_targeting"]
async fn invalid_text_preflight_child() {
assert_eq!(env::var("CUL_PREFLIGHT_CHILD").as_deref(), Ok("1"));
let server = ComputerUseLinux::default();
assert_eq!(server.wtype_text_mode(), Some(WtypeTextMode::Argument));
for text in [
"prefix\0suffix".to_string(),
"ş".repeat(WTYPE_MAX_TEXT_BYTES / 2 + 1),
] {
let params: TypeTextParams =
serde_json::from_value(serde_json::json!({"text": text, "window_id": u64::MAX}))
.unwrap();
let Json(result) = server.type_text(Parameters(params)).await;
assert!(!result.ok);
assert!(
result.message.contains("Literal text was not sent: wtype"),
"{}",
result.message
);
}
}
#[tokio::test]
async fn unavailable_wtype_uses_ydotool_fallback() {
let result = run_wtype_type_text_or_fallback(
Path::new("/definitely/missing/wtype"),
"text",
WtypeTextMode::Argument,
false,
|| async {
TokioCommand::new("sh")
.args(["-c", "exit 0"])
.output()
.await
.map_err(|error| error.to_string())
},
)
.await
.expect("missing wtype should use fallback");
assert_eq!(result.backend, KeyboardCommandBackend::Ydotool);
}
#[test]
fn wtype_text_validation_counts_bytes_and_rejects_nul() {
assert!(validate_wtype_text(&"x".repeat(WTYPE_MAX_TEXT_BYTES)).is_ok());
assert!(validate_wtype_text(&"x".repeat(WTYPE_MAX_TEXT_BYTES + 1))
.unwrap_err()
.contains("not sent"));
// The argument limit counts UTF-8 bytes, not Unicode scalar values.
assert!(validate_wtype_text(&"ş".repeat(WTYPE_MAX_TEXT_BYTES / 2 + 1)).is_err());
assert!(validate_wtype_text("valid prefix\0suffix")
.unwrap_err()
.contains("not sent"));
assert!(validate_wtype_text("").is_ok());
}
#[tokio::test]
async fn launched_wtype_failure_does_not_replay_through_ydotool() {
let dir = std::env::temp_dir().join(format!(
"computer-use-linux-wtype-fallback-{}-{:?}",
std::process::id(),
std::time::SystemTime::now()
));
std::fs::create_dir_all(&dir).expect("create command test directory");
let wtype = dir.join("wtype");
let fallback_marker = dir.join("ydotool-ran");
std::fs::write(&wtype, "#!/bin/sh\nexit 9\n").expect("write fake wtype");
std::fs::set_permissions(&wtype, std::fs::Permissions::from_mode(0o700))
.expect("make fake wtype executable");
let result = run_wtype_type_text_or_fallback(
&wtype,
"text",
WtypeTextMode::Argument,
false,
|| async {
std::fs::write(&fallback_marker, "ran").unwrap();
TokioCommand::new("true")
.output()
.await
.map_err(|error| error.to_string())
},
)
.await;
assert!(result.is_err());
assert!(
!fallback_marker.exists(),
"ydotool replayed input after wtype started"
);
let _ = std::fs::remove_dir_all(dir);
}
#[tokio::test]
async fn cancelling_xdotool_wait_kills_the_child() {
let dir = std::env::temp_dir().join(format!(
"computer-use-linux-xdotool-cancel-{}-{:?}",
std::process::id(),
std::time::SystemTime::now()
));
std::fs::create_dir_all(&dir).expect("create command test directory");
let xdotool = dir.join("xdotool");
let pid_path = dir.join("pid");
std::fs::write(
&xdotool,
format!(
"#!/bin/sh\nprintf '%s' $$ > '{}'\nexec sleep 60\n",
pid_path.display()
),
)
.expect("write fake xdotool");
std::fs::set_permissions(&xdotool, std::fs::Permissions::from_mode(0o700))
.expect("make fake xdotool executable");
let task = tokio::spawn(async move { run_xdotool(&xdotool, &[]).await });
let mut pid = None;
for _ in 0..200 {
if let Ok(value) = std::fs::read_to_string(&pid_path) {
pid = value.parse::<u32>().ok();
if pid.is_some() {
break;
}
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
let pid = pid.expect("fake xdotool did not record its pid");
task.abort();
let _ = task.await;
for _ in 0..50 {
if !Path::new(&format!("/proc/{pid}")).exists() {
let _ = std::fs::remove_dir_all(dir);
return;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
unsafe {
libc::kill(pid as i32, libc::SIGKILL);
}
let _ = std::fs::remove_dir_all(dir);
panic!("cancelled xdotool child {pid} was not killed");
}
#[tokio::test]
async fn cancelling_between_press_and_release_keeps_input_locked() {
let lock = std::sync::Arc::new(tokio::sync::Mutex::new(()));
let guard = std::sync::Arc::clone(&lock).lock_owned().await;
let (pressed_tx, pressed_rx) = tokio::sync::oneshot::channel();
let (allow_release_tx, allow_release_rx) = tokio::sync::oneshot::channel();
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
let waiter = tokio::spawn(async move {
run_cancellation_safe_input(guard, async move {
let _ = pressed_tx.send(());
let _ = allow_release_rx.await;
let _ = released_tx.send(());
Ok::<(), String>(())
})
.await
});
pressed_rx.await.expect("press command did not finish");
waiter.abort();
let _ = waiter.await;
assert!(
lock.try_lock().is_err(),
"input lock was released while the paired operation was incomplete"
);
allow_release_tx
.send(())
.expect("release command stopped on caller cancellation");
timeout(Duration::from_secs(1), released_rx)
.await
.expect("release command did not finish")
.expect("release command dropped its completion marker");
timeout(
Duration::from_secs(1),
std::sync::Arc::clone(&lock).lock_owned(),
)
.await
.expect("input lock remained held after the operation finished");
}
/// The xdotool path must accept exactly the keys the evdev grammar accepts,
/// so switching backends can never silently widen or narrow the surface.
#[test]
fn xdotool_key_spec_rejects_everything_key_chord_rejects() {
for key in ["NotAKey", "", "ctrl+", "ctrl+NotAKey", "f13", "hyper+a"] {
assert_eq!(
xdotool_key_spec(key).is_some(),
key_chord(key).is_some(),
"backend grammars diverged for {key:?}"
);
}
}
#[test]
fn key_sequence_keeps_shortcuts_and_navigation_on_raw_events() {
assert_eq!(
key_sequence("Ctrl+L"),
Some(vec![
"29:1".to_string(),
"38:1".to_string(),
"38:0".to_string(),
"29:0".to_string(),
])
);
assert_eq!(
key_sequence("ArrowLeft"),
Some(vec!["105:1".to_string(), "105:0".to_string()])
);
assert_eq!(
key_sequence("Escape"),
Some(vec!["1:1".to_string(), "1:0".to_string()])
);
assert_eq!(
key_sequence("Enter"),
Some(vec!["28:1".to_string(), "28:0".to_string()])
);
}
#[test]
fn ydotool_type_timeout_scales_with_text_length() {
assert_eq!(ydotool_type_timeout("").as_secs(), 10);
assert_eq!(ydotool_type_timeout("x").as_secs(), 11);
assert_eq!(ydotool_type_timeout(&"x".repeat(200)).as_secs(), 20);
assert_eq!(ydotool_type_timeout(&"x".repeat(500)).as_secs(), 35);
}
#[tokio::test]
async fn command_wait_drains_output_before_exit() {
let mut command = tokio::process::Command::new("sh");
command.args(["-c", "yes noisy | head -c 200000 >&2; exit 7"]);
command.stdout(Stdio::piped());
command.stderr(Stdio::piped());
let output = crate::command_runner::output_with_timeout(
command,
"run test command",
Duration::from_secs(5),
)
.await
.expect("child should exit before timeout");
assert_eq!(output.status.code(), Some(7));
assert!(output.stderr.len() >= 100_000);
}
#[test]
fn perform_action_defaults_to_primary_action_index() {
assert_eq!(requested_or_primary_action(None), "0");
assert_eq!(requested_or_primary_action(Some(" ")), "0");
assert_eq!(
requested_or_primary_action(Some(" show-menu ")),
"show-menu"
);
}
#[test]
fn element_identifier_overrides_cached_object_ref() {
let backend = ComputerUseLinux::default();
backend.cache_nodes(&[node(7, None)]);
let object_ref = backend
.resolve_object_ref(
Some(7),
Some(":1.99/org/a11y/atspi/accessible/3"),
&ElementSelector::default(),
ElementResolvePurpose::Action,
)
.unwrap();
assert_eq!(object_ref, ":1.99/org/a11y/atspi/accessible/3");
}
#[test]
fn element_index_resolves_to_cached_object_ref() {
let backend = ComputerUseLinux::default();
backend.cache_nodes(&[node(7, None)]);
let object_ref = backend
.resolve_object_ref(
Some(7),
None,
&ElementSelector::default(),
ElementResolvePurpose::Action,
)
.unwrap();
assert_eq!(object_ref, ":1.7/org/a11y/atspi/accessible/7");
}
#[test]
fn semantic_selector_resolves_unique_cached_node_by_role_and_name() {
let backend = ComputerUseLinux::default();
let mut search_entry = node(7, None);
search_entry.role = "entry".to_string();
search_entry.name = Some("Search files".to_string());
search_entry.supports_editable_text = true;
backend.cache_nodes(&[search_entry]);
let object_ref = backend
.resolve_object_ref(
None,
None,
&ElementSelector {
role: Some("entry"),
name: Some("search"),
..Default::default()
},
ElementResolvePurpose::SetValue,
)
.unwrap();
assert_eq!(object_ref, ":1.7/org/a11y/atspi/accessible/7");
}
#[test]
fn semantic_selector_prefers_actionable_match() {
let backend = ComputerUseLinux::default();
let mut label = node(4, None);
label.role = "label".to_string();
label.name = Some("Close".to_string());
let mut button = node_with_actions(7, None, vec![click_action()]);
button.role = "push button".to_string();
button.name = Some("Close".to_string());
backend.cache_nodes(&[label, button]);
let object_ref = backend
.resolve_object_ref(
None,
None,
&ElementSelector {
name: Some("close"),
..Default::default()
},
ElementResolvePurpose::Action,
)
.unwrap();
assert_eq!(object_ref, ":1.7/org/a11y/atspi/accessible/7");
}
#[test]
fn semantic_selector_prefers_editable_match() {
let backend = ComputerUseLinux::default();
let mut label = node(4, None);
label.role = "label".to_string();
label.name = Some("Search".to_string());
let mut entry = node(7, None);
entry.role = "entry".to_string();
entry.name = Some("Search".to_string());
entry.supports_editable_text = true;
backend.cache_nodes(&[label, entry]);
let object_ref = backend
.resolve_object_ref(
None,
None,
&ElementSelector {
name: Some("search"),
..Default::default()
},
ElementResolvePurpose::SetValue,
)
.unwrap();
assert_eq!(object_ref, ":1.7/org/a11y/atspi/accessible/7");
}
#[test]
fn semantic_selector_reports_ambiguous_matches() {
let backend = ComputerUseLinux::default();
let mut first = node_with_actions(7, None, vec![click_action()]);
first.name = Some("Close".to_string());
let mut second = node_with_actions(9, None, vec![click_action()]);
second.name = Some("Close".to_string());
backend.cache_nodes(&[first, second]);
let error = backend
.resolve_object_ref(
None,
None,
&ElementSelector {
name: Some("close"),
..Default::default()
},
ElementResolvePurpose::Action,
)
.unwrap_err();
assert!(error.contains("matched multiple cached nodes"));
assert!(error.contains("element_index 7"));
assert!(error.contains("element_index 9"));
}
#[test]
fn semantic_click_selector_prefers_native_activation() {
let backend = ComputerUseLinux::default();
let mut button = node_with_actions(
7,
Some(Bounds {
x: 10,
y: 20,
width: 100,
height: 40,
}),
vec![click_action()],
);
button.name = Some("Run".to_string());
backend.cache_nodes(&[button]);
let target = backend
.resolve_click_target(&ClickParams {
role: Some("button".to_string()),
name: Some("run".to_string()),
..Default::default()
})
.unwrap();
assert!(matches!(
target,
ClickTarget::PrimaryAction {
action_index: 0,
..
}
));
}
#[test]
fn describe_focused_element_editable() {
let element = FocusedElementSummary {
role: "text".to_string(),
name: Some("Message".to_string()),
editable: true,
states: vec!["focused".to_string()],
is_terminal: false,
};
let described = describe_focused_element(&element, true);
assert!(described.contains("editable"));
assert!(!described.contains("WARNING"));
}
#[test]
fn describe_focused_terminal_accepts_keyboard_input_without_editable_text() {
let mut element = focused("terminal", true);
element.name = Some("Terminal".to_string());
let text = describe_focused_element(&element, true);
assert!(!text.contains("WARNING"), "{text}");
assert!(text.contains("terminal keyboard input"), "{text}");
}
#[test]
fn describe_focused_element_warns_on_non_editable_when_typing() {
let element = FocusedElementSummary {
role: "push button".to_string(),
name: Some("OK".to_string()),
editable: false,
states: vec!["focused".to_string()],
is_terminal: false,
};
let described = describe_focused_element(&element, true);
assert!(described.contains("WARNING"));
assert!(described.contains("not editable"));
}
#[test]
fn describe_focused_element_no_warning_for_press_key() {
let element = FocusedElementSummary {
role: "push button".to_string(),
name: None,
editable: false,
states: vec![],
is_terminal: false,
};
let described = describe_focused_element(&element, false);
assert!(!described.contains("WARNING"));
}
#[test]
fn relative_scroll_translates_coordinates() {
let mut params = ScrollParams {
element_index: None,
x: Some(10),
y: Some(20),
direction: "down".to_string(),
pages: None,
window_id: Some(1),
pid: None,
app_id: None,
wm_class: None,
window_title: None,
relative: Some(true),
};
apply_window_relative_scroll_coordinates(&mut params, (100, 200, 800, 600)).unwrap();
assert_eq!(params.x, Some(110));
assert_eq!(params.y, Some(220));
}
#[test]
fn window_targeted_scroll_defaults_to_window_center() {
let mut params = ScrollParams {
element_index: None,
x: None,
y: None,
direction: "down".to_string(),
pages: None,
window_id: Some(1),
pid: None,
app_id: None,
wm_class: None,
window_title: None,
relative: None,
};
apply_window_center_scroll_point(&mut params, (100, 200, 800, 600)).unwrap();
assert_eq!(params.x, Some(500));
assert_eq!(params.y, Some(500));
}
#[test]
fn window_targeted_scroll_with_empty_capture_rect_errors() {
let mut params = ScrollParams {
element_index: None,
x: None,
y: None,
direction: "down".to_string(),
pages: None,
window_id: Some(1),
pid: None,
app_id: None,
wm_class: None,
window_title: None,
relative: None,
};
let error = apply_window_center_scroll_point(&mut params, (0, 0, 0, 0)).unwrap_err();
assert!(error.contains("pass x/y explicitly"));
assert_eq!(params.x, None);
assert_eq!(params.y, None);
}
#[test]
fn relative_scroll_rejects_out_of_bounds() {
let mut params = ScrollParams {
element_index: None,
x: Some(801),
y: Some(20),
direction: "down".to_string(),
pages: None,
window_id: Some(1),
pid: None,
app_id: None,
wm_class: None,
window_title: None,
relative: Some(true),
};
assert!(
apply_window_relative_scroll_coordinates(&mut params, (100, 200, 800, 600)).is_err()
);
}
#[test]
fn shell_execution_requires_exact_operator_opt_in() {
assert!(shell_execution_enabled_value(Some("1")));
for value in [None, Some(""), Some("0"), Some("true"), Some("yes")] {
assert!(!shell_execution_enabled_value(value));
}
}
#[tokio::test]
async fn shell_execution_clears_ambient_credentials_and_accepts_explicit_env() {
let _serial = SHELL_ENV_TEST_LOCK.lock().await;
let _enabled = EnvVarGuard::set(SHELL_ENABLE_ENV, "1");
let _credential = EnvVarGuard::set("OPENAI_API_KEY", "test-ambient-credential");
let output = execute_shell(RunShellParams {
command: "printf '%s|%s' \"${OPENAI_API_KEY-unset}\" \"$EXPLICIT_VALUE\"".into(),
cwd: None,
env: BTreeMap::from([("EXPLICIT_VALUE".into(), "test-explicit-value".into())]),
timeout_seconds: Some(2),
})
.await;
assert!(output.ok, "{:?}", output.error);
assert_eq!(output.stdout, "unset|test-explicit-value");
assert_eq!(output.exit_code, Some(0));
}
#[tokio::test]
async fn shell_execution_disabled_never_runs_requested_command() {
async fn assert_refused() {
let output = execute_shell(RunShellParams {
command: "printf must-not-run".into(),
cwd: None,
env: BTreeMap::new(),
timeout_seconds: None,
})
.await;
assert!(!output.ok);
assert!(output.stdout.is_empty());
assert!(output
.error
.unwrap()
.contains("shell execution is disabled"));
}
let _serial = SHELL_ENV_TEST_LOCK.lock().await;
{
let _disabled = EnvVarGuard::set(SHELL_ENABLE_ENV, "0");
assert_refused().await;
}
{
let _unset = EnvVarGuard::unset(SHELL_ENABLE_ENV);
assert_refused().await;
}
}
#[test]
fn shell_environment_policy_excludes_ambient_credentials() {
for allowed in [
"PATH",
"HOME",
"LANG",
"LC_ALL",
"XDG_RUNTIME_DIR",
"DISPLAY",
"WAYLAND_DISPLAY",
"DBUS_SESSION_BUS_ADDRESS",
] {
assert!(inherited_shell_environment_allowed(allowed));
}
for secret in [
"AWS_SECRET_ACCESS_KEY",
"GITHUB_TOKEN",
"OPENAI_API_KEY",
"SSH_AUTH_SOCK",
"LD_PRELOAD",
] {
assert!(!inherited_shell_environment_allowed(secret));
}
}
#[test]
fn shell_environment_skips_non_utf8_entries() {
use std::os::unix::ffi::OsStringExt;
let variables = [
(OsString::from("PATH"), OsString::from("/usr/bin")),
(OsString::from_vec(vec![0xff]), OsString::from("ignored")),
(OsString::from("LANG"), OsString::from_vec(vec![0xff])),
(OsString::from("GITHUB_TOKEN"), OsString::from("secret")),
];
assert_eq!(
inherited_shell_environment(variables),
vec![("PATH".to_string(), "/usr/bin".to_string())]
);
}
#[test]
fn shell_environment_names_follow_exec_rules() {
for valid in ["A", "_PRIVATE", "NAME_2"] {
assert!(valid_environment_name(valid));
}
for invalid in ["", "2FAST", "BAD-NAME", "A=B", "naïve"] {
assert!(!valid_environment_name(invalid));
}
}
#[test]
fn shell_output_is_bounded_before_mcp_serialization() {
let oversized = vec![b'x'; SHELL_RESPONSE_STREAM_BYTES + 17];
let (visible, truncated) = bounded_shell_stream(&oversized);
assert!(truncated);
assert_eq!(visible.len(), SHELL_RESPONSE_STREAM_BYTES);
let (visible, truncated) = bounded_shell_stream(b"ok");
assert!(!truncated);
assert_eq!(visible, "ok");
}
#[test]
fn shell_audit_digest_is_stable_and_does_not_echo_command_text() {
let digest = shell_command_sha256("printf secret");
assert_eq!(digest.len(), 64);
assert!(digest
.chars()
.all(|character| character.is_ascii_hexdigit()));
assert!(!digest.contains("secret"));
assert_eq!(digest, shell_command_sha256("printf secret"));
}
}
#[cfg(test)]
mod accessibility_tree_note_tests {
use super::{truncated_accessibility_tree_note, unscoped_accessibility_tree_warning};
#[test]
fn unscoped_warning_only_when_tree_succeeded_without_a_scope() {
let warning = unscoped_accessibility_tree_warning(false, true, false).unwrap();
assert!(warning.starts_with("WARNING:"));
assert!(warning.contains("no app target scoped"));
assert!(warning.contains("app_name_or_bundle_identifier"));
assert!(warning.contains("window_id, pid, app_id, wm_class, title"));
for target_requested in [false, true] {
assert!(unscoped_accessibility_tree_warning(true, true, target_requested).is_none());
assert!(unscoped_accessibility_tree_warning(false, false, target_requested).is_none());
assert!(unscoped_accessibility_tree_warning(true, false, target_requested).is_none());
}
}
/// A caller that already passed a target must not be told to pass a target;
/// that advice would replay the same desktop-wide snapshot.
#[test]
fn unscoped_warning_with_a_target_points_at_accessibility_support() {
let warning = unscoped_accessibility_tree_warning(false, true, true).unwrap();
assert!(warning.starts_with("WARNING:"));
assert!(warning.contains("matched no AT-SPI application root"));
assert!(warning.contains("force-renderer-accessibility"));
assert!(warning.contains("list_apps"));
assert!(!warning.contains("Pass app_name_or_bundle_identifier or a window target"));
}
#[test]
fn truncated_note_only_when_the_node_cap_was_hit() {
let note = truncated_accessibility_tree_note(true).unwrap();
assert!(note.contains("raise max_nodes or max_depth"));
assert!(
!note.contains("smaller max_nodes"),
"shrinking the cap belongs to the unscoped warning, not the truncation note"
);
assert!(truncated_accessibility_tree_note(false).is_none());
}
}
#[cfg(test)]
mod focus_probe_feedback_tests {
use super::{focus_probe_feedback, FocusProbe, FocusedElementSummary};
#[test]
fn only_a_complete_empty_search_warns() {
let none = focus_probe_feedback(&FocusProbe::NoneFocused, true, false);
assert!(none.starts_with("WARNING:"), "{none}");
for probe in [FocusProbe::Incomplete, FocusProbe::NoAccessibleApp] {
let text = focus_probe_feedback(&probe, true, false);
assert!(
!text.contains("WARNING"),
"{probe:?} must not claim lost input: {text}"
);
assert!(text.contains("could not be verified"), "{text}");
}
assert!(
focus_probe_feedback(&FocusProbe::Incomplete, true, false).contains("search limits")
);
assert!(
focus_probe_feedback(&FocusProbe::NoAccessibleApp, true, false)
.contains("--force-renderer-accessibility")
);
}
#[test]
fn review_exact_focus_only_softens_shell_only_tree() {
for expects_editable in [false, true] {
let text = focus_probe_feedback(&FocusProbe::NoneFocused, expects_editable, true);
assert!(text.starts_with("WARNING:"), "{text}");
let text = focus_probe_feedback(&FocusProbe::ShellOnly, expects_editable, false);
assert!(text.starts_with("WARNING:"), "{text}");
let text = focus_probe_feedback(&FocusProbe::ShellOnly, expects_editable, true);
assert!(!text.contains("WARNING"), "{text}");
assert!(!text.contains("landed nowhere"), "{text}");
assert!(text.contains("before input"), "{text}");
assert!(text.contains("Field contents were not verified"), "{text}");
}
}
#[test]
fn found_element_is_described() {
let element = FocusedElementSummary {
role: "text".to_string(),
name: None,
editable: true,
states: vec!["focused".to_string()],
is_terminal: false,
};
let text = focus_probe_feedback(&FocusProbe::Found(element), true, false);
assert!(text.contains("text"), "{text}");
assert!(!text.starts_with("WARNING"), "{text}");
}
}
#[cfg(test)]
mod node_target_scope_tests {
use super::*;
fn cached_node(index: u32, object_ref: &str) -> AccessibilityNode {
AccessibilityNode {
index,
parent_index: None,
depth: 1,
object_ref: object_ref.to_string(),
role: "push button".to_string(),
name: Some("Hit me".to_string()),
description: None,
child_count: 0,
bounds: None,
states: vec!["enabled".to_string()],
actions: Vec::new(),
value: None,
text: None,
supports_editable_text: false,
}
}
#[tokio::test]
async fn index_from_another_apps_snapshot_is_rejected() {
let backend = ComputerUseLinux::default();
let node = cached_node(2, ":1.9/org/a11y/atspi/accessible/42");
backend.cache_snapshot(std::slice::from_ref(&node), Some(2_690_687));
let error = backend
.check_node_target(&node, Some(2_690_664))
.await
.unwrap_err();
assert!(error.contains("pid 2690687"), "{error}");
assert!(error.contains("target pid 2690664"), "{error}");
assert!(
error.contains("Call get_app_state for the target app"),
"{error}"
);
}
#[tokio::test]
async fn same_target_or_no_target_is_allowed() {
let backend = ComputerUseLinux::default();
let node = cached_node(2, ":1.9/org/a11y/atspi/accessible/42");
backend.cache_snapshot(std::slice::from_ref(&node), Some(2_690_664));
assert!(backend
.check_node_target(&node, Some(2_690_664))
.await
.is_ok());
assert!(backend.check_node_target(&node, None).await.is_ok());
}
#[tokio::test]
async fn unscoped_snapshot_with_unknown_owner_is_allowed() {
// No snapshot pid, and the owner lookup cannot resolve this ref here,
// so a mismatch cannot be shown and the action proceeds.
let backend = ComputerUseLinux::default();
let node = cached_node(2, ":1.9999/org/a11y/atspi/accessible/42");
backend.cache_snapshot(std::slice::from_ref(&node), None);
assert!(backend.check_node_target(&node, Some(1)).await.is_ok());
}
#[tokio::test]
async fn concurrent_snapshot_commit_waits_until_checked_action_finishes() {
let backend = ComputerUseLinux::default();
let mut original = cached_node(2, ":1.9/org/a11y/atspi/accessible/42");
original.bounds = Some(Bounds {
x: 10,
y: 20,
width: 100,
height: 40,
});
original.actions = vec![AccessibilityAction {
index: 0,
name: "click".into(),
description: String::new(),
keybinding: String::new(),
}];
backend
.commit_snapshot(std::slice::from_ref(&original), Some(10))
.await;
// Model the click/scroll operation: it holds this same lock while its
// target ownership check awaits and until the action has been resolved.
let input_guard = Arc::clone(&backend.input_operation_lock).lock_owned().await;
let checked_node = backend
.cached_node_for(
Some(2),
&ElementSelector::default(),
ElementResolvePurpose::Click,
)
.unwrap();
let barrier = Arc::new(tokio::sync::Barrier::new(2));
let writer_backend = backend.clone();
let writer_barrier = Arc::clone(&barrier);
let writer = tokio::spawn(async move {
let replacement = cached_node(2, ":1.20/org/a11y/atspi/accessible/99");
writer_barrier.wait().await;
writer_backend
.commit_snapshot(&[replacement], Some(20))
.await;
});
barrier.wait().await;
tokio::task::yield_now().await;
assert!(
!writer.is_finished(),
"snapshot publication must wait for input"
);
assert!(backend
.check_node_target(&checked_node, Some(10))
.await
.is_ok());
let params = ClickParams {
element_index: Some(2),
..Default::default()
};
assert!(matches!(backend.resolve_click_target(¶ms).unwrap(),
ClickTarget::PrimaryAction { object_ref, .. } if object_ref == original.object_ref));
assert_eq!(
backend
.resolve_optional_target_point(None, None, Some(2))
.unwrap(),
Some((60, 40))
);
// perform_action/set_value resolve the same snapshot while holding the
// input lock; their explicit object identifiers remain independent.
for purpose in [
ElementResolvePurpose::Action,
ElementResolvePurpose::SetValue,
] {
assert_eq!(
backend
.resolve_object_ref(Some(2), None, &ElementSelector::default(), purpose)
.unwrap(),
original.object_ref
);
}
drop(input_guard);
writer.await.unwrap();
let snapshot = backend.last_snapshot.lock().unwrap();
assert_eq!(snapshot.pid, Some(20));
assert_eq!(
snapshot.nodes[0].object_ref,
":1.20/org/a11y/atspi/accessible/99"
);
}
#[test]
fn a_new_snapshot_replaces_the_recorded_pid() {
let backend = ComputerUseLinux::default();
backend.cache_snapshot(&[], Some(10));
backend.cache_snapshot(&[], None);
assert_eq!(backend.last_snapshot.lock().unwrap().pid, None);
backend.cache_snapshot(&[], Some(11));
backend.clear_cached_nodes();
assert_eq!(backend.last_snapshot.lock().unwrap().pid, None);
}
}