use crate::util::UnwrapPoison;
use anyhow::anyhow;
use serde::Deserialize;
use serde::Serialize;
use serde::de::{self, MapAccess, SeqAccess, Visitor};
use std::collections::HashMap;
use std::fmt;
use std::fmt::Write as _;
use std::path::PathBuf;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, LazyLock, Mutex};
use std::time::{Duration, Instant};
pub(crate) const NORMALIZED_MAX: f64 = 1000.0;
pub(crate) const AX_THIN_INTERACTIVE_ELEMENTS: usize = 5;
#[cfg(any(target_os = "macos", target_os = "linux", test))]
pub(crate) const MAX_TREE_NODES: usize = 4000;
#[cfg(any(target_os = "linux", test))]
pub(crate) const MAX_CHILDREN: usize = 1000;
pub(crate) const RENDER_VALUE_CHARS: usize = 120;
#[cfg_attr(
not(any(target_os = "macos", target_os = "windows", test)),
expect(dead_code)
)]
pub(crate) const MAX_VALUE_CHARS: usize = 2000;
#[cfg_attr(
not(any(target_os = "macos", target_os = "linux", target_os = "windows", test)),
expect(dead_code)
)]
pub(crate) const MAX_NAME_CHARS: usize = 200;
pub(crate) const MAX_ARG_CHARS: usize = 2000;
pub(crate) const MAX_SCROLL_AMOUNT: u32 = 100;
pub(crate) const CAPTURE_BUDGET_PIXELS: u64 = 128 * 1024 * 1024;
#[cfg(any(target_os = "macos", target_os = "linux"))]
pub(crate) const EXCHANGE_DEADLINE: Duration = Duration::from_secs(10);
#[cfg(any(target_os = "macos", target_os = "linux"))]
pub(crate) const OBSERVE_BUDGET: Duration = Duration::from_secs(120);
#[cfg(any(target_os = "macos", target_os = "linux"))]
pub(crate) const ACT_BUDGET: Duration = OBSERVE_BUDGET.saturating_add(EXCHANGE_DEADLINE);
#[cfg(any(target_os = "macos", target_os = "linux"))]
pub(crate) const LIST_BUDGET: Duration = Duration::from_secs(20);
#[cfg(any(target_os = "macos", target_os = "linux"))]
pub(crate) const APP_SLICE: Duration = EXCHANGE_DEADLINE;
#[cfg(any(target_os = "macos", target_os = "linux"))]
pub(crate) const SKIPPED_NOTE: &str = "did not answer within its share of the call — skipped";
#[cfg(any(target_os = "macos", target_os = "linux"))]
pub(crate) fn unread_reason(err: &anyhow::Error, share: &Budget) -> String {
if share.expired() {
SKIPPED_NOTE.to_string()
} else {
format!("{err:#}")
}
}
#[derive(Clone)]
#[cfg(any(target_os = "macos", target_os = "linux", target_os = "windows"))]
pub(crate) struct Budget {
end: Instant,
limit: Duration,
what: &'static str,
target: String,
part: bool,
}
#[cfg(any(target_os = "macos", target_os = "linux", target_os = "windows"))]
impl Budget {
pub(crate) fn new(limit: Duration, what: &'static str, target: impl Into<String>) -> Self {
Self {
end: Instant::now() + limit,
limit,
what,
target: target.into(),
part: false,
}
}
#[cfg(any(target_os = "macos", target_os = "linux"))]
pub(crate) fn slice(&self, share: Duration, target: impl Into<String>) -> Self {
let now = Instant::now();
let end = self.end.min(now + share);
Self {
end,
limit: end.saturating_duration_since(now),
what: self.what,
target: target.into(),
part: true,
}
}
#[cfg(any(target_os = "macos", target_os = "linux"))]
pub(crate) fn target(&self) -> &str {
&self.target
}
#[cfg(any(target_os = "macos", target_os = "linux"))]
fn retarget(&mut self, target: &str) {
target.clone_into(&mut self.target);
}
pub(crate) fn expired(&self) -> bool {
Instant::now() >= self.end
}
pub(crate) fn check(&self) -> anyhow::Result<()> {
if self.expired() {
Err(self.error())
} else {
Ok(())
}
}
pub(crate) fn note(&self) -> String {
tracing::warn!(
target = %self.target,
limit_secs = self.limit.as_secs(),
part = self.part,
"computer use: the {} budget ran out",
self.what
);
if self.part {
format!(
"{} stopped answering accessibility requests — it was skipped, so the {} went on \
without it",
self.target, self.what
)
} else {
format!(
"{} stopped answering accessibility requests — the {} was abandoned after {}s; {}",
self.target,
self.what,
self.limit.as_secs(),
unanswered_next_step()
)
}
}
fn error(&self) -> anyhow::Error {
taxonomy_error(ERR_DEGRADED, self.note())
}
}
#[cfg(any(target_os = "macos", target_os = "linux"))]
pub(crate) fn surface_label(app_name: &str, title: Option<&str>) -> String {
window_description(&cap_label(app_name), &cap_label(title.unwrap_or_default()))
}
#[cfg(any(target_os = "macos", target_os = "linux"))]
pub(crate) fn windows_label(app_name: &str) -> String {
format!("the windows of {}", cap_label(app_name))
}
#[cfg(any(target_os = "macos", target_os = "linux"))]
pub(crate) fn name_surface(budget: &mut Budget, app_name: &str, title: Option<&str>) -> String {
let label = surface_label(app_name, title);
budget.retarget(&label);
label
}
pub(crate) fn unanswered_next_step() -> &'static str {
if super::capture_known_available() {
"retry, or use screenshot instead"
} else {
"retry it"
}
}
#[cfg(any(target_os = "macos", target_os = "linux"))]
pub(crate) fn unanswered(surface: &str, what: &str) -> anyhow::Error {
taxonomy_error(
ERR_DEGRADED,
format!(
"{surface} did not answer {what} — it may be busy or hung; {}",
unanswered_next_step()
),
)
}
const INTERACTIVE_ROLES: &[&str] = &[
"button",
"checkbox",
"radiobutton",
"menuitem",
"menu",
"combobox",
"textfield",
"textarea",
"searchfield",
"slider",
"tab",
"tabgroup",
"link",
"listitem",
"popupbutton",
"stepper",
"colorwell",
"datefield",
"tokenfield",
"outline",
"table",
];
pub(crate) const ERR_PERMISSION_DENIED: &str = "permission-denied";
pub(crate) const ERR_UNSUPPORTED: &str = "unsupported";
pub(crate) const ERR_DEGRADED: &str = "degraded";
pub(crate) const ERR_DESKTOP_UNAVAILABLE: &str = "desktop-unavailable";
pub(crate) const ERR_STALE_ELEMENT: &str = "stale-element";
#[cfg_attr(
not(any(target_os = "macos", target_os = "windows")),
expect(dead_code)
)]
pub(crate) const ERR_FRAMEWORK_FAILED: &str = "framework-failed";
#[cfg_attr(
not(any(target_os = "macos", target_os = "linux", target_os = "windows")),
expect(dead_code)
)]
pub(crate) const ERR_AMBIGUOUS_LOCATOR: &str = "ambiguous-locator";
pub(crate) const ERR_NOT_MATCHED: &str = "not-matched";
#[cfg_attr(
not(any(target_os = "linux", target_os = "macos", target_os = "windows")),
expect(dead_code)
)]
pub(crate) const ERR_SURFACE_GONE: &str = "surface-gone";
#[cfg_attr(not(target_os = "windows"), expect(dead_code))]
pub(crate) const ERR_SELF_TARGET: &str = "self-target";
#[must_use]
pub(crate) fn taxonomy_error(kind: &str, msg: impl fmt::Display) -> anyhow::Error {
anyhow!("{kind}: {msg}")
}
#[cfg_attr(
not(any(target_os = "macos", target_os = "windows")),
expect(dead_code)
)]
pub(crate) fn framework_failed(what: &str, detail: impl fmt::Display) -> anyhow::Error {
anyhow::Error::new(FrameworkFailed(taxonomy_error(
ERR_FRAMEWORK_FAILED,
format!("{what} — {detail}"),
)))
}
pub(crate) fn is_framework_failed(err: &anyhow::Error) -> bool {
err.downcast_ref::<FrameworkFailed>().is_some()
}
#[derive(Debug)]
struct FrameworkFailed(anyhow::Error);
impl fmt::Display for FrameworkFailed {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
self.0.fmt(f)
}
}
impl std::error::Error for FrameworkFailed {}
#[cfg_attr(
not(any(target_os = "linux", target_os = "macos", target_os = "windows", test)),
expect(dead_code)
)]
pub(crate) fn surface_gone(app_name: &str) -> anyhow::Error {
taxonomy_error(
ERR_SURFACE_GONE,
format!(
"the window of '{app_name}' this target names can no longer be identified — it \
closed or was replaced; enumerate windows again"
),
)
}
#[cfg_attr(
not(any(target_os = "linux", target_os = "macos", target_os = "windows", test)),
expect(dead_code)
)]
pub(crate) fn minimized_window(title: &str) -> anyhow::Error {
taxonomy_error(
ERR_DEGRADED,
format!(
"{} is minimized — restore it first; a minimized window has no visible pixels",
window_description("", title)
),
)
}
#[cfg_attr(
not(any(target_os = "linux", target_os = "macos", target_os = "windows")),
expect(dead_code)
)]
pub(crate) fn not_brought_forward(surface: impl fmt::Display) -> anyhow::Error {
taxonomy_error(
ERR_DEGRADED,
format!(
"{surface} could not be brought to the front — the input would have gone to another \
window, so nothing was sent; bring the window forward (click it) and repeat the action"
),
)
}
#[cfg_attr(
not(any(target_os = "macos", target_os = "windows")),
expect(dead_code)
)]
pub(crate) fn permission_denied(msg: impl fmt::Display) -> anyhow::Error {
anyhow::Error::new(PermissionDenied(taxonomy_error(ERR_PERMISSION_DENIED, msg)))
}
fn ceiling_refused(what: &str, width: u32, height: u32, held: u64) -> anyhow::Error {
anyhow::Error::new(CeilingRefusal(taxonomy_error(
ERR_DEGRADED,
format!(
"{what} needs {} MiB of RGBA ({width}x{height}) and this call already holds {} MiB \
of its {} MiB — a capture this large cannot be held whole; name a smaller surface \
if one exists",
mib(u64::from(width) * u64::from(height)),
mib(held),
mib(CAPTURE_BUDGET_PIXELS),
),
)))
}
#[cfg_attr(not(any(target_os = "windows", test)), expect(dead_code))]
pub(crate) fn is_ceiling_refusal(err: &anyhow::Error) -> bool {
err.downcast_ref::<CeilingRefusal>().is_some()
}
#[derive(Debug)]
struct CeilingRefusal(anyhow::Error);
impl fmt::Display for CeilingRefusal {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
self.0.fmt(f)
}
}
impl std::error::Error for CeilingRefusal {}
pub(crate) fn is_permission_denied(err: &anyhow::Error) -> bool {
err.downcast_ref::<PermissionDenied>().is_some()
}
#[derive(Debug)]
struct PermissionDenied(anyhow::Error);
impl fmt::Display for PermissionDenied {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
self.0.fmt(f)
}
}
impl std::error::Error for PermissionDenied {}
#[derive(Debug, Clone, Copy, PartialEq)]
pub(crate) struct SurfaceGeometry {
pub x: f64,
pub y: f64,
pub width: f64,
pub height: f64,
}
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Default)]
pub(crate) struct Region {
pub x0: f64,
pub y0: f64,
pub x1: f64,
pub y1: f64,
}
impl<'de> Deserialize<'de> for Region {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
struct RegionVisitor;
impl<'de> Visitor<'de> for RegionVisitor {
type Value = Region;
fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str("a 4-element array [x0, y0, x1, y1] or object {x0, y0, x1, y1}")
}
fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
where
A: SeqAccess<'de>,
{
let x0 = seq
.next_element()?
.ok_or_else(|| de::Error::missing_field("x0"))?;
let y0 = seq
.next_element()?
.ok_or_else(|| de::Error::missing_field("y0"))?;
let x1 = seq
.next_element()?
.ok_or_else(|| de::Error::missing_field("x1"))?;
let y1 = seq
.next_element()?
.ok_or_else(|| de::Error::missing_field("y1"))?;
if seq.next_element::<de::IgnoredAny>()?.is_some() {
return Err(de::Error::custom(
"a region is exactly [x0, y0, x1, y1] — extra values name nothing",
));
}
Ok(Region { x0, y0, x1, y1 })
}
fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>
where
A: MapAccess<'de>,
{
let mut x0 = None;
let mut y0 = None;
let mut x1 = None;
let mut y1 = None;
while let Some(key) = map.next_key::<String>()? {
match key.as_str() {
"x0" => x0 = Some(map.next_value()?),
"y0" => y0 = Some(map.next_value()?),
"x1" => x1 = Some(map.next_value()?),
"y1" => y1 = Some(map.next_value()?),
other => {
return Err(de::Error::unknown_field(other, &["x0", "y0", "x1", "y1"]));
}
}
}
Ok(Region {
x0: x0.ok_or_else(|| de::Error::missing_field("x0"))?,
y0: y0.ok_or_else(|| de::Error::missing_field("y0"))?,
x1: x1.ok_or_else(|| de::Error::missing_field("x1"))?,
y1: y1.ok_or_else(|| de::Error::missing_field("y1"))?,
})
}
}
deserializer.deserialize_any(RegionVisitor)
}
}
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct Locator {
pub path: Vec<usize>,
pub role: String,
pub name: Option<String>,
}
#[derive(Debug, Clone)]
pub(crate) struct UiNode {
pub role: String,
pub name: Option<String>,
pub value: Option<String>,
pub frame: Option<(f64, f64, f64, f64)>,
pub actions: Vec<String>,
pub focused: bool,
pub children: Vec<UiNode>,
}
#[derive(Debug, Clone)]
pub(crate) enum Impact {
Done,
#[cfg_attr(
not(any(target_os = "linux", target_os = "macos", test)),
expect(dead_code)
)]
DoneWith(String),
Unconfirmed(String),
}
#[must_use]
pub(crate) fn impact_text(done: String, impact: Impact) -> String {
match impact {
Impact::Done => done,
Impact::DoneWith(note) => format!("{done} — {note}"),
Impact::Unconfirmed(detail) => format!(
"applied but unconfirmed — {done}: {detail}. The action was delivered to the system \
but not confirmed; it may already have taken effect, so check the target's state \
before repeating it, and never repeat it blind, because a second attempt would act \
twice."
),
}
}
fn cap_text(text: &str, max: usize) -> (String, bool) {
let kept = crate::util::truncate(text, max);
let cut = kept != text;
(kept, cut)
}
#[cfg_attr(
not(any(target_os = "macos", target_os = "linux", target_os = "windows", test)),
expect(dead_code)
)]
pub(crate) fn cap_label(text: &str) -> String {
cap_text(text, MAX_NAME_CHARS).0
}
pub(crate) fn capture_size(what: &str, width: usize, height: usize) -> anyhow::Result<(u32, u32)> {
if width == 0 || height == 0 {
return Err(taxonomy_error(
ERR_DEGRADED,
format!(
"{what} has no capture area ({width}x{height} px) — it is minimized, off-screen \
or was closed; restore it and re-observe"
),
));
}
match (u32::try_from(width), u32::try_from(height)) {
(Ok(width), Ok(height)) => Ok((width, height)),
_ => Err(taxonomy_error(
ERR_DEGRADED,
format!(
"{what} measures {width}x{height} px, which is more than a capture can express — \
name a smaller surface instead"
),
)),
}
}
#[derive(Debug, Default, Clone, Copy)]
pub(crate) struct Truncation {
pub nodes: Option<usize>,
pub depth: Option<Cut>,
pub children: Option<Cut>,
pub children_cut: Option<Cut>,
pub values: Option<Cut>,
pub names: Option<Cut>,
}
#[derive(Debug, Clone, Copy)]
pub(crate) struct Cut {
pub count: usize,
pub limit: usize,
}
fn counted(count: usize, one: &'static str, many: &'static str) -> &'static str {
if count == 1 { one } else { many }
}
impl Truncation {
#[cfg_attr(
not(any(target_os = "macos", target_os = "linux", target_os = "windows")),
expect(dead_code)
)]
pub(crate) fn hit_node_ceiling(&mut self, limit: usize) {
self.nodes = Some(limit);
}
#[cfg_attr(
not(any(target_os = "macos", target_os = "linux", target_os = "windows")),
expect(dead_code)
)]
pub(crate) fn cut_depth(&mut self, limit: usize) {
self.depth.get_or_insert(Cut { count: 0, limit }).count += 1;
}
#[cfg(target_os = "linux")]
pub(crate) fn cut_children(&mut self, limit: usize) {
self.children.get_or_insert(Cut { count: 0, limit }).count += 1;
}
#[cfg(target_os = "linux")]
pub(crate) fn trim_children(&mut self, limit: usize) {
self.children_cut
.get_or_insert(Cut { count: 0, limit })
.count += 1;
}
#[cfg_attr(
not(any(target_os = "macos", target_os = "windows", test)),
expect(dead_code)
)]
pub(crate) fn keep_value(&mut self, text: &str) -> String {
let (kept, cut) = cap_text(text, MAX_VALUE_CHARS);
if cut {
self.cut_value(MAX_VALUE_CHARS);
}
kept
}
#[cfg_attr(
not(any(target_os = "macos", target_os = "linux", target_os = "windows", test)),
expect(dead_code)
)]
pub(crate) fn keep_name(&mut self, text: &str) -> String {
let (kept, cut) = cap_text(text, MAX_NAME_CHARS);
if cut {
self.cut_name(MAX_NAME_CHARS);
}
kept
}
fn cut_value(&mut self, limit: usize) {
self.values.get_or_insert(Cut { count: 0, limit }).count += 1;
}
fn cut_name(&mut self, limit: usize) {
self.names.get_or_insert(Cut { count: 0, limit }).count += 1;
}
fn statements(&self, values_cut: usize) -> Vec<String> {
let mut out = Vec::new();
if let Some(ceiling) = self.nodes {
out.push(format!(
"[Tree truncated: the {ceiling}-node ceiling was reached — nodes beyond it were \
not read]"
));
}
if let Some(cut) = self.depth {
out.push(format!(
"[Tree truncated: {} {} at the {}-deep ceiling — {} children were not read]",
cut.count,
counted(cut.count, "node", "nodes"),
cut.limit,
counted(cut.count, "its", "their")
));
}
if let Some(cut) = self.children {
out.push(format!(
"[Tree truncated: {} {} {} more than {} children — {} children were not read]",
cut.count,
counted(cut.count, "node", "nodes"),
counted(cut.count, "claims", "claim"),
cut.limit,
counted(cut.count, "its", "their")
));
}
if let Some(cut) = self.children_cut {
out.push(format!(
"[Tree truncated: {} {} listed more than {} children — only the first {} were read]",
cut.count,
counted(cut.count, "node", "nodes"),
cut.limit,
cut.limit
));
}
if let Some(cut) = self.values {
out.push(format!(
"[Tree truncated: {} {} cut at {} chars]",
cut.count,
counted(cut.count, "value", "values"),
cut.limit
));
}
if values_cut > 0 {
out.push(format!(
"[Tree truncated: {values_cut} {} cut at {RENDER_VALUE_CHARS} chars]",
counted(values_cut, "value", "values")
));
}
if let Some(cut) = self.names {
out.push(format!(
"[Tree truncated: {} {} cut at {} chars]",
cut.count,
counted(cut.count, "name", "names"),
cut.limit
));
}
out
}
}
#[derive(Debug, Clone)]
pub(crate) struct Observation {
pub app_name: String,
pub window_title: Option<String>,
pub surface: SurfaceGeometry,
pub root: UiNode,
pub truncation: Truncation,
}
#[derive(Debug, Default)]
pub(crate) struct WindowListing {
pub windows: Vec<WindowInfo>,
pub unread: Vec<UnreadWindows>,
}
#[derive(Debug)]
pub(crate) struct UnreadWindows {
pub app_name: String,
pub reason: String,
}
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct AppInfo {
pub pid: Option<u32>,
pub name: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct WindowKey(String);
impl WindowKey {
pub(crate) fn new(raw: impl Into<String>) -> Self {
Self(raw.into())
}
pub(crate) fn as_str(&self) -> &str {
&self.0
}
}
#[derive(Debug, Clone, PartialEq)]
pub(crate) struct WindowInfo {
pub app_pid: Option<u32>,
pub app_name: String,
pub title: String,
pub key: WindowKey,
pub surface: SurfaceGeometry,
}
#[derive(Debug)]
pub(crate) struct Capture {
pub width: u32,
pub height: u32,
pub rgba: Vec<u8>,
claim: Claim,
}
impl Capture {
pub(crate) fn alloc(
budget: &CaptureBudget,
what: &str,
width: u32,
height: u32,
) -> anyhow::Result<Self> {
let claim = budget.charge(what, width, height)?;
let bytes = rgba_bytes(width, height);
let mut rgba = reserved_bytes(bytes, what, (width, height))?;
rgba.resize(bytes, 0);
Ok(Self {
width,
height,
rgba,
claim,
})
}
#[cfg_attr(not(any(target_os = "macos", test)), expect(dead_code))]
pub(crate) fn from_pixels(
budget: &CaptureBudget,
what: &str,
width: u32,
height: u32,
src: &[u8],
) -> anyhow::Result<Self> {
debug_assert_eq!(src.len(), rgba_bytes(width, height));
let claim = budget.charge(what, width, height)?;
let mut rgba = reserved_bytes(src.len(), what, (width, height))?;
rgba.extend_from_slice(src);
Ok(Self::adopt(claim, width, height, rgba))
}
pub(crate) fn adopt(claim: Claim, width: u32, height: u32, rgba: Vec<u8>) -> Self {
debug_assert_eq!(rgba.len(), rgba_bytes(width, height));
Self {
width,
height,
rgba,
claim,
}
}
pub(crate) fn into_parts(self) -> (u32, u32, Vec<u8>, Claim) {
(self.width, self.height, self.rgba, self.claim)
}
}
fn reserved_bytes(
bytes: usize,
what: &str,
(width, height): (u32, u32),
) -> anyhow::Result<Vec<u8>> {
let mut buffer = Vec::new();
buffer.try_reserve_exact(bytes).map_err(|e| {
taxonomy_error(
ERR_DEGRADED,
format!(
"{what} could not be allocated ({bytes} bytes, {width}x{height}: {e}) — the \
machine is out of memory; retry, or capture a smaller surface"
),
)
})?;
Ok(buffer)
}
#[expect(clippy::cast_possible_truncation)]
fn rgba_bytes(width: u32, height: u32) -> usize {
(u64::from(width) * u64::from(height) * 4) as usize
}
#[derive(Debug, Clone, Default)]
pub(crate) struct CaptureBudget(Arc<AtomicU64>);
impl CaptureBudget {
pub(crate) fn charge(&self, what: &str, width: u32, height: u32) -> anyhow::Result<Claim> {
let (width, height) = capture_size(what, width as usize, height as usize)?;
let pixels = u64::from(width) * u64::from(height);
let spent = self.0.fetch_add(pixels, Ordering::Relaxed) + pixels;
if spent > CAPTURE_BUDGET_PIXELS {
self.0.fetch_sub(pixels, Ordering::Relaxed);
return Err(ceiling_refused(what, width, height, spent - pixels));
}
Ok(Claim {
budget: Arc::clone(&self.0),
pixels,
})
}
#[cfg_attr(not(any(target_os = "linux", test)), expect(dead_code))]
pub(crate) fn remaining(&self) -> u64 {
CAPTURE_BUDGET_PIXELS.saturating_sub(self.0.load(Ordering::Relaxed))
}
#[cfg(test)]
pub(crate) fn held(&self) -> u64 {
self.0.load(Ordering::Relaxed)
}
}
#[derive(Debug)]
pub(crate) struct Claim {
budget: Arc<AtomicU64>,
pixels: u64,
}
impl Drop for Claim {
fn drop(&mut self) {
self.budget.fetch_sub(self.pixels, Ordering::Relaxed);
}
}
fn mib(pixels: u64) -> u64 {
pixels.saturating_mul(4) / (1024 * 1024)
}
#[derive(Debug, Clone)]
pub(crate) enum TargetSpec {
Focused,
App {
app_pid: Option<u32>,
app_name: String,
},
Window {
app_pid: Option<u32>,
app_name: String,
key: WindowKey,
label: String,
},
Screen,
}
impl TargetSpec {
#[must_use]
pub(crate) fn from_window(win: &WindowInfo) -> Self {
TargetSpec::Window {
app_pid: win.app_pid,
app_name: win.app_name.clone(),
key: win.key.clone(),
label: win.title.clone(),
}
}
#[must_use]
pub(crate) fn app(&self) -> Option<(Option<u32>, &str)> {
match self {
TargetSpec::App {
app_pid, app_name, ..
}
| TargetSpec::Window {
app_pid, app_name, ..
} => Some((*app_pid, app_name)),
TargetSpec::Focused | TargetSpec::Screen => None,
}
}
#[must_use]
pub(crate) fn describe(&self) -> String {
match self {
TargetSpec::Focused => "the focused window of the frontmost app".to_string(),
TargetSpec::App { app_name, .. } => {
format!("the focused window of {app_name}")
}
TargetSpec::Window {
app_name, label, ..
} => window_description(app_name, label),
TargetSpec::Screen => "the screen".to_string(),
}
}
}
#[must_use]
pub(crate) fn window_description(app_name: &str, title: &str) -> String {
match (app_name.is_empty(), title.is_empty()) {
(false, false) => format!("window \"{title}\" of {app_name}"),
(false, true) => format!("a window of {app_name}"),
(true, false) => format!("window \"{title}\""),
(true, true) => "the window".to_string(),
}
}
impl PartialEq for TargetSpec {
fn eq(&self, other: &Self) -> bool {
match (self, other) {
(TargetSpec::Focused, TargetSpec::Focused)
| (TargetSpec::Screen, TargetSpec::Screen) => true,
(
TargetSpec::App {
app_pid: a,
app_name: a_name,
},
TargetSpec::App {
app_pid: b,
app_name: b_name,
},
) => a == b && a_name == b_name,
(TargetSpec::Window { key: a_key, .. }, TargetSpec::Window { key: b_key, .. }) => {
a_key == b_key
}
_ => false,
}
}
}
#[derive(Debug, Clone, PartialEq)]
pub(crate) enum ElementAct {
Press,
SetValue { text: String },
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) enum MouseButton {
Left,
Right,
Middle,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
#[serde(rename_all = "snake_case")]
pub(crate) enum Modifier {
Cmd,
Ctrl,
Alt,
Shift,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize, Default)]
#[serde(rename_all = "snake_case")]
pub(crate) enum ScrollDirection {
Up,
#[default]
Down,
Left,
Right,
}
#[derive(Debug, Clone, PartialEq)]
pub(crate) enum RawInput {
Click {
point: (f64, f64),
button: MouseButton,
double: bool,
modifiers: Vec<Modifier>,
},
TypeText {
text: String,
},
KeyChord {
chord: String,
},
Scroll {
point: Option<(f64, f64)>,
direction: ScrollDirection,
amount: u32,
},
Drag {
from: (f64, f64),
to: (f64, f64),
},
}
#[cfg_attr(
not(any(target_os = "linux", target_os = "macos", target_os = "windows", test)),
expect(dead_code)
)]
pub(crate) fn pinned_input(target: &TargetSpec, input: &RawInput) -> bool {
matches!(target, TargetSpec::Window { .. } | TargetSpec::App { .. })
&& matches!(
input,
RawInput::TypeText { .. }
| RawInput::Drag { .. }
| RawInput::Scroll { point: Some(_), .. }
)
}
#[cfg_attr(not(any(target_os = "linux", target_os = "macos")), expect(dead_code))]
pub(crate) const RAISE_SETTLE_STEPS: usize = 4;
#[cfg_attr(not(any(target_os = "linux", target_os = "macos")), expect(dead_code))]
pub(crate) const RAISE_SETTLE_MS: u64 = 25;
#[cfg_attr(
not(any(target_os = "macos", target_os = "linux", target_os = "windows", test)),
expect(dead_code)
)]
pub(crate) fn parse_key_chord(chord: &str) -> anyhow::Result<(Vec<Modifier>, String)> {
let parts: Vec<&str> = chord.split('+').map(str::trim).collect();
if parts.is_empty() || parts.iter().any(|p| p.is_empty()) {
anyhow::bail!(
"invalid key chord '{chord}' — expected e.g. 'cmd+shift+t', 'return', 'ctrl+c'"
);
}
let mut mods = Vec::new();
let mut key: Option<String> = None;
for part in parts {
match part.to_ascii_lowercase().as_str() {
"cmd" | "command" => mods.push(Modifier::Cmd),
"ctrl" | "control" => mods.push(Modifier::Ctrl),
"alt" | "option" => mods.push(Modifier::Alt),
"shift" => mods.push(Modifier::Shift),
other => {
if key.is_some() {
anyhow::bail!(
"invalid key chord '{chord}' — too many key names; expected exactly one"
);
}
key = Some(other.to_string());
}
}
}
let key = key.ok_or_else(|| {
anyhow::anyhow!("invalid key chord '{chord}' — no key name given (only modifiers)")
})?;
Ok((mods, key))
}
pub(crate) fn check_arg_len(field: &str, value: &str) -> anyhow::Result<()> {
let len = value.chars().count();
if len > MAX_ARG_CHARS {
anyhow::bail!("{field} is too long: {len} chars (max {MAX_ARG_CHARS})");
}
Ok(())
}
pub(crate) fn check_scroll_amount(amount: u32) -> anyhow::Result<()> {
if !(1..=MAX_SCROLL_AMOUNT).contains(&amount) {
anyhow::bail!("scroll amount must be 1-{MAX_SCROLL_AMOUNT} ticks — got {amount}");
}
Ok(())
}
pub(crate) fn check_point(field: &str, nx: f64, ny: f64) -> anyhow::Result<()> {
if !nx.is_finite()
|| !ny.is_finite()
|| !(0.0..=NORMALIZED_MAX).contains(&nx)
|| !(0.0..=NORMALIZED_MAX).contains(&ny)
{
anyhow::bail!(
"{field} ({nx}, {ny}) is out of range — coordinates are normalized 0-{NORMALIZED_MAX}"
);
}
Ok(())
}
#[expect(clippy::float_cmp)] pub(crate) fn check_region(region: &Region) -> anyhow::Result<()> {
for (field, v) in [
("x0", region.x0),
("y0", region.y0),
("x1", region.x1),
("y1", region.y1),
] {
if !v.is_finite() || !(0.0..=NORMALIZED_MAX).contains(&v) {
anyhow::bail!(
"region corner {field} is {v} — corners are normalized 0-{NORMALIZED_MAX}"
);
}
}
if region.x0 == region.x1 || region.y0 == region.y1 {
anyhow::bail!(
"region [{}, {}, {}, {}] encloses no area — x0 and x1, and y0 and y1, must differ",
region.x0,
region.y0,
region.x1,
region.y1
);
}
Ok(())
}
#[cfg_attr(
not(any(target_os = "macos", target_os = "linux", target_os = "windows", test)),
expect(dead_code)
)]
pub(crate) fn check_raw_input(input: &RawInput) -> anyhow::Result<()> {
match input {
RawInput::Click { point, .. } => check_point("click point", point.0, point.1),
RawInput::Scroll { point, amount, .. } => {
check_scroll_amount(*amount)?;
point.map_or(Ok(()), |(x, y)| check_point("scroll anchor", x, y))
}
RawInput::Drag { from, to } => {
check_point("drag start", from.0, from.1)?;
check_point("drag end", to.0, to.1)
}
RawInput::TypeText { text } => check_arg_len("text", text),
RawInput::KeyChord { .. } => Ok(()),
}
}
#[cfg_attr(
not(any(target_os = "macos", target_os = "linux", target_os = "windows", test)),
expect(dead_code)
)]
pub(crate) fn normalized_to_surface(
nx: f64,
ny: f64,
geo: &SurfaceGeometry,
) -> anyhow::Result<(f64, f64)> {
check_point("coordinates", nx, ny)?;
Ok((
geo.x + nx / NORMALIZED_MAX * geo.width,
geo.y + ny / NORMALIZED_MAX * geo.height,
))
}
pub(crate) fn surface_to_normalized(
ax: f64,
ay: f64,
geo: &SurfaceGeometry,
) -> anyhow::Result<(f64, f64)> {
if !ax.is_finite() || !ay.is_finite() || geo.width <= 0.0 || geo.height <= 0.0 {
anyhow::bail!("invalid cursor position or surface geometry");
}
let nx = ((ax - geo.x) / geo.width * NORMALIZED_MAX).clamp(0.0, NORMALIZED_MAX);
let ny = ((ay - geo.y) / geo.height * NORMALIZED_MAX).clamp(0.0, NORMALIZED_MAX);
Ok((nx, ny))
}
#[must_use]
pub(crate) fn contains_point(geo: &SurfaceGeometry, ax: f64, ay: f64) -> bool {
ax >= geo.x && ax < geo.x + geo.width && ay >= geo.y && ay < geo.y + geo.height
}
#[expect(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
pub(crate) fn region_to_pixels(
region: &Region,
px_w: u32,
px_h: u32,
) -> anyhow::Result<(u32, u32, u32, u32)> {
check_region(region)?;
let (x0, x1) = (region.x0.min(region.x1), region.x0.max(region.x1));
let (y0, y1) = (region.y0.min(region.y1), region.y0.max(region.y1));
let px = |n: f64, total: u32| -> u32 {
((n / NORMALIZED_MAX * f64::from(total)).round() as u32).min(total)
};
let left = px(x0, px_w);
let right = px(x1, px_w);
let top = px(y0, px_h);
let bottom = px(y1, px_h);
let x = left.min(px_w.saturating_sub(1));
let y = top.min(px_h.saturating_sub(1));
let w = right
.saturating_sub(left)
.max(1)
.min(px_w.saturating_sub(x));
let h = bottom
.saturating_sub(top)
.max(1)
.min(px_h.saturating_sub(y));
Ok((x, y, w, h))
}
pub(crate) fn crop_rgba(
cap: &Capture,
rect: (u32, u32, u32, u32),
budget: &CaptureBudget,
what: &str,
) -> anyhow::Result<Capture> {
let (x, y, w, h) = rect;
if cap.width == 0 || cap.height == 0 {
anyhow::bail!("crop_rgba: source capture is empty");
}
let x = x.min(cap.width.saturating_sub(1));
let y = y.min(cap.height.saturating_sub(1));
let w = w.min(cap.width - x).max(1);
let h = h.min(cap.height - y).max(1);
let claim = budget.charge(what, w, h)?;
let stride = cap.width as usize * 4;
let row_bytes = w as usize * 4;
let mut rgba = reserved_bytes(row_bytes * h as usize, what, (w, h))?;
for index in 0..h as usize {
let start = (y as usize + index) * stride + x as usize * 4;
rgba.extend_from_slice(&cap.rgba[start..start + row_bytes]);
}
Ok(Capture::adopt(claim, w, h, rgba))
}
#[cfg_attr(
not(any(target_os = "macos", target_os = "linux", target_os = "windows", test)),
expect(dead_code)
)]
pub(crate) fn blit_rgba(
dest: &mut [u8],
dest_w: usize,
dest_h: usize,
src: &Capture,
dx: usize,
dy: usize,
) {
let (src_w, src_h) = (src.width as usize, src.height as usize);
if dx >= dest_w || dy >= dest_h {
return;
}
let row_copy = (dest_w - dx).min(src_w);
let rows = (dest_h - dy).min(src_h);
for r in 0..rows {
let src_start = r * src_w * 4;
let dst_start = (dy + r) * dest_w * 4 + dx * 4;
dest[dst_start..dst_start + row_copy * 4]
.copy_from_slice(&src.rgba[src_start..src_start + row_copy * 4]);
}
}
#[cfg_attr(not(any(target_os = "linux", test)), expect(dead_code))]
pub(crate) fn a11y_scale(a11y_screen: &SurfaceGeometry, pixels: (u32, u32)) -> Option<f64> {
let (px, py) = (f64::from(pixels.0), f64::from(pixels.1));
if a11y_screen.width <= 0.0 || a11y_screen.height <= 0.0 || px <= 0.0 || py <= 0.0 {
return None;
}
let (sx, sy) = (px / a11y_screen.width, py / a11y_screen.height);
if !sx.is_finite() || !sy.is_finite() || sx < 0.99 {
return None;
}
if (sx - sy).abs() > 0.02 * sx {
return None;
}
Some(f64::midpoint(sx, sy))
}
#[cfg_attr(not(any(target_os = "linux", test)), expect(dead_code))]
pub(crate) fn scale_geometry(geo: SurfaceGeometry, scale: f64) -> SurfaceGeometry {
SurfaceGeometry {
x: geo.x * scale,
y: geo.y * scale,
width: geo.width * scale,
height: geo.height * scale,
}
}
#[cfg_attr(not(any(target_os = "linux", test)), expect(dead_code))] pub(crate) fn x11_pixels_to_rgba(
data: &[u8],
width: u32,
height: u32,
depth: u8,
lsb_first: bool,
budget: &CaptureBudget,
) -> Result<Capture, anyhow::Error> {
if depth != 24 && depth != 32 {
return Err(taxonomy_error(
ERR_DEGRADED,
format!("unsupported X11 depth {depth} (need 24/32)"),
));
}
let mut cap = Capture::alloc(budget, "a display tile", width, height)?;
if data.len() < cap.rgba.len() {
anyhow::bail!("short X11 image body: {} < {}", data.len(), cap.rgba.len());
}
for (out, px) in cap
.rgba
.as_chunks_mut::<4>()
.0
.iter_mut()
.zip(data.as_chunks::<4>().0)
{
*out = if lsb_first {
[px[2], px[1], px[0], 0xff]
} else {
[px[1], px[2], px[3], 0xff]
};
}
Ok(cap)
}
#[cfg_attr(not(any(target_os = "linux", test)), expect(dead_code))] pub(crate) fn portal_uri_to_path(uri: &str) -> Result<PathBuf, anyhow::Error> {
let stripped = uri.strip_prefix("file://").ok_or_else(|| {
taxonomy_error(ERR_DEGRADED, format!("portal uri is not a file uri: {uri}"))
})?;
let path = if stripped.starts_with('/') {
stripped
} else {
match stripped.find('/') {
Some(idx) => {
let (host, path) = stripped.split_at(idx);
if host != "localhost" {
return Err(taxonomy_error(
ERR_DEGRADED,
"portal uri has unsupported host",
));
}
path
}
None => {
return Err(taxonomy_error(
ERR_DEGRADED,
"portal uri has unsupported host",
));
}
}
};
let bytes = path.as_bytes();
let mut out = Vec::with_capacity(bytes.len());
let mut i = 0;
let bad_escape = || {
taxonomy_error(
ERR_DEGRADED,
format!("bad percent-escape in portal uri: {uri}"),
)
};
while i < bytes.len() {
if bytes[i] == b'%' {
let hex = bytes.get(i + 1..i + 3).ok_or_else(&bad_escape)?;
let hex = std::str::from_utf8(hex).map_err(|_| bad_escape())?;
let byte = u8::from_str_radix(hex, 16).map_err(|_| bad_escape())?;
out.push(byte);
i += 3;
} else {
out.push(bytes[i]);
i += 1;
}
}
let s = String::from_utf8(out).map_err(|_| {
taxonomy_error(
ERR_DEGRADED,
format!("portal uri is not valid UTF-8: {uri}"),
)
})?;
Ok(PathBuf::from(s))
}
#[cfg_attr(
not(any(target_os = "linux", target_os = "windows", test)),
expect(dead_code)
)] #[expect(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
pub(crate) fn window_capture_rect(
win: &SurfaceGeometry,
screen: &SurfaceGeometry,
) -> Result<(u32, u32, u32, u32), anyhow::Error> {
if win.width <= 0.0 || win.height <= 0.0 {
return Err(taxonomy_error(
ERR_DEGRADED,
"the window has no visible area — it is minimized or off-screen; restore it and retry",
));
}
let x0 = (win.x - screen.x).max(0.0);
let y0 = (win.y - screen.y).max(0.0);
let x1 = (win.x + win.width - screen.x).min(screen.width);
let y1 = (win.y + win.height - screen.y).min(screen.height);
if x1 <= x0 || y1 <= y0 {
return Err(taxonomy_error(
ERR_DEGRADED,
"the window lies entirely outside the captured screen — it is on another display or \
minimized; screenshot with target: \"screen\" shows the desktop instead",
));
}
Ok((
x0.round() as u32,
y0.round() as u32,
(x1 - x0).round().max(1.0) as u32,
(y1 - y0).round().max(1.0) as u32,
))
}
struct ObservationCache {
target: TargetSpec,
refs: HashMap<String, Locator>,
}
static LAST_OBSERVATION: LazyLock<Mutex<HashMap<String, ObservationCache>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
static LAST_SURFACE: LazyLock<Mutex<HashMap<String, TargetSpec>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
#[must_use]
pub(crate) fn agent_key() -> String {
crate::agent::CURRENT_TOOL_AGENT_ID
.try_with(Clone::clone)
.unwrap_or(None)
.unwrap_or_else(|| crate::agent::role::DIAGNOSTICS_ROLE.to_string())
}
pub(crate) fn store_observation(target: TargetSpec, obs: &Observation) -> usize {
let mut refs = HashMap::new();
let mut counter = 0usize;
let mut path = Vec::new();
assign_refs(&obs.root, &mut path, &mut refs, &mut counter);
let cache = ObservationCache { target, refs };
let key = agent_key();
let mut guard = LAST_OBSERVATION.lock().unwrap_poison();
guard.insert(key, cache);
counter
}
fn assign_refs(
node: &UiNode,
path: &mut Vec<usize>,
refs: &mut HashMap<String, Locator>,
counter: &mut usize,
) {
*counter += 1;
refs.insert(
format!("e{counter}"),
Locator {
path: path.clone(),
role: node.role.clone(),
name: node.name.clone(),
},
);
for (idx, child) in node.children.iter().enumerate() {
path.push(idx);
assign_refs(child, path, refs, counter);
path.pop();
}
}
pub(crate) fn resolve_ref(reference: &str) -> anyhow::Result<(Locator, TargetSpec)> {
let key = agent_key();
let guard = LAST_OBSERVATION.lock().unwrap_poison();
let cache = guard.get(&key).ok_or_else(|| {
taxonomy_error(
ERR_STALE_ELEMENT,
format!("ref {reference} is not resolvable — no observation cached; run observe first"),
)
})?;
let locator = cache.refs.get(reference).ok_or_else(|| {
taxonomy_error(
ERR_STALE_ELEMENT,
format!("ref {reference} is not resolvable — the observed tree changed or expired; run observe again"),
)
})?;
Ok((locator.clone(), cache.target.clone()))
}
#[must_use]
pub(crate) fn last_surface() -> Option<TargetSpec> {
let key = agent_key();
let guard = LAST_SURFACE.lock().unwrap_poison();
guard.get(&key).cloned()
}
pub(crate) fn set_last_surface(target: TargetSpec) {
let key = agent_key();
let mut guard = LAST_SURFACE.lock().unwrap_poison();
guard.insert(key, target);
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum IdClass {
App,
Window,
}
impl IdClass {
fn prefix(self) -> char {
match self {
Self::App => 'a',
Self::Window => 'w',
}
}
}
pub(crate) const MAX_STORED_TARGETS: usize = 512;
#[derive(Default)]
struct TargetRegistry {
entries: Vec<(String, TargetSpec)>,
next_app: usize,
next_window: usize,
}
impl TargetRegistry {
fn mint(&mut self, class: IdClass) -> String {
let counter = match class {
IdClass::App => &mut self.next_app,
IdClass::Window => &mut self.next_window,
};
*counter += 1;
format!("{}{counter}", class.prefix())
}
fn position_of(&self, surface: &TargetSpec) -> Option<usize> {
self.entries
.iter()
.position(|(_, stored)| stored == surface)
}
fn insert(&mut self, id: String, surface: TargetSpec) {
self.entries.push((id, surface));
if self.entries.len() > MAX_STORED_TARGETS {
self.entries.remove(0);
}
}
}
static TARGETS: LazyLock<Mutex<HashMap<String, TargetRegistry>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
pub(crate) fn store_app_targets(apps: &[TargetSpec]) -> Vec<String> {
store_targets(IdClass::App, apps)
}
pub(crate) fn store_window_targets(windows: &[TargetSpec]) -> Vec<String> {
store_targets(IdClass::Window, windows)
}
fn store_targets(class: IdClass, surfaces: &[TargetSpec]) -> Vec<String> {
let key = agent_key();
let mut guard = TARGETS.lock().unwrap_poison();
let registry = guard.entry(key).or_default();
let mut ids = Vec::with_capacity(surfaces.len());
for surface in surfaces {
if let Some(position) = registry.position_of(surface) {
let (id, stored) = &mut registry.entries[position];
ids.push(id.clone());
*stored = surface.clone();
continue;
}
let id = registry.mint(class);
registry.insert(id.clone(), surface.clone());
ids.push(id);
}
ids
}
pub(crate) fn get_target(id: &str) -> Option<TargetSpec> {
let key = agent_key();
let guard = TARGETS.lock().unwrap_poison();
let registry = guard.get(&key)?;
registry
.entries
.iter()
.find(|(stored, _)| stored == id)
.map(|(_, surface)| surface.clone())
}
pub(crate) fn resolve_target(target: Option<&str>) -> anyhow::Result<TargetSpec> {
let Some(target) = target else {
return Ok(TargetSpec::Focused);
};
if target == "screen" {
return Ok(TargetSpec::Screen);
}
get_target(target).ok_or_else(|| {
anyhow::anyhow!(
"unknown target '{target}' — it is not a stored app/window id; enumerate targets \
with 'apps' (a1-style) or 'windows' (w1-style) first"
)
})
}
pub(crate) fn clear_agent_state(key: &str) {
{
let mut guard = LAST_OBSERVATION.lock().unwrap_poison();
guard.remove(key);
}
{
let mut guard = LAST_SURFACE.lock().unwrap_poison();
guard.remove(key);
}
let mut guard = TARGETS.lock().unwrap_poison();
guard.remove(key);
}
fn normalized_role(role: &str) -> &str {
role.strip_prefix("AX").unwrap_or(role)
}
fn is_interactive(node: &UiNode) -> bool {
if !node.actions.is_empty() {
return true;
}
let role = normalized_role(&node.role);
INTERACTIVE_ROLES
.iter()
.any(|r| r.eq_ignore_ascii_case(role))
}
#[must_use]
pub(crate) fn count_interactive(obs: &Observation) -> usize {
count_stats(&obs.root).1
}
fn count_stats(node: &UiNode) -> (usize, usize) {
let mut total = 1usize;
let mut interactive = usize::from(is_interactive(node));
for child in &node.children {
let (t, i) = count_stats(child);
total += t;
interactive += i;
}
(total, interactive)
}
#[must_use]
pub(crate) fn is_ax_thin(obs: &Observation) -> bool {
count_interactive(obs) < AX_THIN_INTERACTIVE_ELEMENTS
}
#[must_use]
pub(crate) fn render_tree(obs: &Observation) -> String {
let (node_count, interactive_count) = count_stats(&obs.root);
let mut tree = String::new();
let mut counter = 0usize;
let mut values_cut = 0usize;
render_node(&obs.root, 0, &mut tree, &mut counter, &mut values_cut);
let surf = &obs.surface;
let title = obs.window_title.as_deref().unwrap_or("");
let mut out = String::new();
let _ = writeln!(
out,
"[Surface: app=\"{}\" window=\"{}\" {}x{} @ ({}, {}) — {} nodes, {} interactive]",
obs.app_name, title, surf.width, surf.height, surf.x, surf.y, node_count, interactive_count
);
for statement in obs.truncation.statements(values_cut) {
out.push_str(&statement);
out.push('\n');
}
out.push_str(&tree);
out
}
fn render_node(
node: &UiNode,
depth: usize,
out: &mut String,
counter: &mut usize,
values_cut: &mut usize,
) {
*counter += 1;
let indent = " ".repeat(depth);
let mut line = format!("{indent}[e{counter}] {}", normalized_role(&node.role));
if let Some(name) = &node.name {
let _ = write!(line, " \"{name}\"");
}
if !node.actions.is_empty() {
let _ = write!(line, " actions=[{}]", node.actions.join(","));
}
if let Some((x, y, _, _)) = node.frame {
let _ = write!(line, " @ ({x}, {y})");
}
if let Some(value) = &node.value {
let v = crate::util::truncate(value, RENDER_VALUE_CHARS);
if v != *value {
*values_cut += 1;
}
let _ = write!(line, " value=\"{v}\"");
}
if node.focused {
line.push_str(" focused");
}
out.push_str(&line);
out.push('\n');
for child in &node.children {
render_node(child, depth + 1, out, counter, values_cut);
}
}
#[cfg_attr(
not(any(target_os = "macos", target_os = "linux", target_os = "windows")),
expect(dead_code)
)]
#[must_use]
pub(crate) fn pre_order_index(root: &UiNode, target: &UiNode) -> usize {
fn rec(node: &UiNode, target: &UiNode, counter: &mut usize, found: &mut Option<usize>) {
if found.is_some() {
return;
}
if std::ptr::eq(node, target) {
*found = Some(*counter);
}
*counter += 1;
for c in &node.children {
rec(c, target, counter, found);
}
}
let (mut counter, mut found) = (0, None);
rec(root, target, &mut counter, &mut found);
found.unwrap_or(0)
}
#[cfg_attr(
not(any(target_os = "macos", target_os = "linux", target_os = "windows")),
expect(dead_code)
)]
pub(crate) enum LocatorMatch<'a> {
Path(&'a UiNode),
Unique(&'a UiNode),
Ambiguous,
NotFound,
}
#[cfg_attr(
not(any(target_os = "macos", target_os = "linux", target_os = "windows", test)),
expect(dead_code)
)]
#[must_use]
pub(crate) fn resolve_locator<'a>(root: &'a UiNode, locator: &Locator) -> LocatorMatch<'a> {
let mut node = root;
for idx in &locator.path {
let Some(child) = node.children.get(*idx) else {
return search_locator(root, locator);
};
node = child;
}
if node_matches(node, locator) {
return LocatorMatch::Path(node);
}
search_locator(root, locator)
}
#[cfg_attr(
not(any(target_os = "macos", target_os = "linux", target_os = "windows")),
expect(dead_code)
)]
pub(crate) fn resolve_locator_checked<'a>(
root: &'a UiNode,
locator: &Locator,
) -> Result<&'a UiNode, anyhow::Error> {
match resolve_locator(root, locator) {
LocatorMatch::Path(node) | LocatorMatch::Unique(node) => Ok(node),
LocatorMatch::Ambiguous => Err(taxonomy_error(
ERR_AMBIGUOUS_LOCATOR,
"locator matches multiple elements — re-observe and pick a more specific ref",
)),
LocatorMatch::NotFound => Err(taxonomy_error(
ERR_NOT_MATCHED,
"element no longer matches its locator — re-observe",
)),
}
}
#[cfg_attr(
not(any(target_os = "macos", target_os = "linux", target_os = "windows")),
expect(dead_code)
)]
pub(crate) fn screen_observe_error() -> anyhow::Error {
taxonomy_error(
ERR_UNSUPPORTED,
if super::capture_known_available() {
"observe targets an accessibility surface — use screenshot for the screen"
} else {
"observe targets an accessibility surface — use apps/windows to pick one"
},
)
}
#[cfg_attr(
not(any(target_os = "macos", target_os = "linux", target_os = "windows")),
expect(dead_code)
)]
pub(crate) fn screen_act_error() -> anyhow::Error {
taxonomy_error(
ERR_UNSUPPORTED,
if super::capture_known_available() {
"act targets a window — use apps/windows to pick one (use screenshot for the screen)"
} else {
"act targets a window — use apps/windows to pick one"
},
)
}
#[cfg_attr(
not(any(target_os = "macos", target_os = "linux", target_os = "windows", test)),
expect(dead_code)
)]
fn node_matches(node: &UiNode, locator: &Locator) -> bool {
let role_ok = normalized_role(&node.role).eq_ignore_ascii_case(normalized_role(&locator.role));
let name_ok = match (&locator.name, &node.name) {
(None, _) => true,
(Some(l), Some(n)) => l == n,
(Some(_), None) => false,
};
role_ok && name_ok
}
#[cfg_attr(
not(any(target_os = "macos", target_os = "linux", target_os = "windows", test)),
expect(dead_code)
)]
fn search_locator<'a>(root: &'a UiNode, locator: &Locator) -> LocatorMatch<'a> {
let mut found: Option<&'a UiNode> = None;
let mut count = 0usize;
let mut stack = vec![root];
while let Some(node) = stack.pop() {
if node_matches(node, locator) {
count += 1;
if count > 1 {
return LocatorMatch::Ambiguous;
}
found = Some(node);
}
for child in node.children.iter().rev() {
stack.push(child);
}
}
match found {
Some(node) => LocatorMatch::Unique(node),
None => LocatorMatch::NotFound,
}
}
#[async_trait::async_trait]
pub(crate) trait Backend: Send + Sync {
fn is_supported(&self) -> bool;
async fn desktop_unavailable(&self) -> Option<String>;
async fn capture_available(&self) -> bool;
fn capture_unavailable_error(&self) -> anyhow::Error;
async fn list_apps(&self) -> anyhow::Result<Vec<AppInfo>>;
async fn list_windows(&self, app: Option<&AppInfo>) -> anyhow::Result<WindowListing>;
async fn focused_window(&self) -> anyhow::Result<WindowInfo>;
async fn surface_geometry(&self, target: &TargetSpec) -> anyhow::Result<SurfaceGeometry>;
async fn observe(&self, target: &TargetSpec) -> anyhow::Result<Observation>;
async fn act_on_element(
&self,
target: &TargetSpec,
locator: &Locator,
act: ElementAct,
) -> anyhow::Result<Impact>;
async fn raw_input(&self, target: &TargetSpec, input: RawInput) -> anyhow::Result<Impact>;
async fn cursor_position(&self) -> anyhow::Result<(f64, f64)>;
async fn capture(&self, target: &TargetSpec, budget: &CaptureBudget)
-> anyhow::Result<Capture>;
}
#[cfg(test)]
mod tests {
use super::*;
fn surface(x: f64, y: f64, w: f64, h: f64) -> SurfaceGeometry {
SurfaceGeometry {
x,
y,
width: w,
height: h,
}
}
fn node(role: &str, name: Option<&str>) -> UiNode {
UiNode {
role: role.to_string(),
name: name.map(String::from),
value: None,
frame: None,
actions: Vec::new(),
focused: false,
children: Vec::new(),
}
}
fn leaf(role: &str, name: &str) -> UiNode {
let mut n = node(role, Some(name));
n.actions = vec!["press".to_string()];
n
}
fn sample_observation() -> Observation {
let root = UiNode {
role: "AXWindow".to_string(),
name: Some("Sample".to_string()),
value: None,
frame: Some((0.0, 0.0, 800.0, 600.0)),
actions: Vec::new(),
focused: true,
children: vec![
leaf("AXButton", "OK"),
UiNode {
role: "AXGroup".to_string(),
name: None,
value: None,
frame: None,
actions: Vec::new(),
focused: false,
children: vec![leaf("AXButton", "Cancel")],
},
],
};
Observation {
app_name: "TestApp".to_string(),
window_title: Some("Sample".to_string()),
surface: surface(0.0, 0.0, 800.0, 600.0),
root,
truncation: Truncation::default(),
}
}
#[test]
fn normalized_to_surface_maps_and_validates() {
let geo = surface(100.0, 200.0, 800.0, 600.0);
let (sx, sy) = normalized_to_surface(500.0, 250.0, &geo).unwrap();
assert!((sx - 500.0).abs() < 1e-9, "x = {sx}");
assert!((sy - 350.0).abs() < 1e-9, "y = {sy}");
assert!(normalized_to_surface(-1.0, 0.0, &geo).is_err());
assert!(normalized_to_surface(0.0, 1001.0, &geo).is_err());
assert!(normalized_to_surface(f64::NAN, 0.0, &geo).is_err());
}
#[test]
fn normalized_to_surface_handles_negative_origins() {
let geo = surface(0.0, -100.0, 3840.0, 1180.0);
let (px, py) = normalized_to_surface(500.0, 500.0, &geo).unwrap();
assert!((px - 1920.0).abs() < 1e-9, "x = {px}");
assert!((py - 490.0).abs() < 1e-9, "y = {py}");
}
#[test]
fn blit_rgba_clips_and_copies() {
let mut dest = vec![0u8; 4 * 4 * 4];
let mut src = Capture::alloc(&CaptureBudget::default(), "a test frame", 2, 2).unwrap();
src.rgba
.copy_from_slice(&[1, 2, 3, 255, 4, 5, 6, 255, 7, 8, 9, 255, 10, 11, 12, 255]);
blit_rgba(&mut dest, 4, 4, &src, 1, 1);
assert_eq!(&dest[4 * 4 + 4..4 * 4 + 8], &[1, 2, 3, 255]);
assert_eq!(&dest[4 * 4 + 8..4 * 4 + 12], &[4, 5, 6, 255]);
}
#[test]
fn raw_input_bounds_are_checked_at_the_backend_entry() {
let refused = [
RawInput::Click {
point: (NORMALIZED_MAX + 1.0, 0.0),
button: MouseButton::Left,
double: false,
modifiers: Vec::new(),
},
RawInput::Scroll {
point: None,
direction: ScrollDirection::Down,
amount: MAX_SCROLL_AMOUNT + 1,
},
RawInput::Scroll {
point: None,
direction: ScrollDirection::Down,
amount: 0,
},
RawInput::Drag {
from: (0.0, 0.0),
to: (0.0, -1.0),
},
RawInput::TypeText {
text: "t".repeat(MAX_ARG_CHARS + 1),
},
];
for input in refused {
assert!(
check_raw_input(&input).is_err(),
"{input:?} must be refused"
);
}
assert!(
check_raw_input(&RawInput::Scroll {
point: Some((1.0, NORMALIZED_MAX)),
direction: ScrollDirection::Up,
amount: MAX_SCROLL_AMOUNT,
})
.is_ok()
);
}
#[test]
fn region_to_pixels_maps_orders_and_clamps() {
let region = Region {
x0: 200.0,
y0: 100.0,
x1: 800.0,
y1: 500.0,
};
assert_eq!(
region_to_pixels(®ion, 1000, 1000).unwrap(),
(200, 100, 600, 400)
);
let rev = Region {
x0: 800.0,
y0: 500.0,
x1: 200.0,
y1: 100.0,
};
assert_eq!(
region_to_pixels(&rev, 1000, 1000).unwrap(),
(200, 100, 600, 400)
);
let bad = Region {
x0: 1100.0,
y0: 0.0,
x1: 200.0,
y1: 100.0,
};
assert!(region_to_pixels(&bad, 1000, 1000).is_err());
let flat = Region {
x0: 100.0,
y0: 100.0,
x1: 900.0,
y1: 100.0,
};
assert!(region_to_pixels(&flat, 1000, 1000).is_err());
assert!(
serde_json::from_value::<Region>(
serde_json::json!({"x0": 0, "y0": 0, "x1": 1, "y1": 1, "x2": 2})
)
.is_err()
);
assert!(
serde_json::from_value::<Region>(
serde_json::json!({"x0": 0, "y0": 0, "x1": 1, "y1": 1})
)
.is_ok()
);
assert!(serde_json::from_value::<Region>(serde_json::json!([0, 0, 1, 1, 2])).is_err());
assert!(serde_json::from_value::<Region>(serde_json::json!([0, 0, 1, 1])).is_ok());
let big = Region {
x0: 500.0,
y0: 500.0,
x1: 1000.0,
y1: 1000.0,
};
let (x, y, w, h) = region_to_pixels(&big, 200, 200).unwrap();
assert!(x + w <= 200 && y + h <= 200);
assert!(w >= 1 && h >= 1);
}
#[test]
fn crop_rgba_extracts_sub_rect() {
let budget = CaptureBudget::default();
let mut cap = Capture::alloc(&budget, "a test frame", 4, 2).unwrap();
for (i, (r, c)) in [
(0, 0),
(0, 1),
(0, 2),
(0, 3),
(1, 0),
(1, 1),
(1, 2),
(1, 3),
]
.into_iter()
.enumerate()
{
cap.rgba[i * 4..(i + 1) * 4].copy_from_slice(&[r * 40, c * 10, 0, 255]);
}
let crop = crop_rgba(&cap, (1, 0, 2, 2), &budget, "the zoom crop").unwrap();
assert_eq!(crop.width, 2);
assert_eq!(crop.height, 2);
assert_eq!(crop.rgba.len(), 2 * 2 * 4);
assert_eq!(budget.held(), 4 * 2 + 2 * 2);
assert_eq!(&crop.rgba[..4], &[0, 10, 0, 255]);
let clamp = crop_rgba(&cap, (10, 10, 50, 50), &budget, "the zoom crop").unwrap();
assert!(clamp.width >= 1 && clamp.height >= 1);
}
#[test]
fn x11_pixels_convert_both_byte_orders() {
let budget = CaptureBudget::default();
let tile =
x11_pixels_to_rgba(&[10, 20, 30, 0, 40, 50, 60, 0], 2, 1, 24, true, &budget).unwrap();
assert_eq!(tile.rgba, vec![30, 20, 10, 255, 60, 50, 40, 255]);
let tile = x11_pixels_to_rgba(&[0, 30, 20, 10], 1, 1, 32, false, &budget).unwrap();
assert_eq!(tile.rgba, vec![30, 20, 10, 255]);
let err = x11_pixels_to_rgba(&[0; 4], 1, 1, 16, true, &budget).unwrap_err();
assert!(err.to_string().contains("16"), "{err:?}");
}
#[test]
fn portal_uri_decodes_to_path() {
assert_eq!(
portal_uri_to_path("file:///tmp/shot%20one.png").unwrap(),
PathBuf::from("/tmp/shot one.png")
);
assert_eq!(
portal_uri_to_path("file://localhost/tmp/shot%20one.png").unwrap(),
PathBuf::from("/tmp/shot one.png")
);
assert!(
portal_uri_to_path("file:///tmp/a+b.png")
.unwrap()
.to_str()
.is_some_and(|p| p.contains('+'))
);
assert!(portal_uri_to_path("https://x/y.png").is_err());
assert!(portal_uri_to_path("file://otherhost/tmp/a.png").is_err());
assert!(portal_uri_to_path("file://localhost").is_err());
assert!(portal_uri_to_path("file:///tmp/a%2").is_err());
assert!(portal_uri_to_path("file:///tmp/a%").is_err());
assert!(portal_uri_to_path("file:///tmp/a%zz.png").is_err());
}
#[test]
fn window_rect_intersects_screen() {
let screen = surface(-1920.0, 0.0, 3840.0, 1080.0);
let win = surface(-1920.0, 100.0, 800.0, 600.0);
assert_eq!(
window_capture_rect(&win, &screen).unwrap(),
(0, 100, 800, 600)
);
let partial = surface(1500.0, 900.0, 800.0, 400.0);
assert_eq!(
window_capture_rect(&partial, &screen).unwrap(),
(3420, 900, 420, 180)
);
let off = surface(5000.0, 0.0, 100.0, 100.0);
let err = window_capture_rect(&off, &screen).unwrap_err();
assert!(
err.to_string().contains("outside the captured screen"),
"{err:?}"
);
assert!(window_capture_rect(&surface(0.0, 0.0, 0.0, 0.0), &screen).is_err());
}
#[test]
fn render_tree_assigns_pre_order_refs_and_flags() {
let obs = sample_observation();
let text = render_tree(&obs);
let lines: Vec<&str> = text.lines().collect();
assert!(
lines[0].starts_with(
"[Surface: app=\"TestApp\" window=\"Sample\" 800x600 @ (0, 0) — 4 nodes, 2 interactive]"
),
"got {}",
lines[0]
);
assert!(
lines[1].starts_with("[e1] Window \"Sample\""),
"got {}",
lines[1]
);
assert!(
lines[2].starts_with(" [e2] Button \"OK\""),
"got {}",
lines[2]
);
assert!(lines[2].contains("actions=[press]"), "got {}", lines[2]);
assert!(lines[3].starts_with(" [e3] Group"), "got {}", lines[3]);
assert!(
lines[4].starts_with(" [e4] Button \"Cancel\""),
"got {}",
lines[4]
);
assert_eq!(count_interactive(&obs), 2);
assert!(is_ax_thin(&obs));
}
#[test]
fn render_tree_states_every_ceiling_it_hit() {
let long = "x".repeat(200);
let root = UiNode {
value: Some(long.clone()),
..node("AXTextField", Some("field"))
};
let obs = Observation {
app_name: "A".to_string(),
window_title: None,
surface: surface(0.0, 0.0, 10.0, 10.0),
root,
truncation: Truncation {
nodes: Some(MAX_TREE_NODES),
depth: Some(Cut {
count: 2,
limit: 25,
}),
values: Some(Cut {
count: 1,
limit: MAX_VALUE_CHARS,
}),
names: Some(Cut {
count: 3,
limit: MAX_NAME_CHARS,
}),
children: Some(Cut {
count: 4,
limit: MAX_CHILDREN,
}),
children_cut: Some(Cut {
count: 1,
limit: MAX_CHILDREN,
}),
},
};
let text = render_tree(&obs);
assert!(text.contains("value=\""), "value should render");
assert!(!text.contains(&long), "long value must be truncated");
let header_end = text
.lines()
.position(|line| line.starts_with("[e"))
.expect("the tree renders under the header");
let header = text.lines().take(header_end).collect::<Vec<_>>().join("\n");
assert!(
header.contains(&format!("the {MAX_TREE_NODES}-node ceiling")),
"{header}"
);
assert!(
header.contains("2 nodes at the 25-deep ceiling"),
"{header}"
);
assert!(
header.contains(&format!("1 value cut at {RENDER_VALUE_CHARS} chars")),
"{header}"
);
assert!(
header.contains(&format!("1 value cut at {MAX_VALUE_CHARS} chars")),
"{header}"
);
assert!(
header.contains(&format!("3 names cut at {MAX_NAME_CHARS} chars")),
"{header}"
);
assert!(
header.contains(&format!("4 nodes claim more than {MAX_CHILDREN} children")),
"{header}"
);
assert!(
header.contains(&format!(
"1 node listed more than {MAX_CHILDREN} children — only the first {MAX_CHILDREN} \
were read"
)),
"{header}"
);
}
#[test]
fn a_single_node_at_a_ceiling_reads_as_one_node() {
let mut truncation = Truncation::default();
truncation.cut_depth(25);
assert_eq!(
truncation.statements(0),
["[Tree truncated: 1 node at the 25-deep ceiling — its children were not read]"]
);
}
#[test]
fn keep_value_records_only_what_it_cut() {
let mut truncation = Truncation::default();
let fits = "z".repeat(MAX_VALUE_CHARS);
assert_eq!(truncation.keep_value(&fits), fits);
assert!(truncation.values.is_none(), "nothing was cut");
let long = "z".repeat(MAX_VALUE_CHARS + 5);
let kept = truncation.keep_value(&long);
assert_eq!(kept.chars().count(), MAX_VALUE_CHARS);
assert!(kept.starts_with(&"z".repeat(MAX_VALUE_CHARS - 1)), "{kept}");
assert!(kept.ends_with('…'), "the cut is marked: {kept}");
let cut = truncation.values.expect("the cut is recorded");
assert_eq!((cut.count, cut.limit), (1, MAX_VALUE_CHARS));
let name = truncation.keep_name(&"n".repeat(MAX_NAME_CHARS + 1));
assert_eq!(name.chars().count(), MAX_NAME_CHARS);
assert!(
truncation
.names
.is_some_and(|cut| cut.limit == MAX_NAME_CHARS),
"the name cut is recorded"
);
assert_eq!(truncation.values.expect("one value was cut").count, 1);
let label = cap_label(&"l".repeat(MAX_NAME_CHARS + 1));
assert_eq!(label.chars().count(), MAX_NAME_CHARS);
assert!(label.ends_with('…'), "{label}");
}
#[test]
fn capture_size_refuses_what_a_capture_cannot_express() {
assert_eq!(
capture_size("a window still", 800, 600).unwrap(),
(800, 600)
);
assert!(capture_size("a window still", 0, 600).is_err());
let err = capture_size("a display frame", usize::MAX, 1).unwrap_err();
assert!(err.to_string().starts_with(ERR_DEGRADED), "{err}");
}
#[test]
fn capture_budget_refuses_a_surface_it_cannot_hold() {
let budget = CaptureBudget::default();
let err = budget
.charge("the screen composite", u32::MAX, u32::MAX)
.unwrap_err();
let text = err.to_string();
assert!(text.starts_with(ERR_DEGRADED), "{text}");
assert!(text.contains("the screen composite"), "{text}");
assert!(
text.contains("already holds 0 MiB of its 512 MiB"),
"{text}"
);
assert!(budget.charge("a display frame", 3840, 2160).is_ok());
let empty = budget.charge("a window still", 0, 1080).unwrap_err();
assert!(empty.to_string().contains("no capture area"), "{empty}");
let ceiling = budget
.charge("the screen composite", u32::MAX, u32::MAX)
.unwrap_err();
assert!(is_ceiling_refusal(&ceiling), "{ceiling}");
let other = taxonomy_error(ERR_DEGRADED, "a frame could not be allocated");
assert!(!is_ceiling_refusal(&other));
}
#[test]
fn capture_alloc_and_from_pixels_take_their_pixels_from_the_budget() {
let budget = CaptureBudget::default();
let blank = Capture::alloc(&budget, "a test frame", 4, 2).unwrap();
assert_eq!(blank.rgba.len(), 4 * 2 * 4);
assert!(
blank.rgba.iter().all(|byte| *byte == 0),
"the buffer is blank"
);
assert_eq!(budget.held(), 8);
let pixels = vec![7u8; 4];
let copied = Capture::from_pixels(&budget, "a small frame", 1, 1, &pixels).unwrap();
assert_eq!(copied.rgba, pixels, "the pixels are taken over as they are");
assert_eq!(
budget.held(),
8 + 1,
"the claim is the pixels, not the bytes"
);
}
#[test]
fn capture_budget_follows_the_pixels_a_call_holds() {
let budget = CaptureBudget::default();
let half = u32::try_from(CAPTURE_BUDGET_PIXELS / 2).unwrap();
let composite = budget.charge("the composite", half, 1).unwrap();
let frame = budget.charge("a display frame", half, 1).unwrap();
assert_eq!(budget.held(), CAPTURE_BUDGET_PIXELS);
let err = budget.charge("the zoom crop", half, 1).unwrap_err();
assert!(err.to_string().contains("the zoom crop"), "{err}");
assert!(err.to_string().contains("already holds 512 MiB"), "{err}");
drop(frame);
assert_eq!(budget.held(), u64::from(half));
drop(composite);
assert_eq!(budget.held(), 0);
assert!(budget.charge("the zoom crop", half, 1).is_ok());
let claim = budget
.charge("a decoded image", 4, 4)
.expect("the allowance is free again");
assert_eq!(budget.remaining(), CAPTURE_BUDGET_PIXELS - 16);
let adopted = Capture::adopt(claim, 4, 4, vec![0; 4 * 4 * 4]);
assert_eq!(adopted.rgba.len(), 4 * 4 * 4);
drop(adopted);
assert_eq!(budget.remaining(), CAPTURE_BUDGET_PIXELS);
}
#[test]
fn impact_text_states_the_middle_outcome_without_advice_to_retry() {
assert_eq!(impact_text("clicked".to_string(), Impact::Done), "clicked");
assert_eq!(
impact_text(
"clicked".to_string(),
Impact::DoneWith("ctrl may still be held down".to_string()),
),
"clicked — ctrl may still be held down"
);
let text = impact_text(
"clicked element e3".to_string(),
Impact::Unconfirmed("the call was abandoned".to_string()),
);
assert!(text.starts_with("applied but unconfirmed"), "{text}");
assert!(text.contains("may already have taken effect"), "{text}");
assert!(text.contains("act twice"), "{text}");
assert!(!text.contains("retry"), "{text}");
}
#[test]
fn store_and_resolve_ref_round_trip_then_stale() {
let obs = sample_observation();
let target = TargetSpec::Focused;
let count = store_observation(target.clone(), &obs);
let (locator, resolved_target) = resolve_ref("e2").unwrap();
assert_eq!(locator.role, "AXButton");
assert_eq!(locator.name.as_deref(), Some("OK"));
assert_eq!(locator.path, vec![0]);
assert_eq!(resolved_target, TargetSpec::Focused);
assert_eq!(count, 4);
let obs2 = Observation {
app_name: "Other".to_string(),
window_title: None,
surface: surface(0.0, 0.0, 10.0, 10.0),
root: node("AXWindow", Some("other")),
truncation: Truncation::default(),
};
store_observation(target, &obs2);
let err = resolve_ref("e2").unwrap_err();
let msg = format!("{err}");
assert!(msg.contains("stale-element"), "got {msg}");
let err = resolve_ref("zzz").unwrap_err();
assert!(format!("{err}").contains("stale-element"));
}
#[serial_test::serial(targets)]
#[test]
fn resolve_target_resolves_focused_screen_stored_and_unknown() {
assert_eq!(resolve_target(None).unwrap(), TargetSpec::Focused);
assert_eq!(resolve_target(Some("screen")).unwrap(), TargetSpec::Screen);
let app = TargetSpec::App {
app_pid: Some(42),
app_name: "Safari".to_string(),
};
let ids = store_app_targets(std::slice::from_ref(&app));
assert_eq!(ids.len(), 1);
assert_eq!(resolve_target(Some(&ids[0])).unwrap(), app);
let err = resolve_target(Some("w9999")).unwrap_err();
assert!(format!("{err}").contains("unknown target"));
}
#[serial_test::serial(targets)]
#[test]
fn store_targets_keeps_ids_for_seen_surfaces_and_never_reuses_them() {
let window = |name: &str, key: &str| TargetSpec::Window {
app_pid: Some(7),
app_name: name.to_string(),
key: WindowKey::new(key),
label: format!("{key} title"),
};
let first = store_window_targets(&[window("Beta", "k1"), window("Beta", "k2")]);
let again = store_window_targets(&[window("Beta", "k2"), window("Beta", "k1")]);
assert_eq!(again, vec![first[1].clone(), first[0].clone()]);
let after = store_window_targets(&[window("Beta", "k1"), window("Beta", "k3")]);
assert_eq!(after[0], first[0]);
assert_ne!(after[1], first[1]);
assert_eq!(
resolve_target(Some(&first[1])).unwrap(),
window("Beta", "k2"),
"the id of a vanished window must not name its replacement"
);
let renamed = TargetSpec::Window {
app_pid: Some(7),
app_name: "Beta (renamed)".to_string(),
key: WindowKey::new("k1"),
label: "a new title".to_string(),
};
assert_eq!(store_window_targets(&[renamed])[0], first[0]);
assert_eq!(
resolve_target(Some(&first[0])).unwrap().describe(),
"window \"a new title\" of Beta (renamed)"
);
}
#[test]
fn a11y_scale_maps_only_reconcilable_spaces() {
let screen = surface(0.0, 0.0, 1440.0, 900.0);
assert_eq!(a11y_scale(&screen, (1440, 900)), Some(1.0));
assert_eq!(a11y_scale(&screen, (2880, 1800)), Some(2.0));
assert_eq!(a11y_scale(&screen, (720, 450)), None);
assert_eq!(a11y_scale(&screen, (2880, 900)), None);
assert_eq!(a11y_scale(&surface(0.0, 0.0, 0.0, 0.0), (10, 10)), None);
let scaled = scale_geometry(surface(10.0, 20.0, 100.0, 50.0), 2.0);
assert_eq!(scaled, surface(20.0, 40.0, 200.0, 100.0));
}
#[test]
fn resolve_locator_path_hits_unique_and_not_found() {
let obs = sample_observation();
let root = &obs.root;
let hit = resolve_locator(
root,
&Locator {
path: vec![0],
role: "button".to_string(),
name: Some("OK".to_string()),
},
);
assert!(matches!(hit, LocatorMatch::Path(_)));
let fallback = resolve_locator(
root,
&Locator {
path: vec![99],
role: "button".to_string(),
name: Some("OK".to_string()),
},
);
assert!(matches!(fallback, LocatorMatch::Unique(_)));
let mismatch = resolve_locator(
root,
&Locator {
path: vec![1],
role: "button".to_string(),
name: Some("OK".to_string()),
},
);
assert!(matches!(mismatch, LocatorMatch::Unique(_)));
let none = resolve_locator(
root,
&Locator {
path: Vec::new(),
role: "AXSlider".to_string(),
name: None,
},
);
assert!(matches!(none, LocatorMatch::NotFound));
}
#[test]
fn resolve_locator_ambiguous_when_many_match() {
let obs = sample_observation();
let root = &obs.root;
let all_buttons = resolve_locator(
root,
&Locator {
path: Vec::new(),
role: "AXButton".to_string(),
name: None,
},
);
assert!(matches!(all_buttons, LocatorMatch::Ambiguous));
let pinned = resolve_locator(
root,
&Locator {
path: vec![0],
role: "AXButton".to_string(),
name: None,
},
);
assert!(matches!(pinned, LocatorMatch::Path(_)));
}
#[test]
fn parse_key_chord_splits_modifiers_and_key() {
let (mods, key) = parse_key_chord("cmd+shift+t").unwrap();
assert_eq!(mods, vec![Modifier::Cmd, Modifier::Shift]);
assert_eq!(key, "t");
let (mods, key) = parse_key_chord("Control+Alt+Delete").unwrap();
assert_eq!(mods, vec![Modifier::Ctrl, Modifier::Alt]);
assert_eq!(key, "delete");
let (mods, key) = parse_key_chord("Return").unwrap();
assert!(mods.is_empty());
assert_eq!(key, "return");
let (_, key) = parse_key_chord("5").unwrap();
assert_eq!(key, "5");
let (_, key) = parse_key_chord("F5").unwrap();
assert_eq!(key, "f5");
let err = parse_key_chord("cmd+ctrl").unwrap_err();
assert!(format!("{err}").contains("no key name"), "got {err}");
assert!(parse_key_chord("cmd+").is_err());
let err = parse_key_chord("a+b").unwrap_err();
assert!(format!("{err}").contains("too many key names"), "got {err}");
}
#[test]
#[cfg(any(target_os = "macos", target_os = "linux"))]
fn a_share_cannot_outlive_its_call_or_lengthen_a_short_one() {
let call = Budget::new(
Duration::from_secs(10),
"window listing",
"the running applications",
);
let long = call.slice(Duration::from_secs(60), "the windows of Numbers");
assert_eq!(long.end, call.end);
assert!(long.limit <= call.limit);
let short = call.slice(Duration::ZERO, "the windows of Numbers");
assert_eq!(short.limit, Duration::ZERO);
assert!(short.expired());
assert!(!call.expired());
}
#[test]
fn a_spent_call_says_it_was_abandoned_and_names_the_target() {
let call = Budget::new(Duration::ZERO, "window listing", "the windows of Numbers");
let text = format!("{:#}", call.check().expect_err("the clock is spent"));
assert!(text.starts_with("degraded: "), "{text}");
assert!(
text.contains("the windows of Numbers stopped answering accessibility requests"),
"{text}"
);
assert!(
text.contains("the window listing was abandoned after 0s"),
"{text}"
);
}
#[test]
#[cfg(any(target_os = "macos", target_os = "linux"))]
fn a_spent_share_says_the_item_was_skipped_not_the_call_abandoned() {
let call = Budget::new(
Duration::from_secs(20),
"window listing",
"the running applications",
);
let share = call.slice(Duration::ZERO, "the windows of Numbers");
let note = share.note();
assert!(
note.contains("the windows of Numbers stopped answering accessibility requests"),
"{note}"
);
assert!(
note.contains("it was skipped, so the window listing went on without it"),
"{note}"
);
assert!(!note.contains("abandoned"), "{note}");
assert!(!note.contains("retry"), "{note}");
assert_eq!(
unread_reason(&anyhow!("window refused"), &share),
SKIPPED_NOTE
);
assert_eq!(
unread_reason(&anyhow!("window refused"), &call),
"window refused"
);
}
}