use super::super::core::{
self, ElementAct, Impact, Locator, Observation, SurfaceGeometry, TargetSpec, Truncation, UiNode,
};
use super::{input, window};
use std::time::Duration;
use windows::Win32::Foundation::{E_ACCESSDENIED, E_NOTIMPL};
use windows::Win32::Foundation::{HWND, POINT, RECT};
use windows::Win32::UI::Accessibility::{
ExpandCollapseState_Expanded, IUIAutomation, IUIAutomationElement,
IUIAutomationExpandCollapsePattern, IUIAutomationInvokePattern,
IUIAutomationLegacyIAccessiblePattern, IUIAutomationSelectionItemPattern,
IUIAutomationTogglePattern, IUIAutomationTreeWalker, IUIAutomationValuePattern,
UIA_AppBarControlTypeId, UIA_ButtonControlTypeId, UIA_CalendarControlTypeId,
UIA_CheckBoxControlTypeId, UIA_ComboBoxControlTypeId, UIA_CustomControlTypeId,
UIA_DataGridControlTypeId, UIA_DataItemControlTypeId, UIA_DocumentControlTypeId,
UIA_E_ELEMENTNOTAVAILABLE, UIA_E_ELEMENTNOTENABLED, UIA_E_NOTSUPPORTED, UIA_E_TIMEOUT,
UIA_EditControlTypeId, UIA_ExpandCollapsePatternId, UIA_GroupControlTypeId,
UIA_HeaderControlTypeId, UIA_HeaderItemControlTypeId, UIA_HyperlinkControlTypeId,
UIA_ImageControlTypeId, UIA_InvokePatternId, UIA_LegacyIAccessiblePatternId,
UIA_ListControlTypeId, UIA_ListItemControlTypeId, UIA_MenuBarControlTypeId,
UIA_MenuControlTypeId, UIA_MenuItemControlTypeId, UIA_PATTERN_ID, UIA_PaneControlTypeId,
UIA_ProgressBarControlTypeId, UIA_RadioButtonControlTypeId, UIA_ScrollBarControlTypeId,
UIA_SelectionItemPatternId, UIA_SemanticZoomControlTypeId, UIA_SeparatorControlTypeId,
UIA_SliderControlTypeId, UIA_SpinnerControlTypeId, UIA_SplitButtonControlTypeId,
UIA_StatusBarControlTypeId, UIA_TabControlTypeId, UIA_TabItemControlTypeId,
UIA_TableControlTypeId, UIA_TextControlTypeId, UIA_ThumbControlTypeId,
UIA_TitleBarControlTypeId, UIA_TogglePatternId, UIA_ToolBarControlTypeId,
UIA_ToolTipControlTypeId, UIA_TreeControlTypeId, UIA_TreeItemControlTypeId, UIA_ValuePatternId,
UIA_WindowControlTypeId,
};
use windows::core::BSTR;
pub(super) const WALK_DEADLINE: Duration = Duration::from_secs(10);
const MAX_NODES: usize = 2000;
const MAX_DEPTH: usize = 25;
const VALUE_ROLES: &[&str] = &[
"textfield",
"combobox",
"text",
"document",
"spinner",
"slider",
];
pub(super) fn observe(uia: &IUIAutomation, target: &TargetSpec) -> anyhow::Result<Observation> {
if matches!(target, TargetSpec::Screen) {
return Err(core::screen_observe_error());
}
let resolved = window::resolve(target)?;
let title = window::title_text(resolved.hwnd);
let element = window_element(uia, &resolved)?;
let budget = core::Budget::new(
WALK_DEADLINE,
"tree walk",
core::window_description(&resolved.app_name, &title),
);
let tree = build_tree(uia, &element, resolved.surface, &budget)?;
let mut truncation = tree.truncation;
let window_title = Some(truncation.keep_name(&title));
Ok(Observation {
app_name: resolved.app_name,
window_title,
surface: resolved.surface,
root: tree.root,
truncation,
})
}
pub(super) fn act_on_element(
uia: &IUIAutomation,
target: &TargetSpec,
locator: &Locator,
act: &ElementAct,
) -> anyhow::Result<Impact> {
let resolved = window::resolve(target)?;
window::ensure_actuation_allowed(Some(resolved.hwnd))?;
window::ensure_not_own_window(resolved.hwnd)?;
let element = window_element(uia, &resolved)?;
let budget = core::Budget::new(
WALK_DEADLINE,
"tree walk",
core::window_description(&resolved.app_name, &resolved.title),
);
let tree = build_tree(uia, &element, resolved.surface, &budget)?;
let node = core::resolve_locator_checked(&tree.root, locator)?;
let index = core::pre_order_index(&tree.root, node);
let element = tree.handles.get(index).ok_or_else(|| {
core::taxonomy_error(
core::ERR_STALE_ELEMENT,
"element is no longer in the window's tree — re-observe and use a fresh ref",
)
})?;
match act {
ElementAct::Press => press(element, resolved.hwnd),
ElementAct::SetValue { text } => set_value(element, resolved.hwnd, text),
}
}
fn window_element(
uia: &IUIAutomation,
resolved: &window::TargetWindow,
) -> anyhow::Result<IUIAutomationElement> {
unsafe { uia.ElementFromHandle(resolved.hwnd) }.map_err(|e| {
map_uia_error(
&format!("open the accessibility tree of \"{}\"", resolved.title),
&e,
)
})
}
struct Tree {
root: UiNode,
handles: Vec<IUIAutomationElement>,
truncation: Truncation,
}
struct Walk<'a> {
walker: IUIAutomationTreeWalker,
origin: (f64, f64),
budget: &'a core::Budget,
nodes: usize,
handles: Vec<IUIAutomationElement>,
truncation: Truncation,
}
fn build_tree(
uia: &IUIAutomation,
root: &IUIAutomationElement,
surface: SurfaceGeometry,
budget: &core::Budget,
) -> anyhow::Result<Tree> {
let walker = unsafe { uia.ControlViewWalker() }
.map_err(|e| map_uia_error("open the accessibility tree walker", &e))?;
let mut walk = Walk {
walker,
origin: (surface.x, surface.y),
budget,
nodes: 0,
handles: Vec::new(),
truncation: Truncation::default(),
};
let root = walk.node(root, 0)?;
Ok(Tree {
root,
handles: walk.handles,
truncation: walk.truncation,
})
}
impl Walk<'_> {
fn node(&mut self, element: &IUIAutomationElement, depth: usize) -> anyhow::Result<UiNode> {
self.budget.check()?;
self.nodes += 1;
self.handles.push(element.clone());
let mut node = read_node(element, self.origin, &mut self.truncation, self.budget)?;
self.budget.check()?;
if depth >= MAX_DEPTH {
if unsafe { self.walker.GetFirstChildElement(element) }.is_ok() {
self.truncation.cut_depth(MAX_DEPTH);
}
return Ok(node);
}
let mut child = unsafe { self.walker.GetFirstChildElement(element) }.ok();
while let Some(element) = child {
if self.nodes >= MAX_NODES {
self.truncation.hit_node_ceiling(MAX_NODES);
break;
}
self.budget.check()?;
node.children.push(self.node(&element, depth + 1)?);
child = unsafe { self.walker.GetNextSiblingElement(&element) }.ok();
}
Ok(node)
}
}
fn read_node(
element: &IUIAutomationElement,
origin: (f64, f64),
truncation: &mut Truncation,
budget: &core::Budget,
) -> anyhow::Result<UiNode> {
budget.check()?;
let control_type = unsafe { element.CurrentControlType() }.map_or(0, |id| id.0);
let (role, actions) = role_and_actions(control_type);
budget.check()?;
let name = unsafe { element.CurrentName() }
.ok()
.map(|name| name.to_string())
.filter(|name| !name.is_empty())
.map(|name| truncation.keep_name(&name));
budget.check()?;
let frame = unsafe { element.CurrentBoundingRectangle() }
.ok()
.and_then(|rect| surface_frame(rect, origin));
budget.check()?;
let focused =
unsafe { element.CurrentHasKeyboardFocus() }.is_ok_and(windows::core::BOOL::as_bool);
let value = if VALUE_ROLES.contains(&role) {
read_value(element, budget)?.map(|value| truncation.keep_value(&value))
} else {
None
};
Ok(UiNode {
role: role.to_string(),
name,
value,
frame,
actions: actions.iter().map(|action| (*action).to_string()).collect(),
focused,
children: Vec::new(),
})
}
fn read_value(
element: &IUIAutomationElement,
budget: &core::Budget,
) -> anyhow::Result<Option<String>> {
budget.check()?;
if unsafe { element.CurrentIsPassword() }.is_ok_and(windows::core::BOOL::as_bool) {
return Ok(None);
}
budget.check()?;
let Ok(pattern) =
(unsafe { element.GetCurrentPatternAs::<IUIAutomationValuePattern>(UIA_ValuePatternId) })
else {
return Ok(None);
};
budget.check()?;
let Ok(value) = (unsafe { pattern.CurrentValue() }) else {
return Ok(None);
};
let value = value.to_string();
Ok(if value.is_empty() { None } else { Some(value) })
}
fn surface_frame(rect: RECT, origin: (f64, f64)) -> Option<(f64, f64, f64, f64)> {
let width = f64::from(rect.right - rect.left);
let height = f64::from(rect.bottom - rect.top);
if width <= 0.0 || height <= 0.0 {
return None;
}
Some((
f64::from(rect.left) - origin.0,
f64::from(rect.top) - origin.1,
width,
height,
))
}
fn role_and_actions(control_type: i32) -> (&'static str, &'static [&'static str]) {
const PRESS: &[&str] = &["press"];
const SELECT: &[&str] = &["select"];
const VALUE: &[&str] = &["set-value"];
match control_type {
x if x == UIA_ButtonControlTypeId.0 => ("button", PRESS),
x if x == UIA_SplitButtonControlTypeId.0 => ("button", &["press", "expand"]),
x if x == UIA_CheckBoxControlTypeId.0 => ("checkbox", &["toggle"]),
x if x == UIA_RadioButtonControlTypeId.0 => ("radiobutton", SELECT),
x if x == UIA_ComboBoxControlTypeId.0 => ("combobox", &["expand", "set-value"]),
x if x == UIA_EditControlTypeId.0 => ("textfield", VALUE),
x if x == UIA_HyperlinkControlTypeId.0 => ("link", PRESS),
x if x == UIA_MenuItemControlTypeId.0 => ("menuitem", PRESS),
x if x == UIA_MenuControlTypeId.0 => ("menu", &[]),
x if x == UIA_MenuBarControlTypeId.0 => ("menubar", &[]),
x if x == UIA_ListItemControlTypeId.0 => ("listitem", SELECT),
x if x == UIA_DataItemControlTypeId.0 => ("listitem", SELECT),
x if x == UIA_ListControlTypeId.0 => ("list", &[]),
x if x == UIA_TabItemControlTypeId.0 => ("tab", SELECT),
x if x == UIA_TabControlTypeId.0 => ("tabgroup", &[]),
x if x == UIA_TreeItemControlTypeId.0 => ("treeitem", &["select", "expand"]),
x if x == UIA_TreeControlTypeId.0 => ("tree", &[]),
x if x == UIA_TableControlTypeId.0 => ("table", &[]),
x if x == UIA_DataGridControlTypeId.0 => ("table", &[]),
x if x == UIA_HeaderControlTypeId.0 => ("header", &[]),
x if x == UIA_HeaderItemControlTypeId.0 => ("headeritem", SELECT),
x if x == UIA_SliderControlTypeId.0 => ("slider", VALUE),
x if x == UIA_SpinnerControlTypeId.0 => ("spinner", VALUE),
x if x == UIA_ScrollBarControlTypeId.0 => ("scrollbar", &[]),
x if x == UIA_ProgressBarControlTypeId.0 => ("progressbar", &[]),
x if x == UIA_StatusBarControlTypeId.0 => ("statusbar", &[]),
x if x == UIA_ToolBarControlTypeId.0 => ("toolbar", &[]),
x if x == UIA_ToolTipControlTypeId.0 => ("tooltip", &[]),
x if x == UIA_TitleBarControlTypeId.0 => ("titlebar", &[]),
x if x == UIA_SeparatorControlTypeId.0 => ("separator", &[]),
x if x == UIA_ThumbControlTypeId.0 => ("thumb", &[]),
x if x == UIA_ImageControlTypeId.0 => ("image", &[]),
x if x == UIA_TextControlTypeId.0 => ("text", &[]),
x if x == UIA_DocumentControlTypeId.0 => ("document", VALUE),
x if x == UIA_GroupControlTypeId.0 => ("group", &[]),
x if x == UIA_PaneControlTypeId.0 => ("pane", &[]),
x if x == UIA_WindowControlTypeId.0 => ("window", &[]),
x if x == UIA_CalendarControlTypeId.0 => ("calendar", &[]),
x if x == UIA_CustomControlTypeId.0 => ("custom", &[]),
x if x == UIA_AppBarControlTypeId.0 => ("toolbar", &[]),
x if x == UIA_SemanticZoomControlTypeId.0 => ("group", &[]),
_ => ("unknown", &[]),
}
}
fn press(element: &IUIAutomationElement, hwnd: HWND) -> anyhow::Result<Impact> {
let pressed = attempt(
element,
UIA_InvokePatternId,
|p: &IUIAutomationInvokePattern| unsafe { p.Invoke() },
)
.or_else(|| {
attempt(
element,
UIA_TogglePatternId,
|p: &IUIAutomationTogglePattern| unsafe { p.Toggle() },
)
})
.or_else(|| {
attempt(
element,
UIA_SelectionItemPatternId,
|p: &IUIAutomationSelectionItemPattern| unsafe { p.Select() },
)
})
.or_else(|| {
attempt(
element,
UIA_ExpandCollapsePatternId,
|p: &IUIAutomationExpandCollapsePattern| unsafe {
if p.CurrentExpandCollapseState()
.is_ok_and(|state| state == ExpandCollapseState_Expanded)
{
p.Collapse()
} else {
p.Expand()
}
},
)
})
.or_else(|| {
attempt(
element,
UIA_LegacyIAccessiblePatternId,
|p: &IUIAutomationLegacyIAccessiblePattern| unsafe { p.DoDefaultAction() },
)
});
if let Some(result) = pressed {
return result;
}
let point = clickable_point(element).ok_or_else(|| {
core::taxonomy_error(
core::ERR_UNSUPPORTED,
"element supports no press action and has no visible point — re-observe, or click it by \
coordinates",
)
})?;
window::activate(hwnd);
window::ensure_foreground(hwnd)?;
input::click_absolute(point.x, point.y)
}
fn attempt<T: windows::core::Interface>(
element: &IUIAutomationElement,
id: UIA_PATTERN_ID,
action: impl FnOnce(&T) -> windows::core::Result<()>,
) -> Option<anyhow::Result<Impact>> {
let pattern = unsafe { element.GetCurrentPatternAs::<T>(id) }.ok()?;
match action(&pattern) {
Ok(()) => Some(Ok(Impact::Done)),
Err(error) if is_unsupported(&error) => None,
Err(error) => Some(map_uia_action_error("press the element", &error)),
}
}
fn is_unsupported(error: &windows::core::Error) -> bool {
let code = error.code().0.cast_unsigned();
code == UIA_E_NOTSUPPORTED || code == E_NOTIMPL.0.cast_unsigned()
}
fn is_editable(element: &IUIAutomationElement) -> bool {
let control_type = unsafe { element.CurrentControlType() }.map_or(0, |id| id.0);
[
UIA_EditControlTypeId.0,
UIA_SpinnerControlTypeId.0,
UIA_DocumentControlTypeId.0,
]
.contains(&control_type)
}
fn clickable_point(element: &IUIAutomationElement) -> Option<POINT> {
let mut point = POINT::default();
let reported = unsafe { element.GetClickablePoint(&raw mut point) };
if reported.is_ok_and(windows::core::BOOL::as_bool) {
return Some(point);
}
let rect = unsafe { element.CurrentBoundingRectangle() }.ok()?;
if rect.right <= rect.left || rect.bottom <= rect.top {
return None;
}
Some(POINT {
x: rect.left + (rect.right - rect.left) / 2,
y: rect.top + (rect.bottom - rect.top) / 2,
})
}
fn set_value(element: &IUIAutomationElement, hwnd: HWND, text: &str) -> anyhow::Result<Impact> {
if unsafe { element.CurrentIsPassword() }.is_ok_and(windows::core::BOOL::as_bool) {
return Err(core::taxonomy_error(
core::ERR_UNSUPPORTED,
"element is a password field — its value cannot be set through the accessibility \
channel; click it and use `type` without a ref",
));
}
unsafe {
if let Ok(pattern) =
element.GetCurrentPatternAs::<IUIAutomationValuePattern>(UIA_ValuePatternId)
{
let read_only = pattern
.CurrentIsReadOnly()
.is_ok_and(windows::core::BOOL::as_bool);
if !read_only {
match pattern.SetValue(&BSTR::from(text)) {
Ok(()) => return Ok(Impact::Done),
Err(e) if is_unsupported(&e) => {}
Err(e) => return map_uia_action_error("set the element's value", &e),
}
}
}
}
if !is_editable(element) {
return Err(core::taxonomy_error(
core::ERR_UNSUPPORTED,
"element has no writable value and is not an editable field — use click by \
coordinates and raw typing instead",
));
}
window::activate(hwnd);
window::ensure_foreground(hwnd)?;
unsafe { element.SetFocus() }.map_err(|e| map_uia_error("focus the element", &e))?;
if let Impact::Unconfirmed(detail) = input::select_all()? {
return Err(core::taxonomy_error(
core::ERR_DEGRADED,
format!(
"the element's value was not set and no text was typed — selecting its existing \
text was only partly delivered ({detail}), so the field's selection is now \
unknown"
),
));
}
input::type_text_into_focus(text)
}
fn map_uia_action_error(action: &str, error: &windows::core::Error) -> anyhow::Result<Impact> {
if error.code().0.cast_unsigned() == UIA_E_TIMEOUT {
return Ok(Impact::Unconfirmed(format!(
"the element's provider did not answer {action} within its own timeout, so the call \
was abandoned while it may still complete — {error}"
)));
}
Err(map_uia_error(action, error))
}
fn map_uia_error(action: &str, error: &windows::core::Error) -> anyhow::Error {
if error.code() == E_ACCESSDENIED {
return core::permission_denied(format!("failed to {action}: {error}"));
}
match error.code().0.cast_unsigned() {
UIA_E_ELEMENTNOTAVAILABLE => core::taxonomy_error(
core::ERR_STALE_ELEMENT,
format!(
"failed to {action}: the element is no longer there — re-observe and act on the \
element the tree shows now ({error})"
),
),
UIA_E_ELEMENTNOTENABLED => core::taxonomy_error(
core::ERR_UNSUPPORTED,
format!(
"failed to {action}: the control is disabled, so the action is not available for \
it until its state changes — act on an enabled element, or use coordinates \
({error})"
),
),
UIA_E_NOTSUPPORTED => core::taxonomy_error(
core::ERR_UNSUPPORTED,
format!(
"failed to {action}: the control does not support it — act on it by coordinates \
or use another element ({error})"
),
),
_ => core::taxonomy_error(core::ERR_DEGRADED, format!("failed to {action}: {error}")),
}
}