#![allow(unsafe_code)]
pub mod input;
pub mod kbd;
pub mod mouse;
use actl_core::keys::KeySpec;
use actl_core::{CtlError, ErrorCode, UiNode};
use uiautomation::types::ControlType;
use uiautomation::{UIAutomation, UIElement, UITreeWalker};
pub const MAX_ELEMENTS: usize = 5000;
pub const MAX_DEPTH: u32 = 40;
pub struct WindowInfo {
pub title: String,
pub class: String,
pub pid: u32,
}
pub struct CaptureResult {
pub window: WindowInfo,
pub window_runtime_id: Vec<i32>,
pub nodes: Vec<UiNode>,
pub truncated: bool,
pub wake_ms: u32,
}
pub fn capture(app: Option<&str>) -> Result<CaptureResult, CtlError> {
let auto = UIAutomation::new().map_err(internal)?;
let walker = auto.create_tree_walker().map_err(internal)?;
let target = match app {
Some(pattern) => find_window(&auto, &walker, pattern)?,
None => top_level_of_focused(&auto, &walker)?,
};
let window = WindowInfo {
title: target.get_name().unwrap_or_default(),
class: target.get_classname().unwrap_or_default(),
pid: target.get_process_id().unwrap_or_default(),
};
let wake_ms = ensure_window_responsive(&target);
let window_runtime_id = target.get_runtime_id().unwrap_or_default();
let mut nodes = Vec::new();
walk(&walker, &target, 0, None, &mut nodes);
let truncated = nodes.len() >= MAX_ELEMENTS;
Ok(CaptureResult {
window,
window_runtime_id,
nodes,
truncated,
wake_ms,
})
}
const SNAPSHOT_KEEP: usize = 20;
fn snapshot_dir() -> Option<std::path::PathBuf> {
let base = std::env::var("LOCALAPPDATA").ok()?;
let dir = std::path::Path::new(&base).join("actl").join("snapshots");
std::fs::create_dir_all(&dir).ok()?;
Some(dir)
}
pub fn persist_snapshot(id: &str, window_title: &str, window_runtime_id: &[i32]) {
let Some(dir) = snapshot_dir() else { return };
let record = serde_json::json!({
"snapshot_id": id,
"window_title": window_title,
"window_runtime_id": window_runtime_id,
});
let _ = std::fs::write(
dir.join(format!("{id}.json")),
serde_json::to_string(&record).unwrap_or_default(),
);
if let Ok(entries) = std::fs::read_dir(&dir) {
let mut files: Vec<_> = entries
.filter_map(|e| e.ok())
.filter(|e| e.path().extension().is_some_and(|x| x == "json"))
.filter_map(|e| {
let m = e.metadata().ok()?;
let t = m.modified().ok()?;
Some((t, e.path()))
})
.collect();
files.sort();
let excess = files.len().saturating_sub(SNAPSHOT_KEEP);
for (_, path) in files.into_iter().take(excess) {
let _ = std::fs::remove_file(path);
}
}
}
fn load_snapshot(id: &str) -> Option<(String, Vec<i32>)> {
let dir = snapshot_dir()?;
let text = std::fs::read_to_string(dir.join(format!("{id}.json"))).ok()?;
let v: serde_json::Value = serde_json::from_str(&text).ok()?;
let title = v["window_title"].as_str()?.to_string();
let rid = v["window_runtime_id"]
.as_array()?
.iter()
.filter_map(|x| x.as_i64().map(|n| n as i32))
.collect();
Some((title, rid))
}
pub fn check_snapshot_freshness(snapshot_id: &str, current: &UIElement) -> Result<(), CtlError> {
let Some((_, recorded)) = load_snapshot(snapshot_id) else {
return Err(CtlError::new(
ErrorCode::StaleRef,
format!("snapshot {snapshot_id:?} not found on disk (expired or different machine)"),
));
};
let now = current.get_runtime_id().unwrap_or_default();
if !recorded.is_empty() && now != recorded {
return Err(CtlError::new(
ErrorCode::StaleRef,
format!(
"window instance changed since snapshot {snapshot_id:?} (same-title window reopened); re-snapshot before replaying refs"
),
));
}
Ok(())
}
fn ensure_window_responsive(elem: &UIElement) -> u32 {
use windows::Win32::UI::WindowsAndMessaging::{
SMTO_ABORTIFHUNG, SW_SHOW, SendMessageTimeoutW, ShowWindowAsync, WM_NULL,
};
let Some(hwnd) = native_hwnd(elem) else {
return 0;
};
let pumps = |timeout_ms: u32| unsafe {
SendMessageTimeoutW(
hwnd,
WM_NULL,
windows::Win32::Foundation::WPARAM(0),
windows::Win32::Foundation::LPARAM(0),
SMTO_ABORTIFHUNG,
timeout_ms,
None,
)
.0 != 0
};
if pumps(timing().probe_ms) {
return 0;
}
let started = std::time::Instant::now();
unsafe {
let _ = ShowWindowAsync(hwnd, SW_SHOW);
}
while started.elapsed() < std::time::Duration::from_millis(timing().wake_bound_ms) {
std::thread::sleep(std::time::Duration::from_millis(timing().poll_ms));
if pumps(60) {
break;
}
}
started.elapsed().as_millis() as u32
}
fn window_title_of(hwnd: windows::Win32::Foundation::HWND) -> Option<String> {
use windows::Win32::UI::WindowsAndMessaging::GetWindowTextW;
let mut buf = [0u16; 256];
let len = unsafe { GetWindowTextW(hwnd, &mut buf) };
if len <= 0 {
return None;
}
Some(String::from_utf16_lossy(&buf[..len as usize]))
}
fn native_hwnd(elem: &UIElement) -> Option<windows::Win32::Foundation::HWND> {
let handle = elem.get_native_window_handle().ok()?;
if handle.is_invalid() {
return None;
}
let raw: windows::Win32::Foundation::HANDLE = unsafe { std::mem::transmute(handle) };
Some(windows::Win32::Foundation::HWND(raw.0))
}
fn top_windows(auto: &UIAutomation, walker: &UITreeWalker) -> Vec<UIElement> {
let mut out = Vec::new();
for w in root_children(auto, walker) {
out.push(w.clone());
out.extend(owned_dialogs_of(walker, &w));
}
out
}
fn root_children(auto: &UIAutomation, walker: &UITreeWalker) -> Vec<UIElement> {
let mut out = Vec::new();
let Ok(root) = auto.get_root_element() else {
return out;
};
let mut child = walker.get_first_child(&root).ok();
while let Some(w) = child {
out.push(w.clone());
child = walker.get_next_sibling(&w).ok();
}
out
}
fn owned_dialogs_of(walker: &UITreeWalker, host: &UIElement) -> Vec<UIElement> {
let mut out = Vec::new();
let mut sub = walker.get_first_child(host).ok();
while let Some(d) = sub {
if d.get_control_type()
.map(|t| t == ControlType::Window)
.unwrap_or(false)
{
out.push(d.clone());
}
sub = walker.get_next_sibling(&d).ok();
}
out
}
fn scan_top_window(auto: &UIAutomation, walker: &UITreeWalker, pattern: &str) -> Option<UIElement> {
top_windows(auto, walker)
.into_iter()
.find(|w| name_matches(w, pattern))
}
fn name_matches(elem: &UIElement, pattern: &str) -> bool {
elem.get_name()
.map(|n| n.contains(pattern))
.unwrap_or(false)
}
const WINUI_AUX_CLASSES: [&str; 2] = ["ApplicationFrameWindow", "ApplicationFrameTitleBarWindow"];
fn find_window(
auto: &UIAutomation,
walker: &UITreeWalker,
pattern: &str,
) -> Result<UIElement, CtlError> {
let mut matches: Vec<UIElement> = root_children(auto, walker)
.into_iter()
.filter(|w| name_matches(w, pattern))
.collect();
if matches.is_empty() {
for host in root_children(auto, walker) {
matches.extend(
owned_dialogs_of(walker, &host)
.into_iter()
.filter(|w| name_matches(w, pattern)),
);
}
}
let has_primary = matches.iter().any(|w| {
let cls = w.get_classname().unwrap_or_default();
!WINUI_AUX_CLASSES.contains(&cls.as_str())
});
let pool: Vec<UIElement> = if has_primary {
matches
.into_iter()
.filter(|w| {
let cls = w.get_classname().unwrap_or_default();
!WINUI_AUX_CLASSES.contains(&cls.as_str())
})
.collect()
} else {
matches
};
match pool.len() {
0 => Err(CtlError::new(
ErrorCode::NotFound,
format!("no top-level window title contains {pattern:?}"),
)),
1 => Ok(pool.into_iter().next().expect("len == 1")),
n => {
let titles: Vec<String> = pool
.iter()
.filter_map(|w| w.get_name().ok())
.take(5)
.collect();
Err(CtlError::new(
ErrorCode::Ambiguous,
format!(
"pattern {pattern:?} matches {n} windows: {titles:?}; tighten the selector"
),
))
}
}
}
fn top_level_of_focused(auto: &UIAutomation, walker: &UITreeWalker) -> Result<UIElement, CtlError> {
let mut cur = auto.get_focused_element().map_err(internal)?;
loop {
match walker.get_parent(&cur) {
Ok(p) => cur = p,
Err(_) => return Ok(cur), }
}
}
fn walk(
walker: &UITreeWalker,
elem: &UIElement,
depth: u32,
parent: Option<usize>,
nodes: &mut Vec<UiNode>,
) {
if nodes.len() >= MAX_ELEMENTS || depth > MAX_DEPTH {
return;
}
nodes.push(to_node(elem, depth, parent));
let idx = nodes.len() - 1;
let mut child = walker.get_first_child(elem).ok();
while let Some(c) = child {
walk(walker, &c, depth + 1, Some(idx), nodes);
child = walker.get_next_sibling(&c).ok();
}
}
fn to_node(elem: &UIElement, depth: u32, parent: Option<usize>) -> UiNode {
UiNode {
depth,
role: format!(
"{:?}",
elem.get_control_type().unwrap_or(ControlType::Custom)
),
name: elem.get_name().ok().filter(|n| !n.is_empty()),
automation_id: elem.get_automation_id().ok().filter(|a| !a.is_empty()),
parent,
}
}
fn timing() -> &'static actl_core::timing::Timing {
actl_core::timing::Timing::load()
}
fn internal(e: impl std::fmt::Debug) -> CtlError {
CtlError::new(ErrorCode::Internal, format!("{e:?}"))
}
use actl_core::target::Target;
use uiautomation::patterns::{
UIExpandCollapsePattern, UIInvokePattern, UISelectionItemPattern, UITogglePattern,
UIValuePattern,
};
pub struct Located {
pub window_title: String,
pub role: String,
pub name: Option<String>,
pub automation_id: Option<String>,
pub resolved_by: &'static str,
pub(crate) element: UIElement,
}
pub fn locate(app: Option<&str>, target: &Target, near: Option<&str>) -> Result<Located, CtlError> {
let auto = UIAutomation::new().map_err(internal)?;
let walker = auto.create_tree_walker().map_err(internal)?;
let root = match app {
Some(pattern) => find_window(&auto, &walker, pattern)?,
None => {
if matches!(target, Target::Ref(_)) {
return Err(CtlError::protocol(format!(
"{} requires --app: refs are scoped to the window the snapshot came from; \
pass --app <title-substr> or re-snapshot",
target.describe()
)));
}
top_level_of_focused(&auto, &walker)?
}
};
let window_title = root.get_name().unwrap_or_default();
let _wake_ms = ensure_window_responsive(&root);
if let Target::Ref(n) = target {
let mut counter = 0u32;
return match collect_until_ref(&walker, &root, 0, *n, &mut counter) {
Some(hit) => Ok(located_from(hit, window_title, "ref-replay")),
None => Err(CtlError::new(
ErrorCode::StaleRef,
format!(
"{} does not resolve — the tree changed since the snapshot",
target.describe()
),
)),
};
}
let mut counter = 0u32;
let mut matches = Vec::new();
collect_matches(&walker, &root, 0, target, false, &mut counter, &mut matches);
if matches.is_empty() {
if let Target::Name(_) = target {
let mut fuzzy = Vec::new();
collect_matches(&walker, &root, 0, target, true, &mut counter, &mut fuzzy);
if !fuzzy.is_empty() {
fuzzy.sort_by_key(|c: &Candidate| c.ref_no);
return select(
fuzzy,
target,
near,
&walker,
&root,
window_title,
"fuzzy-name",
);
}
}
return Err(CtlError::new(
ErrorCode::NotFound,
format!(
"no element matches {} (exact and fuzzy levels)",
target.describe()
),
));
}
matches.sort_by_key(|c: &Candidate| c.ref_no);
select(
matches,
target,
near,
&walker,
&root,
window_title,
"primary",
)
}
fn select(
matches: Vec<Candidate>,
target: &Target,
near: Option<&str>,
walker: &UITreeWalker,
root: &UIElement,
window_title: String,
level: &'static str,
) -> Result<Located, CtlError> {
if let Target::RoleAt(_, n) = target {
return match matches.get((*n as usize).saturating_sub(1)) {
Some(c) => Ok(located_from(
c.element.clone(),
window_title,
"role-ordinal",
)),
None => Err(CtlError::new(
ErrorCode::NotFound,
format!(
"{}: only {} match(es) in the window",
target.describe(),
matches.len()
),
)),
};
}
if matches.len() == 1 {
return Ok(located_from(
matches[0].element.clone(),
window_title,
level,
));
}
if let Some(anchor_pat) = near {
let mut counter = 0u32;
let mut anchors = Vec::new();
let anchor_target =
actl_core::parse_target(anchor_pat).unwrap_or(Target::Name(anchor_pat.to_string()));
collect_matches(
walker,
root,
0,
&anchor_target,
false,
&mut counter,
&mut anchors,
);
if anchors.len() != 1 {
return Err(CtlError::new(
ErrorCode::NotFound,
format!(
"--near anchor {anchor_pat:?} must match exactly one element, got {}",
anchors.len()
),
));
}
if let Ok(ar) = anchors[0].element.get_bounding_rectangle() {
let (ax, ay) = center(&ar);
let mut best: Option<(f64, usize)> = None;
for (i, c) in matches.iter().enumerate() {
if let Ok(r) = c.element.get_bounding_rectangle() {
let (cx, cy) = center(&r);
let dx = (cx - ax) as f64;
let dy = (cy - ay) as f64;
let d = dx * dx + dy * dy;
if best.map(|(bd, _)| d < bd).unwrap_or(true) {
best = Some((d, i));
}
}
}
if let Some((_, i)) = best {
return Ok(located_from(
matches[i].element.clone(),
window_title,
"anchor",
));
}
}
}
let evidence: Vec<serde_json::Value> = matches
.iter()
.take(8)
.map(|c| {
serde_json::json!({
"ref": format!("@e{}", c.ref_no),
"role": c.role,
"name": c.name,
"automation_id": c.automation_id,
})
})
.collect();
Err(CtlError::with_evidence(
ErrorCode::Ambiguous,
format!(
"{} matches {} elements in the window; pick one by @eN or role ordinal (--near anchor also applies)",
target.describe(),
matches.len()
),
serde_json::json!({ "candidates": evidence }),
))
}
fn center(r: &uiautomation::types::Rect) -> (i32, i32) {
(
(r.get_left() + r.get_right()) / 2,
(r.get_top() + r.get_bottom()) / 2,
)
}
fn located_from(elem: UIElement, window_title: String, resolved_by: &'static str) -> Located {
Located {
window_title,
role: format!(
"{:?}",
elem.get_control_type().unwrap_or(ControlType::Custom)
),
name: elem.get_name().ok().filter(|n| !n.is_empty()),
automation_id: elem.get_automation_id().ok().filter(|a| !a.is_empty()),
resolved_by,
element: elem,
}
}
struct Candidate {
element: UIElement,
ref_no: u32,
role: String,
name: Option<String>,
automation_id: Option<String>,
}
fn collect_until_ref(
walker: &UITreeWalker,
elem: &UIElement,
depth: u32,
want: u32,
counter: &mut u32,
) -> Option<UIElement> {
if depth > MAX_DEPTH {
return None;
}
let role = role_of(elem);
if actl_core::is_interactive_role(&role) {
*counter += 1;
if *counter == want {
return Some(elem.clone());
}
}
let mut child = walker.get_first_child(elem).ok();
while let Some(c) = child {
if let Some(hit) = collect_until_ref(walker, &c, depth + 1, want, counter) {
return Some(hit);
}
child = walker.get_next_sibling(&c).ok();
}
None
}
fn collect_matches(
walker: &UITreeWalker,
elem: &UIElement,
depth: u32,
target: &Target,
fuzzy: bool,
counter: &mut u32,
out: &mut Vec<Candidate>,
) {
if depth > MAX_DEPTH {
return;
}
let role = role_of(elem);
let ref_no = if actl_core::is_interactive_role(&role) {
*counter += 1;
*counter
} else {
0 };
let name = elem.get_name().ok().filter(|n| !n.is_empty());
let id = elem.get_automation_id().ok().filter(|a| !a.is_empty());
let hit = match target {
Target::Ref(_) => false,
Target::Name(s) => match (&name, fuzzy) {
(Some(n), true) => actl_core::target::fuzzy_contains(n, s),
(Some(n), false) => n.contains(s.as_str()),
(None, _) => false,
},
Target::Id(s) => id.as_deref() == Some(s.as_str()),
Target::Role(s) | Target::RoleAt(s, _) => role == *s,
};
if hit {
out.push(Candidate {
element: elem.clone(),
ref_no,
role: role.clone(),
name: name.clone(),
automation_id: id.clone(),
});
}
let mut child = walker.get_first_child(elem).ok();
while let Some(c) = child {
collect_matches(walker, &c, depth + 1, target, fuzzy, counter, out);
child = walker.get_next_sibling(&c).ok();
}
}
fn role_of(elem: &UIElement) -> String {
format!(
"{:?}",
elem.get_control_type().unwrap_or(ControlType::Custom)
)
}
pub fn click(
app: Option<&str>,
target: &Target,
near: Option<&str>,
) -> Result<(Located, &'static str), CtlError> {
let loc = locate(app, target, near)?;
if let Ok(invoke) = loc.element.get_pattern::<UIInvokePattern>() {
invoke.invoke().map_err(internal)?;
return Ok((loc, "invoke"));
}
if let Ok(sel) = loc.element.get_pattern::<UISelectionItemPattern>() {
sel.select().map_err(internal)?;
return Ok((loc, "selection-item"));
}
if let Ok(toggle) = loc.element.get_pattern::<UITogglePattern>() {
toggle.toggle().map_err(internal)?;
return Ok((loc, "toggle"));
}
if let Ok(exp) = loc.element.get_pattern::<UIExpandCollapsePattern>() {
exp.expand().map_err(internal)?;
return Ok((loc, "expand-collapse"));
}
Err(CtlError::new(
ErrorCode::NotActionable,
format!(
"{} ({}) exposes none of Invoke/SelectionItem/Toggle/ExpandCollapse; \
scroll/focus first, try `set-value`, or `--physical` for a real click",
target.describe(),
loc.role
),
))
}
pub const PROPS: &[&str] = &["name", "role", "automation_id", "offscreen", "value"];
pub struct PropertyInfo {
pub name: Option<String>,
pub role: String,
pub automation_id: Option<String>,
pub offscreen: Option<bool>,
pub value: Option<String>,
}
pub fn read_property(
app: Option<&str>,
target: &Target,
property: Option<&str>,
near: Option<&str>,
) -> Result<PropertyInfo, CtlError> {
if let Some(p) = property {
if !PROPS.contains(&p) {
return Err(CtlError::protocol(format!(
"unknown property {p:?}: expected one of {PROPS:?}"
)));
}
}
let loc = locate(app, target, near)?;
let want = |p: &str| property.map(|sel| sel == p).unwrap_or(true);
let value = want("value")
.then(|| {
loc.element
.get_pattern::<UIValuePattern>()
.ok()
.and_then(|pat| pat.get_value().ok())
.filter(|v| !v.is_empty())
})
.flatten();
Ok(PropertyInfo {
name: want("name").then_some(loc.name.clone()).flatten(),
role: if want("role") {
loc.role.clone()
} else {
String::new()
},
automation_id: want("automation_id")
.then_some(loc.automation_id.clone())
.flatten(),
offscreen: want("offscreen")
.then(|| loc.element.is_offscreen().ok())
.flatten(),
value,
})
}
pub struct InputResult {
pub window_title: String,
pub role: String,
pub name: Option<String>,
}
impl From<&Located> for InputResult {
fn from(loc: &Located) -> Self {
Self {
window_title: loc.window_title.clone(),
role: loc.role.clone(),
name: loc.name.clone(),
}
}
}
pub fn type_text(
app: Option<&str>,
target: &Target,
text: &str,
paste: bool,
near: Option<&str>,
) -> Result<TypeOutcome, CtlError> {
let loc = locate(app, target, near)?;
loc.element.set_focus().map_err(|_| {
CtlError::new(
ErrorCode::NotActionable,
format!(
"{} ({}) cannot take keyboard focus",
target.describe(),
loc.role
),
)
})?;
if let Some(pattern) = app {
std::thread::sleep(std::time::Duration::from_millis(timing().type_focus_ms));
let fg = kbd::foreground_title().unwrap_or_default();
if !fg.contains(pattern) {
return Err(CtlError::new(
ErrorCode::PermDenied,
format!(
"text injection ABORTED by verify-then-inject guard: \
physical foreground is {fg:?}, expected a window matching {pattern:?}; \
no text was sent"
),
));
}
}
let _lock = input::InputLock::acquire(timing().lock_wait_ms)?;
let total = text.chars().count();
let outcome = |delivered: usize, stopped_early: bool| TypeOutcome {
located: (&loc).into(),
delivered,
total,
stopped_early,
};
if paste {
set_clipboard_text(text)?;
kbd::send_key_spec(&KeySpec {
modifiers: vec!["ctrl".into()],
keys: vec![actl_core::keys::Key::Char('v')],
})?;
Ok(outcome(total, false))
} else {
let chars: Vec<char> = text.chars().collect();
let pattern = app.map(str::to_string);
let delivered = input::send_in_segments(
&chars,
timing().segment_chars,
|seg| {
kbd::send_key_spec(&KeySpec {
modifiers: Vec::new(),
keys: seg.iter().map(|c| actl_core::keys::Key::Char(*c)).collect(),
})
},
|| {
pattern
.as_ref()
.map(|p| {
kbd::foreground_title()
.map(|t| t.contains(p.as_str()))
.unwrap_or(false)
})
.unwrap_or(false)
},
)?;
Ok(outcome(delivered, delivered < total))
}
}
pub struct TypeOutcome {
pub located: InputResult,
pub delivered: usize,
pub total: usize,
pub stopped_early: bool,
}
pub fn press_keys(spec: &KeySpec, app: Option<&str>) -> Result<Option<String>, CtlError> {
let focused = prepare_keyboard(app)?;
let _lock = input::InputLock::acquire(timing().lock_wait_ms)?;
kbd::send_key_spec(spec)?;
Ok(focused)
}
pub fn press_half(spec: &KeySpec, app: Option<&str>) -> Result<Option<String>, CtlError> {
let focused = prepare_keyboard(app)?;
let _lock = input::InputLock::acquire(timing().lock_wait_ms)?;
kbd::send_key_partial(spec, true)?;
Ok(focused)
}
fn prepare_keyboard(app: Option<&str>) -> Result<Option<String>, CtlError> {
Ok(match app {
Some(pattern) => {
let title = focus_window(pattern)?;
std::thread::sleep(std::time::Duration::from_millis(timing().press_focus_ms));
ensure_element_focus(pattern)?;
let fg = kbd::foreground_title().unwrap_or_default();
if !fg.contains(pattern) {
return Err(CtlError::new(
ErrorCode::PermDenied,
format!(
"keyboard injection ABORTED by verify-then-inject guard: \
physical foreground is {fg:?}, expected a window matching {pattern:?}; \
no keys were sent"
),
));
}
Some(title)
}
None => None,
})
}
fn ensure_element_focus(pattern: &str) -> Result<(), CtlError> {
for _ in 0..timing().focus_retries {
if let Some(edit) = find_editable_element(pattern) {
let _ = edit.set_focus();
}
std::thread::sleep(std::time::Duration::from_millis(timing().poll_ms));
if kbd::focused_root_title()
.map(|t| t.contains(pattern))
.unwrap_or(false)
{
return Ok(());
}
}
Ok(())
}
fn find_editable_element(pattern: &str) -> Option<UIElement> {
let auto = UIAutomation::new().ok()?;
let walker = auto.create_tree_walker().ok()?;
let win = find_window(&auto, &walker, pattern).ok()?;
fn dfs(walker: &UITreeWalker, elem: &UIElement, depth: u32) -> Option<UIElement> {
if depth > 12 {
return None;
}
let role = format!("{:?}", elem.get_control_type().ok()?);
if role == "Document" || role == "Edit" {
return Some(elem.clone());
}
let mut child = walker.get_first_child(elem).ok();
while let Some(c) = child {
if let Some(hit) = dfs(walker, &c, depth + 1) {
return Some(hit);
}
child = walker.get_next_sibling(&c).ok();
}
None
}
dfs(&walker, &win, 0)
}
pub fn set_value(
app: Option<&str>,
target: &Target,
value: &str,
near: Option<&str>,
) -> Result<InputResult, CtlError> {
let loc = locate(app, target, near)?;
let pattern = loc.element.get_pattern::<UIValuePattern>().map_err(|_| {
CtlError::new(
ErrorCode::NotActionable,
format!(
"{} ({}) exposes no Value pattern; try `type` (keyboard path) instead",
target.describe(),
loc.role
),
)
})?;
pattern.set_value(value).map_err(internal)?;
Ok((&loc).into())
}
pub fn wait_window(substr: &str, timeout_ms: u64) -> Result<String, CtlError> {
let auto = UIAutomation::new().map_err(internal)?;
let walker = auto.create_tree_walker().map_err(internal)?;
let deadline = std::time::Instant::now() + std::time::Duration::from_millis(timeout_ms);
loop {
if let Some(title) = scan_top_windows(&auto, &walker, substr) {
return Ok(title);
}
if std::time::Instant::now() >= deadline {
return Err(CtlError::new(
ErrorCode::AssertionFailed,
format!(
"expected window containing {substr:?} did not appear within {timeout_ms}ms"
),
));
}
std::thread::sleep(std::time::Duration::from_millis(timing().poll_ms));
}
}
pub fn window_identities() -> Result<Vec<Vec<i32>>, CtlError> {
let auto = UIAutomation::new().map_err(internal)?;
let walker = auto.create_tree_walker().map_err(internal)?;
Ok(top_windows(&auto, &walker)
.into_iter()
.filter_map(|w| w.get_runtime_id().ok())
.collect())
}
pub fn wait_new_window(
substr: &str,
timeout_ms: u64,
baseline: &[Vec<i32>],
) -> Result<String, CtlError> {
let auto = UIAutomation::new().map_err(internal)?;
let walker = auto.create_tree_walker().map_err(internal)?;
let deadline = std::time::Instant::now() + std::time::Duration::from_millis(timeout_ms);
loop {
if let Some(title) = top_windows(&auto, &walker)
.into_iter()
.filter(|w| name_matches(w, substr))
.find(|w| {
w.get_runtime_id()
.map(|id| !baseline.contains(&id))
.unwrap_or(false) })
.and_then(|w| w.get_name().ok())
{
return Ok(title);
}
if std::time::Instant::now() >= deadline {
return Err(CtlError::new(
ErrorCode::AssertionFailed,
format!(
"no NEW window containing {substr:?} appeared within {timeout_ms}ms \
(a pre-existing window with that title does not satisfy --expect)"
),
));
}
std::thread::sleep(std::time::Duration::from_millis(timing().poll_ms));
}
}
fn scan_top_windows(auto: &UIAutomation, walker: &UITreeWalker, substr: &str) -> Option<String> {
scan_top_window(auto, walker, substr).and_then(|w| w.get_name().ok())
}
pub fn focus_window(pattern: &str) -> Result<String, CtlError> {
let auto = UIAutomation::new().map_err(internal)?;
let walker = auto.create_tree_walker().map_err(internal)?;
let win = find_window(&auto, &walker, pattern)?;
win.set_focus().map_err(|e| {
CtlError::new(
ErrorCode::NotActionable,
format!("cannot focus window {pattern:?}: {e:?}"),
)
})?;
Ok(win.get_name().unwrap_or_default())
}
pub fn list_windows() -> Result<Vec<(String, String, u32)>, CtlError> {
let auto = UIAutomation::new().map_err(internal)?;
let walker = auto.create_tree_walker().map_err(internal)?;
Ok(top_windows(&auto, &walker)
.into_iter()
.filter_map(|w| {
let title = w.get_name().ok()?;
if title.is_empty() {
return None;
}
Some((
title,
w.get_classname().unwrap_or_default(),
w.get_process_id().unwrap_or_default(),
))
})
.collect())
}
const CF_UNICODETEXT: u32 = 13;
pub fn close_window(pattern: &str) -> Result<String, CtlError> {
use windows::Win32::UI::WindowsAndMessaging::{PostMessageW, WM_CLOSE};
let auto = UIAutomation::new().map_err(internal)?;
let walker = auto.create_tree_walker().map_err(internal)?;
let win = find_window(&auto, &walker, pattern)?;
let title = win.get_name().unwrap_or_default();
let Some(hwnd) = native_hwnd(&win) else {
return Err(CtlError::new(
ErrorCode::NotActionable,
format!("window {pattern:?} exposes no native handle"),
));
};
unsafe {
PostMessageW(
Some(hwnd),
WM_CLOSE,
windows::Win32::Foundation::WPARAM(0),
windows::Win32::Foundation::LPARAM(0),
)
}
.map_err(|e| CtlError::internal(format!("PostMessageW(WM_CLOSE) failed for {title:?}: {e}")))?;
Ok(title)
}
pub fn wait_element(app: Option<&str>, target: &Target, timeout_ms: u64) -> Result<(), CtlError> {
let deadline = std::time::Instant::now() + std::time::Duration::from_millis(timeout_ms);
loop {
if locate(app, target, None).is_ok() {
return Ok(());
}
if std::time::Instant::now() >= deadline {
return Err(CtlError::new(
ErrorCode::Timeout,
format!(
"element {} did not resolve within {timeout_ms}ms",
target.describe()
),
));
}
std::thread::sleep(std::time::Duration::from_millis(timing().poll_ms));
}
}
use windows::Win32::Foundation::HGLOBAL;
use windows::Win32::System::DataExchange::{
CloseClipboard, EmptyClipboard, GetClipboardData, OpenClipboard, SetClipboardData,
};
use windows::Win32::System::Memory::{GMEM_MOVEABLE, GlobalAlloc, GlobalLock, GlobalUnlock};
struct ClipGuard;
impl Drop for ClipGuard {
fn drop(&mut self) {
unsafe { CloseClipboard() }.ok();
}
}
fn open_clipboard() -> Result<ClipGuard, CtlError> {
if unsafe { OpenClipboard(None) }.is_ok() {
Ok(ClipGuard)
} else {
Err(CtlError::new(
ErrorCode::NotActionable,
"clipboard is held by another process; retry shortly",
))
}
}
pub fn get_clipboard_text() -> Result<Option<String>, CtlError> {
unsafe {
let _guard = open_clipboard()?;
if !windows::Win32::System::DataExchange::IsClipboardFormatAvailable(CF_UNICODETEXT).is_ok()
{
return Ok(None);
}
let Ok(h) = GetClipboardData(CF_UNICODETEXT) else {
return Ok(None);
};
let hg = HGLOBAL(h.0);
let p = GlobalLock(hg) as *const u16;
if p.is_null() {
return Ok(None);
}
let mut len = 0usize;
while *p.add(len) != 0 {
len += 1;
}
let text = String::from_utf16_lossy(std::slice::from_raw_parts(p, len));
let _ = GlobalUnlock(hg);
Ok(Some(text))
}
}
pub fn set_clipboard_text(text: &str) -> Result<(), CtlError> {
use windows::Win32::Foundation::{GlobalFree, HANDLE};
let mut wide: Vec<u16> = text.encode_utf16().collect();
wide.push(0);
unsafe {
let _guard = open_clipboard()?;
EmptyClipboard().map_err(|e| CtlError::internal(format!("EmptyClipboard: {e}")))?;
let h = GlobalAlloc(GMEM_MOVEABLE, wide.len() * 2)
.map_err(|e| CtlError::internal(format!("GlobalAlloc: {e}")))?;
let p = GlobalLock(h) as *mut u16;
if p.is_null() {
let _ = GlobalFree(Some(h));
return Err(CtlError::internal("GlobalLock failed"));
}
std::ptr::copy_nonoverlapping(wide.as_ptr(), p, wide.len());
let _ = GlobalUnlock(h);
if SetClipboardData(CF_UNICODETEXT, Some(HANDLE(h.0))).is_err() {
let _ = GlobalFree(Some(h));
return Err(CtlError::internal("SetClipboardData failed"));
}
Ok(())
}
}
pub enum PointerAction {
LeftClick,
RightClick,
DoubleClick,
Hover,
Drag { to: (i32, i32) },
Wheel { notches: i32 },
}
pub fn pointer_physical(
app: Option<&str>,
target: &Target,
near: Option<&str>,
action: PointerAction,
) -> Result<(Located, (i32, i32)), CtlError> {
use windows::Win32::Foundation::POINT;
use windows::Win32::UI::WindowsAndMessaging::{GA_ROOT, GetAncestor, WindowFromPoint};
let loc = locate(app, target, near)?;
let Some(point) = loc.element.get_clickable_point().ok().flatten() else {
return Err(CtlError::new(
ErrorCode::NotActionable,
format!(
"{} ({}) exposes no clickable point; refusing to guess a center point",
target.describe(),
loc.role
),
));
};
let (px, py) = (point.get_x(), point.get_y());
let auto = UIAutomation::new().map_err(internal)?;
let walker = auto.create_tree_walker().map_err(internal)?;
let target_title = {
let mut root = loc.element.clone();
while native_hwnd(&root).is_none() {
match walker.get_parent(&root) {
Ok(p) => root = p,
Err(_) => break,
}
}
let title = native_hwnd(&root).and_then(window_title_of);
match title {
Some(t) => t,
None => find_window(&auto, &walker, &loc.window_title)
.ok()
.and_then(|w| native_hwnd(&w))
.and_then(window_title_of)
.ok_or_else(|| {
CtlError::new(
ErrorCode::NotActionable,
"target window has no native handle for hit-checking",
)
})?,
}
};
let root_of = |hwnd: windows::Win32::Foundation::HWND| {
let r = unsafe { GetAncestor(hwnd, GA_ROOT) };
(!r.0.is_null()).then_some(r)
};
let hit = unsafe { WindowFromPoint(POINT { x: px, y: py }) };
let hit_title = (!hit.0.is_null())
.then_some(hit)
.and_then(root_of)
.and_then(window_title_of);
if hit_title.as_deref() != Some(target_title.as_str()) {
return Err(CtlError::new(
ErrorCode::NotActionable,
format!(
"clickable point ({px},{py}) is occupied by another window \
(occluded or minimized); refusing physical pointer action"
),
));
}
let _lock = input::InputLock::acquire(timing().lock_wait_ms)?;
kbd::wait_modifiers_clear()?;
match action {
PointerAction::LeftClick => mouse::left_click_at(px, py)?,
PointerAction::RightClick => mouse::right_click_at(px, py)?,
PointerAction::DoubleClick => mouse::double_click_at(px, py)?,
PointerAction::Hover => mouse::move_to(px, py)?,
PointerAction::Drag { to: (tx, ty) } => mouse::drag_to(px, py, tx, ty)?,
PointerAction::Wheel { notches } => mouse::wheel_at(px, py, notches)?,
}
Ok((loc, (px, py)))
}
pub fn click_physical(
app: Option<&str>,
target: &Target,
near: Option<&str>,
) -> Result<(Located, (i32, i32)), CtlError> {
pointer_physical(app, target, near, PointerAction::LeftClick)
}
pub fn drag_physical(
app: Option<&str>,
from: &Target,
from_near: Option<&str>,
to: &Target,
to_near: Option<&str>,
) -> Result<(Located, (i32, i32)), CtlError> {
let (_, (fx, fy)) = pointer_physical(app, from, from_near, PointerAction::Hover)?;
let Some(dest) = locate(app, to, to_near)?
.element
.get_clickable_point()
.ok()
.flatten()
else {
return Err(CtlError::new(
ErrorCode::NotActionable,
format!(
"{} exposes no clickable point for drag destination",
to.describe()
),
));
};
let point = (dest.get_x(), dest.get_y());
let _lock = input::InputLock::acquire(timing().lock_wait_ms)?;
mouse::drag_to(fx, fy, point.0, point.1)?;
Ok((locate(app, from, from_near)?, (fx, fy)))
}
pub fn resize_window(pattern: &str, width: i32, height: i32) -> Result<String, CtlError> {
use windows::Win32::Foundation::{HWND, RECT};
use windows::Win32::UI::WindowsAndMessaging::{
GetWindowRect, SWP_NOACTIVATE, SWP_NOMOVE, SWP_NOZORDER, SetWindowPos,
};
let auto = UIAutomation::new().map_err(internal)?;
let walker = auto.create_tree_walker().map_err(internal)?;
let win = find_window(&auto, &walker, pattern)?;
let title = win.get_name().unwrap_or_default();
let Some(hwnd) = native_hwnd(&win) else {
return Err(CtlError::new(
ErrorCode::NotActionable,
format!("window {pattern:?} has no native handle"),
));
};
let mut rect = RECT::default();
unsafe { GetWindowRect(hwnd, &mut rect) }
.map_err(|e| CtlError::internal(format!("GetWindowRect: {e}")))?;
let w = width.max(120);
let h = height.max(120);
unsafe {
SetWindowPos(
hwnd,
Some(HWND(std::ptr::null_mut())),
rect.left,
rect.top,
w,
h,
SWP_NOMOVE | SWP_NOZORDER | SWP_NOACTIVATE,
)
}
.map_err(|e| CtlError::internal(format!("SetWindowPos: {e}")))?;
Ok(title)
}
pub fn scroll_element(
app: Option<&str>,
target: &Target,
near: Option<&str>,
notches: i32,
) -> Result<(Located, &'static str), CtlError> {
use uiautomation::patterns::UIScrollPattern;
use uiautomation::types::ScrollAmount;
let loc = locate(app, target, near)?;
if let Ok(scroll) = loc.element.get_pattern::<UIScrollPattern>() {
let amount = if notches >= 0 {
ScrollAmount::SmallIncrement
} else {
ScrollAmount::SmallDecrement
};
let mut ok = true;
for _ in 0..notches.abs() {
if scroll
.scroll(uiautomation::types::ScrollAmount::NoAmount, amount)
.is_err()
{
ok = false;
break;
}
}
if ok {
return Ok((loc, "uia-scroll"));
}
}
let (_, _) = pointer_physical(app, target, near, PointerAction::Wheel { notches })?;
Ok((loc, "physical-wheel"))
}
pub enum VerifyKind {
Exists,
Value { expected: String, contains: bool },
Checked,
}
pub struct VerifyOutcome {
pub located: Located,
pub actual: Option<String>,
pub mode: &'static str,
}
pub fn verify(
app: Option<&str>,
target: &Target,
near: Option<&str>,
kind: VerifyKind,
) -> Result<VerifyOutcome, CtlError> {
use uiautomation::patterns::UITogglePattern;
let fail = |what: &str, actual: Option<String>| {
CtlError::with_evidence(
ErrorCode::AssertionFailed,
format!("verify {what} failed"),
serde_json::json!({ "actual": actual }),
)
};
match kind {
VerifyKind::Exists => {
let loc = locate(app, target, near)?;
Ok(VerifyOutcome {
located: loc,
actual: None,
mode: "exists",
})
}
VerifyKind::Value { expected, contains } => {
let loc = locate(app, target, near)?;
let actual = read_value(&loc.element);
let ok = match (&actual, contains) {
(Some(a), true) => a.contains(&expected),
(Some(a), false) => a == &expected,
(None, _) => false,
};
if !ok {
return Err(fail("value", actual));
}
Ok(VerifyOutcome {
located: loc,
actual,
mode: if contains { "value-contains" } else { "value" },
})
}
VerifyKind::Checked => {
let loc = locate(app, target, near)?;
let state = loc
.element
.get_pattern::<UITogglePattern>()
.ok()
.and_then(|t| t.get_toggle_state().ok())
.map(|s| format!("{s:?}"));
if state.as_deref() != Some("On") {
return Err(fail("checked", state));
}
Ok(VerifyOutcome {
located: loc,
actual: state,
mode: "checked",
})
}
}
}
fn read_value(elem: &UIElement) -> Option<String> {
elem.get_pattern::<UIValuePattern>()
.ok()
.and_then(|v| v.get_value().ok())
}
pub fn wait_gone(app: Option<&str>, target: &Target, timeout_ms: u64) -> Result<(), CtlError> {
let deadline = std::time::Instant::now() + std::time::Duration::from_millis(timeout_ms);
loop {
if locate(app, target, None).is_err() {
return Ok(());
}
if std::time::Instant::now() >= deadline {
return Err(CtlError::new(
ErrorCode::Timeout,
format!(
"{} still resolves after {timeout_ms}ms (expected it to disappear)",
target.describe()
),
));
}
std::thread::sleep(std::time::Duration::from_millis(timing().poll_ms));
}
}
pub fn extract_table(app: Option<&str>, target: &Target) -> Result<Vec<Vec<String>>, CtlError> {
let loc = locate(app, target, None)?;
let auto = UIAutomation::new().map_err(internal)?;
let walker = auto.create_tree_walker().map_err(internal)?;
fn rows_under(walker: &UITreeWalker, elem: &UIElement, depth: u32, out: &mut Vec<UIElement>) {
if depth > MAX_DEPTH {
return;
}
let mut child = walker.get_first_child(elem).ok();
while let Some(c) = child {
let role = role_of(&c);
if role == "DataItem" || role == "ListItem" {
out.push(c.clone());
} else {
rows_under(walker, &c, depth + 1, out);
}
child = walker.get_next_sibling(&c).ok();
}
}
fn cells_under(walker: &UITreeWalker, elem: &UIElement, depth: u32, out: &mut Vec<String>) {
if depth > MAX_DEPTH {
return;
}
let role = role_of(elem);
if role == "Text" || role == "Edit" {
if let Some(n) = elem.get_name().ok().filter(|n| !n.is_empty()) {
out.push(n);
}
}
let mut child = walker.get_first_child(elem).ok();
while let Some(c) = child {
cells_under(walker, &c, depth + 1, out);
child = walker.get_next_sibling(&c).ok();
}
}
let mut row_elems = Vec::new();
rows_under(&walker, &loc.element, 0, &mut row_elems);
if row_elems.is_empty() {
return Err(CtlError::new(
ErrorCode::NotActionable,
format!(
"{} ({}) contains no DataItem/ListItem rows",
target.describe(),
loc.role
),
));
}
let mut table = Vec::new();
for r in &row_elems {
let mut cells = Vec::new();
cells_under(&walker, r, 0, &mut cells);
if !cells.is_empty() {
table.push(cells);
}
}
Ok(table)
}
pub fn check_snapshot_freshness_by_pattern(
snapshot_id: &str,
pattern: &str,
) -> Result<(), CtlError> {
let auto = UIAutomation::new().map_err(internal)?;
let walker = auto.create_tree_walker().map_err(internal)?;
let win = find_window(&auto, &walker, pattern)?;
check_snapshot_freshness(snapshot_id, &win)
}