#![cfg(target_os = "windows")]
use std::path::Path;
use std::time::{Duration, Instant};
use std::{mem, thread};
use serde_json::{json, Value};
use windows::core::{BSTR, VARIANT};
use windows::Win32::Foundation::{BOOL, HWND, LPARAM, RECT, TRUE};
use windows::Win32::Graphics::Gdi::{
BitBlt, CreateCompatibleBitmap, CreateCompatibleDC, DeleteDC, DeleteObject, GetDC, GetDIBits,
ReleaseDC, SelectObject, BITMAPINFO, BITMAPINFOHEADER, BI_RGB, DIB_RGB_COLORS, SRCCOPY,
};
use windows::Win32::System::Com::{
CoCreateInstance, CoInitializeEx, CLSCTX_ALL, COINIT_MULTITHREADED,
};
use windows::Win32::UI::Accessibility::{
CUIAutomation, IUIAutomation, IUIAutomationCondition, IUIAutomationElement,
IUIAutomationExpandCollapsePattern, IUIAutomationInvokePattern,
IUIAutomationTogglePattern, IUIAutomationValuePattern, TreeScope_Children,
TreeScope_Descendants, UIA_PATTERN_ID, UIA_PROPERTY_ID, UIA_CONTROLTYPE_ID,
};
use windows::Win32::UI::Input::KeyboardAndMouse::{
SendInput, INPUT, INPUT_0, INPUT_KEYBOARD, INPUT_MOUSE, KEYBDINPUT, KEYEVENTF_KEYUP,
KEYEVENTF_UNICODE, MOUSEEVENTF_ABSOLUTE, MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP,
MOUSEEVENTF_MOVE, MOUSEINPUT, VIRTUAL_KEY,
};
use windows::Win32::UI::WindowsAndMessaging::{
EnumWindows, GetClientRect, GetWindowTextW,
GetWindowThreadProcessId, IsWindowVisible, SetForegroundWindow,
SM_CXSCREEN, SM_CYSCREEN, GetSystemMetrics,
};
use windows::Win32::Graphics::Gdi::HDC;
const PW_CLIENTONLY: u32 = 0x00000001;
const PW_RENDERFULLCONTENT: u32 = 0x00000002;
#[link(name = "user32")]
extern "system" {
fn PrintWindow(hwnd: HWND, hdc_blt: HDC, flags: u32) -> BOOL;
}
use crate::selector::{self, Condition, Selector, Step};
const PROP_NAME: UIA_PROPERTY_ID = UIA_PROPERTY_ID(30005);
const PROP_AUTOMATION_ID: UIA_PROPERTY_ID = UIA_PROPERTY_ID(30011);
const PROP_CLASS_NAME: UIA_PROPERTY_ID = UIA_PROPERTY_ID(30012);
const PROP_CONTROL_TYPE: UIA_PROPERTY_ID = UIA_PROPERTY_ID(30003);
const PAT_INVOKE: UIA_PATTERN_ID = UIA_PATTERN_ID(10000);
const PAT_VALUE: UIA_PATTERN_ID = UIA_PATTERN_ID(10002);
const PAT_TOGGLE: UIA_PATTERN_ID = UIA_PATTERN_ID(10015);
const PAT_EXPAND_COLLAPSE: UIA_PATTERN_ID = UIA_PATTERN_ID(10005);
type UiaResult<T> = Result<T, Box<dyn std::error::Error>>;
pub struct Uia {
automation: IUIAutomation,
window: IUIAutomationElement,
hwnd: HWND,
}
impl Uia {
pub fn connect(window_spec: &str) -> UiaResult<Self> {
unsafe { CoInitializeEx(None, COINIT_MULTITHREADED).ok()? };
let automation: IUIAutomation =
unsafe { CoCreateInstance(&CUIAutomation, None, CLSCTX_ALL)? };
let hwnd = find_window(window_spec)?;
let window = unsafe { automation.ElementFromHandle(hwnd)? };
Ok(Uia {
automation,
window,
hwnd,
})
}
pub fn click(&self, selector_str: &str) -> UiaResult<Value> {
let element = self.find(selector_str)?;
let name = get_name(&element);
if let Ok(pattern) = unsafe {
element.GetCurrentPatternAs::<IUIAutomationInvokePattern>(PAT_INVOKE)
} {
if unsafe { pattern.Invoke() }.is_ok() {
return Ok(json!({
"result": "invoked",
"name": name,
}));
}
}
if let Ok(pattern) = unsafe {
element.GetCurrentPatternAs::<IUIAutomationTogglePattern>(PAT_TOGGLE)
} {
if unsafe { pattern.Toggle() }.is_ok() {
return Ok(json!({
"result": "toggled",
"name": name,
}));
}
}
let rect = get_rect(&element)?;
let cx = (rect.left + rect.right) / 2;
let cy = (rect.top + rect.bottom) / 2;
click_at(cx, cy)?;
Ok(json!({
"result": "clicked_at",
"name": name,
"x": cx,
"y": cy,
}))
}
pub fn type_text(&self, selector_str: &str, text: &str) -> UiaResult<Value> {
let element = self.find(selector_str)?;
let name = get_name(&element);
if let Ok(pattern) = unsafe {
element.GetCurrentPatternAs::<IUIAutomationValuePattern>(PAT_VALUE)
} {
unsafe { pattern.SetValue(&BSTR::from(text))? };
return Ok(json!({
"result": "set_value",
"name": name,
"text": text,
}));
}
unsafe { element.SetFocus()? };
thread::sleep(Duration::from_millis(50));
send_string(text)?;
Ok(json!({
"result": "typed_keys",
"name": name,
"text": text,
}))
}
pub fn query(&self, selector_str: &str) -> UiaResult<Value> {
let element = self.find(selector_str)?;
Ok(describe_element(&element))
}
pub fn screenshot(&self, path: &Path) -> UiaResult<()> {
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent).ok();
}
unsafe {
let mut rect = RECT::default();
GetClientRect(self.hwnd, &mut rect)?;
let width = rect.right - rect.left;
let height = rect.bottom - rect.top;
if width <= 0 || height <= 0 {
return Err("window has zero size".into());
}
let hdc_window = GetDC(self.hwnd);
let hdc_mem = CreateCompatibleDC(hdc_window);
let hbm = CreateCompatibleBitmap(hdc_window, width, height);
let old = SelectObject(hdc_mem, hbm);
let ok = PrintWindow(self.hwnd, hdc_mem, PW_CLIENTONLY | PW_RENDERFULLCONTENT);
if !ok.as_bool() {
let _ = BitBlt(hdc_mem, 0, 0, width, height, hdc_window, 0, 0, SRCCOPY);
}
let row_bytes = ((width * 3 + 3) / 4) * 4; let data_size = (row_bytes * height) as usize;
let mut pixels = vec![0u8; data_size];
let mut bmi = BITMAPINFO {
bmiHeader: BITMAPINFOHEADER {
biSize: mem::size_of::<BITMAPINFOHEADER>() as u32,
biWidth: width,
biHeight: height, biPlanes: 1,
biBitCount: 24,
biCompression: BI_RGB.0 as u32,
biSizeImage: data_size as u32,
..Default::default()
},
..Default::default()
};
GetDIBits(
hdc_mem,
hbm,
0,
height as u32,
Some(pixels.as_mut_ptr() as *mut _),
&mut bmi,
DIB_RGB_COLORS,
);
SelectObject(hdc_mem, old);
let _ = DeleteObject(hbm);
let _ = DeleteDC(hdc_mem);
ReleaseDC(self.hwnd, hdc_window);
write_bmp(path, width, height, row_bytes, &pixels)?;
}
Ok(())
}
pub fn tree(&self, selector_str: Option<&str>, depth: u32) -> UiaResult<Value> {
let root = if let Some(sel) = selector_str {
self.find(sel)?
} else {
self.window.clone()
};
Ok(build_tree(&self.automation, &root, depth, 0))
}
pub fn list_windows(&self) -> UiaResult<Value> {
let windows = enumerate_windows()?;
Ok(json!(windows))
}
pub fn wait_for(&self, selector_str: &str, timeout_ms: u64) -> UiaResult<Value> {
let start = Instant::now();
let sel = selector::parse(selector_str).map_err(|e| format!("selector: {}", e))?;
loop {
if let Ok(element) = self.find_parsed(&sel) {
let name = get_name(&element);
return Ok(json!({
"found": true,
"name": name,
"elapsed_ms": start.elapsed().as_millis() as u64,
}));
}
if start.elapsed().as_millis() >= timeout_ms as u128 {
return Ok(json!({
"found": false,
"elapsed_ms": start.elapsed().as_millis() as u64,
"status": "timeout",
}));
}
thread::sleep(Duration::from_millis(200));
}
}
pub fn focus(&self, selector_str: &str) -> UiaResult<Value> {
let element = self.find(selector_str)?;
unsafe { element.SetFocus()? };
Ok(json!({ "result": "focused", "name": get_name(&element) }))
}
pub fn raw_keys(&self, keys: &str) -> UiaResult<Value> {
let _ = unsafe { SetForegroundWindow(self.hwnd) };
thread::sleep(Duration::from_millis(100));
send_string(keys)?;
Ok(json!({ "result": "sent", "keys": keys }))
}
pub fn expand(&self, selector_str: &str) -> UiaResult<Value> {
let element = self.find(selector_str)?;
let pattern = unsafe {
element
.GetCurrentPatternAs::<IUIAutomationExpandCollapsePattern>(
PAT_EXPAND_COLLAPSE,
)?
};
unsafe { pattern.Expand()? };
Ok(json!({ "result": "expanded", "name": get_name(&element) }))
}
fn find(&self, selector_str: &str) -> UiaResult<IUIAutomationElement> {
let sel = selector::parse(selector_str).map_err(|e| format!("selector: {}", e))?;
self.find_parsed(&sel)
}
fn find_parsed(&self, sel: &Selector) -> UiaResult<IUIAutomationElement> {
let mut current = self.window.clone();
for step in &sel.steps {
let condition = self.build_condition(step)?;
let scope = if sel.is_single_step() {
TreeScope_Descendants
} else {
TreeScope_Children
};
if let Some(idx) = step.index {
let all = unsafe { current.FindAll(scope, &condition)? };
let len = unsafe { all.Length()? };
if idx >= len as usize {
return Err(format!(
"index [{}] out of range (found {} elements)",
idx, len
)
.into());
}
current = unsafe { all.GetElement(idx as i32)? };
} else {
current = unsafe {
current.FindFirst(scope, &condition)?
};
}
}
Ok(current)
}
fn build_condition(&self, step: &Step) -> UiaResult<IUIAutomationCondition> {
let mut conditions: Vec<IUIAutomationCondition> = Vec::new();
for cond in &step.conditions {
let c = match cond {
Condition::Name(name) => {
if name.contains('*') {
let clean = name.replace('*', "");
unsafe {
self.automation.CreatePropertyCondition(
PROP_NAME,
&VARIANT::from(BSTR::from(clean.as_str())),
)?
}
} else {
unsafe {
self.automation.CreatePropertyCondition(
PROP_NAME,
&VARIANT::from(BSTR::from(name.as_str())),
)?
}
}
}
Condition::AutomationId(aid) => unsafe {
self.automation.CreatePropertyCondition(
PROP_AUTOMATION_ID,
&VARIANT::from(BSTR::from(aid.as_str())),
)?
},
Condition::ControlType(type_name) => {
let type_id = selector::control_type_id(type_name).ok_or_else(|| {
format!("unknown control type: {}", type_name)
})?;
unsafe {
self.automation.CreatePropertyCondition(
PROP_CONTROL_TYPE,
&VARIANT::from(type_id),
)?
}
}
Condition::ClassName(class) => unsafe {
self.automation.CreatePropertyCondition(
PROP_CLASS_NAME,
&VARIANT::from(BSTR::from(class.as_str())),
)?
},
};
conditions.push(c);
}
if conditions.len() == 1 {
Ok(conditions.into_iter().next().unwrap())
} else {
let mut result = conditions[0].clone();
for c in &conditions[1..] {
result = unsafe { self.automation.CreateAndCondition(&result, c)? };
}
Ok(result)
}
}
}
fn find_window(spec: &str) -> UiaResult<HWND> {
if let Some(pid_str) = spec.strip_prefix("pid:") {
let target_pid: u32 = pid_str.parse().map_err(|_| format!("invalid pid: {}", pid_str))?;
let windows = enumerate_windows()?;
for w in &windows {
if w.get("pid").and_then(|v| v.as_u64()) == Some(target_pid as u64) {
let hwnd_val = w.get("hwnd").and_then(|v| v.as_u64()).unwrap_or(0);
return Ok(HWND(hwnd_val as *mut _));
}
}
return Err(format!("no window for pid {}", target_pid).into());
}
if let Some(class) = spec.strip_prefix("class:") {
let wide: Vec<u16> = class.encode_utf16().chain(std::iter::once(0)).collect();
let hwnd = unsafe {
windows::Win32::UI::WindowsAndMessaging::FindWindowW(
windows::core::PCWSTR(wide.as_ptr()),
None,
)?
};
if hwnd.0.is_null() {
return Err(format!("no window with class '{}'", class).into());
}
return Ok(hwnd);
}
let is_wildcard = spec.contains('*');
let pattern = spec.replace('*', "");
let windows = enumerate_windows()?;
for w in &windows {
let title = w.get("title").and_then(|v| v.as_str()).unwrap_or("");
let matches = if is_wildcard {
title.to_lowercase().contains(&pattern.to_lowercase())
} else {
title == spec
};
if matches {
let hwnd_val = w.get("hwnd").and_then(|v| v.as_u64()).unwrap_or(0);
return Ok(HWND(hwnd_val as *mut _));
}
}
Err(format!("no window matching '{}'", spec).into())
}
fn enumerate_windows() -> UiaResult<Vec<Value>> {
let mut results: Vec<Value> = Vec::new();
unsafe extern "system" fn enum_callback(hwnd: HWND, lparam: LPARAM) -> BOOL {
let results = &mut *(lparam.0 as *mut Vec<Value>);
if !IsWindowVisible(hwnd).as_bool() {
return TRUE;
}
let mut title_buf = [0u16; 512];
let len = GetWindowTextW(hwnd, &mut title_buf);
if len == 0 {
return TRUE;
}
let title = String::from_utf16_lossy(&title_buf[..len as usize]);
let mut pid = 0u32;
GetWindowThreadProcessId(hwnd, Some(&mut pid));
results.push(json!({
"hwnd": hwnd.0 as u64,
"title": title,
"pid": pid,
}));
TRUE
}
unsafe {
EnumWindows(
Some(enum_callback),
LPARAM(&mut results as *mut Vec<Value> as isize),
)?;
}
Ok(results)
}
fn get_name(element: &IUIAutomationElement) -> String {
unsafe {
element
.CurrentName()
.map(|b| b.to_string())
.unwrap_or_default()
}
}
fn get_rect(element: &IUIAutomationElement) -> UiaResult<RECT> {
unsafe {
let r = element.CurrentBoundingRectangle()?;
Ok(r)
}
}
fn describe_element(element: &IUIAutomationElement) -> Value {
unsafe {
let name = element.CurrentName().map(|b| b.to_string()).unwrap_or_default();
let aid = element
.CurrentAutomationId()
.map(|b| b.to_string())
.unwrap_or_default();
let class = element
.CurrentClassName()
.map(|b| b.to_string())
.unwrap_or_default();
let control_type = element.CurrentControlType().unwrap_or(UIA_CONTROLTYPE_ID(0));
let ct_id = control_type.0;
let enabled = element.CurrentIsEnabled().map(|b| b.as_bool()).unwrap_or(false);
let focused = element
.CurrentHasKeyboardFocus()
.map(|b| b.as_bool())
.unwrap_or(false);
let rect = element.CurrentBoundingRectangle().unwrap_or(RECT::default());
let value = element
.GetCurrentPatternAs::<IUIAutomationValuePattern>(PAT_VALUE)
.ok()
.and_then(|p| p.CurrentValue().ok())
.map(|b| b.to_string());
json!({
"name": name,
"automationId": aid,
"className": class,
"controlType": selector::control_type_name(ct_id),
"controlTypeId": ct_id,
"enabled": enabled,
"focused": focused,
"value": value,
"rect": {
"x": rect.left,
"y": rect.top,
"w": rect.right - rect.left,
"h": rect.bottom - rect.top,
},
})
}
}
fn build_tree(
automation: &IUIAutomation,
element: &IUIAutomationElement,
max_depth: u32,
current_depth: u32,
) -> Value {
let mut node = describe_element(element);
if current_depth < max_depth {
let true_condition = unsafe { automation.CreateTrueCondition() };
if let Ok(cond) = true_condition {
if let Ok(children) = unsafe { element.FindAll(TreeScope_Children, &cond) } {
let count = unsafe { children.Length().unwrap_or(0) };
let mut child_nodes = Vec::new();
for i in 0..count {
if let Ok(child) = unsafe { children.GetElement(i) } {
child_nodes.push(build_tree(automation, &child, max_depth, current_depth + 1));
}
}
if !child_nodes.is_empty() {
node.as_object_mut()
.unwrap()
.insert("children".into(), json!(child_nodes));
}
}
}
}
node
}
fn click_at(x: i32, y: i32) -> UiaResult<()> {
let screen_w = unsafe { GetSystemMetrics(SM_CXSCREEN) };
let screen_h = unsafe { GetSystemMetrics(SM_CYSCREEN) };
let abs_x = (x * 65535) / screen_w;
let abs_y = (y * 65535) / screen_h;
let inputs = [
INPUT {
r#type: INPUT_MOUSE,
Anonymous: INPUT_0 {
mi: MOUSEINPUT {
dx: abs_x,
dy: abs_y,
mouseData: 0,
dwFlags: MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE,
time: 0,
dwExtraInfo: 0,
},
},
},
INPUT {
r#type: INPUT_MOUSE,
Anonymous: INPUT_0 {
mi: MOUSEINPUT {
dx: abs_x,
dy: abs_y,
mouseData: 0,
dwFlags: MOUSEEVENTF_LEFTDOWN | MOUSEEVENTF_ABSOLUTE,
time: 0,
dwExtraInfo: 0,
},
},
},
INPUT {
r#type: INPUT_MOUSE,
Anonymous: INPUT_0 {
mi: MOUSEINPUT {
dx: abs_x,
dy: abs_y,
mouseData: 0,
dwFlags: MOUSEEVENTF_LEFTUP | MOUSEEVENTF_ABSOLUTE,
time: 0,
dwExtraInfo: 0,
},
},
},
];
unsafe {
SendInput(&inputs, mem::size_of::<INPUT>() as i32);
}
thread::sleep(Duration::from_millis(50));
Ok(())
}
fn send_string(text: &str) -> UiaResult<()> {
for c in text.chars() {
let scan = c as u16;
let inputs = [
INPUT {
r#type: INPUT_KEYBOARD,
Anonymous: INPUT_0 {
ki: KEYBDINPUT {
wVk: VIRTUAL_KEY(0),
wScan: scan,
dwFlags: KEYEVENTF_UNICODE,
time: 0,
dwExtraInfo: 0,
},
},
},
INPUT {
r#type: INPUT_KEYBOARD,
Anonymous: INPUT_0 {
ki: KEYBDINPUT {
wVk: VIRTUAL_KEY(0),
wScan: scan,
dwFlags: KEYEVENTF_UNICODE | KEYEVENTF_KEYUP,
time: 0,
dwExtraInfo: 0,
},
},
},
];
unsafe {
SendInput(&inputs, mem::size_of::<INPUT>() as i32);
}
thread::sleep(Duration::from_millis(10));
}
Ok(())
}
fn write_bmp(path: &Path, width: i32, height: i32, _row_bytes: i32, pixels: &[u8]) -> UiaResult<()> {
let data_size = pixels.len() as u32;
let file_size = 14 + 40 + data_size;
let mut buf = Vec::with_capacity(file_size as usize);
buf.extend_from_slice(b"BM");
buf.extend_from_slice(&file_size.to_le_bytes());
buf.extend_from_slice(&[0u8; 4]); buf.extend_from_slice(&54u32.to_le_bytes());
buf.extend_from_slice(&40u32.to_le_bytes());
buf.extend_from_slice(&width.to_le_bytes());
buf.extend_from_slice(&height.to_le_bytes());
buf.extend_from_slice(&1u16.to_le_bytes()); buf.extend_from_slice(&24u16.to_le_bytes()); buf.extend_from_slice(&0u32.to_le_bytes()); buf.extend_from_slice(&data_size.to_le_bytes());
buf.extend_from_slice(&2835u32.to_le_bytes()); buf.extend_from_slice(&2835u32.to_le_bytes()); buf.extend_from_slice(&0u32.to_le_bytes()); buf.extend_from_slice(&0u32.to_le_bytes());
buf.extend_from_slice(pixels);
std::fs::write(path, buf)?;
Ok(())
}