use async_trait::async_trait;
use crate::backend::DesktopBackend;
use crate::errors::{CarDesktopError, Result};
use crate::models::{
ClickRequest, DisplayId, Frame, KeyPressRequest, PermissionRequest, PermissionSnapshot,
TypeRequest, UiMap, WindowFilter, WindowHandle, WindowInfo,
};
#[cfg(target_os = "windows")]
use crate::models::{A11yElementRecord, A11yNode, Bounds, Key, MouseButton, WindowFrame};
#[cfg(target_os = "windows")]
use crate::perception::{AX_DEPTH_CAP, AX_NODE_CAP};
#[cfg(target_os = "windows")]
use car_browser::models::Modifier;
#[cfg(target_os = "windows")]
mod imp {
use super::*;
use windows::Win32::Foundation::{BOOL, HWND, LPARAM, RECT, TRUE};
use windows::Win32::Graphics::Gdi::{
BitBlt, CreateCompatibleBitmap, CreateCompatibleDC, DeleteDC, DeleteObject, GetDC,
GetDIBits, ReleaseDC, SelectObject, BITMAPINFO, BITMAPINFOHEADER, BI_RGB, DIB_RGB_COLORS,
HGDIOBJ, SRCCOPY,
};
use windows::Win32::System::Threading::{
OpenProcess, QueryFullProcessImageNameW, PROCESS_NAME_WIN32,
PROCESS_QUERY_LIMITED_INFORMATION,
};
use windows::Win32::UI::Input::KeyboardAndMouse::{
SendInput, SetFocus, INPUT, INPUT_0, INPUT_KEYBOARD, INPUT_MOUSE, KEYBDINPUT,
KEYBD_EVENT_FLAGS, KEYEVENTF_KEYUP, KEYEVENTF_UNICODE, MOUSEEVENTF_LEFTDOWN,
MOUSEEVENTF_LEFTUP, MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP, MOUSEEVENTF_RIGHTDOWN,
MOUSEEVENTF_RIGHTUP, MOUSEINPUT, VIRTUAL_KEY, VK_BACK, VK_CONTROL, VK_DELETE, VK_DOWN,
VK_END, VK_ESCAPE, VK_F1, VK_F10, VK_F11, VK_F12, VK_F2, VK_F3, VK_F4, VK_F5, VK_F6, VK_F7,
VK_F8, VK_F9, VK_HOME, VK_LEFT, VK_LWIN, VK_MENU, VK_NEXT, VK_OEM_2, VK_OEM_COMMA,
VK_OEM_PERIOD, VK_PRIOR, VK_RETURN, VK_RIGHT, VK_SHIFT, VK_SPACE, VK_TAB, VK_UP,
};
use windows::Win32::UI::WindowsAndMessaging::{
EnumWindows, GetWindowRect, GetWindowTextLengthW, GetWindowTextW, GetWindowThreadProcessId,
IsWindowVisible, SetForegroundWindow, SystemParametersInfoW, SPI_GETWORKAREA,
SYSTEM_PARAMETERS_INFO_UPDATE_FLAGS,
};
struct Collector {
hwnds: Vec<HWND>,
}
unsafe extern "system" fn enum_cb(hwnd: HWND, lparam: LPARAM) -> BOOL {
let collector = &mut *(lparam.0 as *mut Collector);
collector.hwnds.push(hwnd);
TRUE
}
fn wide_to_string(buf: &[u16]) -> String {
let end = buf.iter().position(|&c| c == 0).unwrap_or(buf.len());
String::from_utf16_lossy(&buf[..end])
}
fn window_title(hwnd: HWND) -> String {
unsafe {
let len = GetWindowTextLengthW(hwnd);
if len <= 0 {
return String::new();
}
let mut buf = vec![0u16; (len as usize) + 1];
let got = GetWindowTextW(hwnd, &mut buf);
wide_to_string(&buf[..got as usize])
}
}
fn window_pid(hwnd: HWND) -> u32 {
let mut pid = 0u32;
unsafe {
GetWindowThreadProcessId(hwnd, Some(&mut pid));
}
pid
}
fn process_exe_name(pid: u32) -> Option<String> {
if pid == 0 {
return None;
}
unsafe {
let handle = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, false, pid).ok()?;
let mut buf = vec![0u16; 1024];
let mut size = buf.len() as u32;
let res = QueryFullProcessImageNameW(
handle,
PROCESS_NAME_WIN32,
windows::core::PWSTR(buf.as_mut_ptr()),
&mut size,
);
let _ = windows::Win32::Foundation::CloseHandle(handle);
res.ok()?;
let full = wide_to_string(&buf[..size as usize]);
full.rsplit(['\\', '/']).next().map(|s| s.to_string())
}
}
fn window_frame(hwnd: HWND) -> WindowFrame {
let mut r = RECT::default();
unsafe {
let _ = GetWindowRect(hwnd, &mut r);
}
WindowFrame {
x: r.left as f64,
y: r.top as f64,
width: (r.right - r.left) as f64,
height: (r.bottom - r.top) as f64,
}
}
pub fn list_windows(filter: &WindowFilter) -> Result<Vec<WindowInfo>> {
let mut collector = Collector { hwnds: Vec::new() };
unsafe {
EnumWindows(
Some(enum_cb),
LPARAM(&mut collector as *mut Collector as isize),
)
.map_err(|e| os_err(format!("EnumWindows: {e}")))?;
}
let mut out = Vec::new();
for hwnd in collector.hwnds {
let on_screen = unsafe { IsWindowVisible(hwnd).as_bool() };
let title = window_title(hwnd);
if title.is_empty() && filter.title_contains.is_none() {
continue;
}
let pid = window_pid(hwnd);
let exe = process_exe_name(pid);
let info = WindowInfo {
handle: WindowHandle::new(pid, hwnd.0 as u64),
title,
bundle_id: exe.clone(),
owner_name: exe
.as_deref()
.map(|e| e.trim_end_matches(".exe").to_string())
.unwrap_or_default(),
frame: window_frame(hwnd),
layer: 0,
on_screen,
};
if filter.matches(&info) {
out.push(info);
}
}
Ok(out)
}
pub fn focus_window(window: WindowHandle) -> Result<()> {
let hwnd = HWND(window.window_id as isize);
unsafe {
let _ = SetForegroundWindow(hwnd);
let _ = SetFocus(hwnd);
}
Ok(())
}
fn send_inputs(inputs: &[INPUT]) -> Result<()> {
let sent = unsafe { SendInput(inputs, std::mem::size_of::<INPUT>() as i32) };
if sent as usize != inputs.len() {
return Err(os_err(format!(
"SendInput delivered {sent}/{} events (input may be blocked, e.g. by an elevated window)",
inputs.len()
)));
}
Ok(())
}
fn mouse_input(flags: windows::Win32::UI::Input::KeyboardAndMouse::MOUSE_EVENT_FLAGS) -> INPUT {
INPUT {
r#type: INPUT_MOUSE,
Anonymous: INPUT_0 {
mi: MOUSEINPUT {
dx: 0,
dy: 0,
mouseData: 0,
dwFlags: flags,
time: 0,
dwExtraInfo: 0,
},
},
}
}
pub fn click(request: &ClickRequest) -> Result<()> {
let (x, y) = if let Some(p) = request.point {
p
} else if let Some(eid) = &request.element_id {
resolve_element_point(request.window, eid).ok_or_else(|| {
CarDesktopError::UnknownElement {
element_id: eid.clone(),
}
})?
} else {
let hwnd = HWND(request.window.window_id as isize);
window_frame(hwnd).center()
};
if request.dry_run {
tracing::info!(x, y, "click dry-run");
return Ok(());
}
unsafe {
windows::Win32::UI::WindowsAndMessaging::SetCursorPos(x as i32, y as i32)
.map_err(|e| os_err(format!("SetCursorPos: {e}")))?;
}
let (down, up) = match request.button {
MouseButton::Left => (MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP),
MouseButton::Right => (MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP),
MouseButton::Middle => (MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP),
};
send_inputs(&[mouse_input(down), mouse_input(up)])
}
fn key_input(vk: VIRTUAL_KEY, up: bool) -> INPUT {
INPUT {
r#type: INPUT_KEYBOARD,
Anonymous: INPUT_0 {
ki: KEYBDINPUT {
wVk: vk,
wScan: 0,
dwFlags: if up {
KEYEVENTF_KEYUP
} else {
KEYBD_EVENT_FLAGS(0)
},
time: 0,
dwExtraInfo: 0,
},
},
}
}
fn unicode_input(unit: u16, up: bool) -> INPUT {
INPUT {
r#type: INPUT_KEYBOARD,
Anonymous: INPUT_0 {
ki: KEYBDINPUT {
wVk: VIRTUAL_KEY(0),
wScan: unit,
dwFlags: KEYEVENTF_UNICODE
| if up {
KEYEVENTF_KEYUP
} else {
KEYBD_EVENT_FLAGS(0)
},
time: 0,
dwExtraInfo: 0,
},
},
}
}
pub fn type_text(request: &TypeRequest) -> Result<()> {
if request.dry_run {
tracing::info!(len = request.text.len(), "type_text dry-run");
return Ok(());
}
let mut inputs: Vec<INPUT> = Vec::new();
for ch in request.text.chars() {
if ch == '\n' {
inputs.push(key_input(VK_RETURN, false));
inputs.push(key_input(VK_RETURN, true));
continue;
}
let mut buf = [0u16; 2];
for unit in ch.encode_utf16(&mut buf) {
inputs.push(unicode_input(*unit, false));
inputs.push(unicode_input(*unit, true));
}
}
if inputs.is_empty() {
return Ok(());
}
send_inputs(&inputs)
}
fn modifier_vk(m: Modifier) -> VIRTUAL_KEY {
match m {
Modifier::Shift => VK_SHIFT,
Modifier::Control => VK_CONTROL,
Modifier::Alt => VK_MENU,
Modifier::Meta => VK_LWIN,
}
}
fn key_vk(key: Key) -> Option<VIRTUAL_KEY> {
Some(match key {
Key::Return => VK_RETURN,
Key::Escape => VK_ESCAPE,
Key::Tab => VK_TAB,
Key::Space => VK_SPACE,
Key::Backspace => VK_BACK,
Key::Delete => VK_DELETE,
Key::ArrowUp => VK_UP,
Key::ArrowDown => VK_DOWN,
Key::ArrowLeft => VK_LEFT,
Key::ArrowRight => VK_RIGHT,
Key::Home => VK_HOME,
Key::End => VK_END,
Key::PageUp => VK_PRIOR,
Key::PageDown => VK_NEXT,
Key::F1 => VK_F1,
Key::F2 => VK_F2,
Key::F3 => VK_F3,
Key::F4 => VK_F4,
Key::F5 => VK_F5,
Key::F6 => VK_F6,
Key::F7 => VK_F7,
Key::F8 => VK_F8,
Key::F9 => VK_F9,
Key::F10 => VK_F10,
Key::F11 => VK_F11,
Key::F12 => VK_F12,
Key::Comma => VK_OEM_COMMA,
Key::Period => VK_OEM_PERIOD,
Key::Slash => VK_OEM_2,
Key::Char(_) => return None,
})
}
pub fn keypress(request: &KeyPressRequest) -> Result<()> {
if request.dry_run {
tracing::info!(?request.key, "keypress dry-run");
return Ok(());
}
let mods: Vec<VIRTUAL_KEY> = request.modifiers.iter().map(|m| modifier_vk(*m)).collect();
let mut inputs: Vec<INPUT> = Vec::new();
for vk in &mods {
inputs.push(key_input(*vk, false));
}
match key_vk(request.key) {
Some(vk) => {
inputs.push(key_input(vk, false));
inputs.push(key_input(vk, true));
}
None => {
if let Key::Char(c) = request.key {
let mut buf = [0u16; 2];
for unit in c.encode_utf16(&mut buf) {
inputs.push(unicode_input(*unit, false));
inputs.push(unicode_input(*unit, true));
}
}
}
}
for vk in mods.iter().rev() {
inputs.push(key_input(*vk, true));
}
send_inputs(&inputs)
}
fn capture_rect(x: i32, y: i32, w: i32, h: i32) -> Result<Frame> {
if w <= 0 || h <= 0 {
return Err(os_err("capture rect is empty".into()));
}
unsafe {
let screen = GetDC(HWND(0));
if screen.0 == 0 {
return Err(os_err("GetDC(screen) failed".into()));
}
let mem = CreateCompatibleDC(screen);
let bmp = CreateCompatibleBitmap(screen, w, h);
let old = SelectObject(mem, HGDIOBJ(bmp.0));
let blt = BitBlt(mem, 0, 0, w, h, screen, x, y, SRCCOPY);
let mut info = BITMAPINFO {
bmiHeader: BITMAPINFOHEADER {
biSize: std::mem::size_of::<BITMAPINFOHEADER>() as u32,
biWidth: w,
biHeight: -h, biPlanes: 1,
biBitCount: 32,
biCompression: BI_RGB.0,
..Default::default()
},
..Default::default()
};
let mut buf = vec![0u8; (w * h * 4) as usize];
let scanlines = GetDIBits(
mem,
bmp,
0,
h as u32,
Some(buf.as_mut_ptr() as *mut _),
&mut info,
DIB_RGB_COLORS,
);
SelectObject(mem, old);
let _ = DeleteObject(bmp);
let _ = DeleteDC(mem);
ReleaseDC(HWND(0), screen);
if blt.is_err() || scanlines == 0 {
return Err(os_err("BitBlt/GetDIBits failed".into()));
}
for px in buf.chunks_exact_mut(4) {
px.swap(0, 2);
}
let frame = Frame {
width: w as u32,
height: h as u32,
scale_factor: 1.0,
rgba: buf,
captured_at: chrono::Utc::now(),
};
frame.validate().map_err(os_err)?;
Ok(frame)
}
}
pub fn capture_display(_display: DisplayId) -> Result<Frame> {
let mut work = RECT::default();
unsafe {
let _ = SystemParametersInfoW(
SPI_GETWORKAREA,
0,
Some(&mut work as *mut RECT as *mut _),
SYSTEM_PARAMETERS_INFO_UPDATE_FLAGS(0),
);
}
let (w, h) = (work.right - work.left, work.bottom - work.top);
capture_rect(work.left, work.top, w.max(1), h.max(1))
}
pub fn capture_window(window: WindowHandle) -> Result<Frame> {
let hwnd = HWND(window.window_id as isize);
let f = window_frame(hwnd);
capture_rect(f.x as i32, f.y as i32, f.width as i32, f.height as i32)
}
use std::collections::{HashMap, VecDeque};
use windows::Win32::System::Com::{
CoCreateInstance, CoInitializeEx, CLSCTX_INPROC_SERVER, COINIT_MULTITHREADED,
};
use windows::Win32::UI::Accessibility::{
CUIAutomation, IUIAutomation, IUIAutomationElement, IUIAutomationTreeWalker,
IUIAutomationValuePattern, UIA_ValuePatternId, UIA_CONTROLTYPE_ID,
};
pub struct UiaWalk {
pub root: A11yNode,
pub index: Vec<String>,
pub by_id: HashMap<String, A11yElementRecord>,
pub truncated: bool,
}
fn control_type_role(ct: UIA_CONTROLTYPE_ID) -> String {
let name = match ct.0 {
50000 => "Button",
50001 => "Calendar",
50002 => "CheckBox",
50003 => "ComboBox",
50004 => "Edit",
50005 => "Hyperlink",
50006 => "Image",
50007 => "ListItem",
50008 => "List",
50009 => "Menu",
50010 => "MenuBar",
50011 => "MenuItem",
50012 => "ProgressBar",
50013 => "RadioButton",
50014 => "ScrollBar",
50015 => "Slider",
50016 => "Spinner",
50017 => "StatusBar",
50018 => "Tab",
50019 => "TabItem",
50020 => "Text",
50021 => "ToolBar",
50022 => "ToolTip",
50023 => "Tree",
50024 => "TreeItem",
50025 => "Custom",
50026 => "Group",
50027 => "Thumb",
50028 => "DataGrid",
50029 => "DataItem",
50030 => "Document",
50031 => "SplitButton",
50032 => "Window",
50033 => "Pane",
50034 => "Header",
50035 => "HeaderItem",
50036 => "Table",
50037 => "TitleBar",
50038 => "Separator",
50039 => "SemanticZoom",
50040 => "AppBar",
other => return format!("ControlType{other}"),
};
name.to_string()
}
fn uia_children(
walker: &IUIAutomationTreeWalker,
el: &IUIAutomationElement,
node_id: &str,
) -> (Vec<String>, Vec<IUIAutomationElement>) {
let mut ids = Vec::new();
let mut els = Vec::new();
unsafe {
let mut cur = walker.GetFirstChildElement(el).ok();
let mut i = 0usize;
while let Some(c) = cur {
ids.push(format!("{node_id}/{i}"));
let next = walker.GetNextSiblingElement(&c).ok();
els.push(c);
i += 1;
if els.len() >= AX_NODE_CAP {
break;
}
cur = next;
}
}
(ids, els)
}
fn describe_uia(
el: &IUIAutomationElement,
walker: &IUIAutomationTreeWalker,
node_id: &str,
) -> (A11yNode, Vec<IUIAutomationElement>) {
unsafe {
let name = el
.CurrentName()
.ok()
.map(|b| b.to_string())
.filter(|s| !s.is_empty());
let role = el
.CurrentControlType()
.map(control_type_role)
.unwrap_or_else(|_| "Unknown".to_string());
let rect = el.CurrentBoundingRectangle().unwrap_or_default();
let bounds = Bounds::new(
rect.left as f64,
rect.top as f64,
(rect.right - rect.left) as f64,
(rect.bottom - rect.top) as f64,
);
let disabled = !el.CurrentIsEnabled().map(|b| b.as_bool()).unwrap_or(true);
let focused = el
.CurrentHasKeyboardFocus()
.map(|b| b.as_bool())
.unwrap_or(false);
let focusable = el
.CurrentIsKeyboardFocusable()
.map(|b| b.as_bool())
.unwrap_or(false);
let value = el
.GetCurrentPatternAs::<IUIAutomationValuePattern>(UIA_ValuePatternId)
.ok()
.and_then(|vp| vp.CurrentValue().ok())
.map(|b| b.to_string())
.filter(|s| !s.is_empty());
let (child_ids, child_els) = uia_children(walker, el, node_id);
(
A11yNode {
node_id: node_id.to_string(),
role,
name,
value,
bounds,
children: child_ids,
focusable,
focused,
disabled,
},
child_els,
)
}
}
pub fn walk_window_uia(window: WindowHandle) -> Result<UiaWalk> {
unsafe {
let _ = CoInitializeEx(None, COINIT_MULTITHREADED);
let automation: IUIAutomation =
CoCreateInstance(&CUIAutomation, None, CLSCTX_INPROC_SERVER)
.map_err(|e| os_err(format!("create UIAutomation: {e}")))?;
let hwnd = HWND(window.window_id as isize);
let root_el = automation
.ElementFromHandle(hwnd)
.map_err(|e| os_err(format!("ElementFromHandle: {e}")))?;
let walker = automation
.ControlViewWalker()
.map_err(|e| os_err(format!("ControlViewWalker: {e}")))?;
let mut queue: VecDeque<(IUIAutomationElement, usize, String)> = VecDeque::new();
queue.push_back((root_el, 0, "root".to_string()));
let mut index: Vec<String> = Vec::new();
let mut by_id: HashMap<String, A11yElementRecord> = HashMap::new();
let mut truncated = false;
let mut root_node: Option<A11yNode> = None;
while let Some((el, depth, id)) = queue.pop_front() {
if depth >= AX_DEPTH_CAP || index.len() >= AX_NODE_CAP {
truncated = true;
continue;
}
let (node, children) = describe_uia(&el, &walker, &id);
index.push(node.node_id.clone());
by_id.insert(
node.node_id.clone(),
A11yElementRecord {
bounds: node.bounds,
role: node.role.clone(),
name: node.name.clone(),
value: node.value.clone(),
focusable: node.focusable,
focused: node.focused,
disabled: node.disabled,
},
);
if root_node.is_none() {
root_node = Some(node.clone());
}
for (i, child) in children.into_iter().enumerate() {
if index.len() >= AX_NODE_CAP {
truncated = true;
break;
}
queue.push_back((child, depth + 1, format!("{id}/{i}")));
}
}
let root =
root_node.ok_or_else(|| os_err("UIA walk produced no root node".to_string()))?;
Ok(UiaWalk {
root,
index,
by_id,
truncated,
})
}
}
pub fn resolve_element_point(window: WindowHandle, element_id: &str) -> Option<(f64, f64)> {
let walk = walk_window_uia(window).ok()?;
let rec = walk.by_id.get(element_id)?;
Some((
rec.bounds.x + rec.bounds.width / 2.0,
rec.bounds.y + rec.bounds.height / 2.0,
))
}
}
#[cfg(not(target_os = "windows"))]
const PLATFORM: &str = "windows";
#[allow(dead_code)]
fn os_err(detail: String) -> CarDesktopError {
CarDesktopError::OsApi {
detail,
source: None,
}
}
pub struct WindowsBackend;
impl WindowsBackend {
pub fn new() -> Self {
Self
}
}
impl Default for WindowsBackend {
fn default() -> Self {
Self::new()
}
}
#[cfg(not(target_os = "windows"))]
fn unsupported<T>() -> Result<T> {
Err(CarDesktopError::PlatformUnsupported { platform: PLATFORM })
}
#[async_trait]
impl DesktopBackend for WindowsBackend {
async fn list_windows(&self, filter: WindowFilter) -> Result<Vec<WindowInfo>> {
#[cfg(target_os = "windows")]
{
imp::list_windows(&filter)
}
#[cfg(not(target_os = "windows"))]
{
let _ = filter;
unsupported()
}
}
async fn observe_window(&self, window: WindowHandle) -> Result<UiMap> {
#[cfg(target_os = "windows")]
{
let windows = imp::list_windows(&WindowFilter {
pid: Some(window.pid),
visible_only: false,
..WindowFilter::default()
})?;
let info = windows
.into_iter()
.find(|w| w.handle.window_id == window.window_id)
.ok_or_else(|| os_err("window not found".into()))?;
let frame = imp::capture_window(window).ok();
match imp::walk_window_uia(window) {
Ok(walk) => Ok(UiMap {
window: info,
frame,
a11y_root: Some(walk.root),
a11y_index: walk.index,
a11y_by_id: walk.by_id,
a11y_truncated: walk.truncated,
a11y_empty: false,
observed_at: chrono::Utc::now(),
}),
Err(_) => Ok(UiMap {
window: info,
frame,
a11y_root: None,
a11y_index: Vec::new(),
a11y_by_id: std::collections::HashMap::new(),
a11y_truncated: false,
a11y_empty: true,
observed_at: chrono::Utc::now(),
}),
}
}
#[cfg(not(target_os = "windows"))]
{
let _ = window;
unsupported()
}
}
async fn capture_display(&self, display: DisplayId) -> Result<Frame> {
#[cfg(target_os = "windows")]
{
imp::capture_display(display)
}
#[cfg(not(target_os = "windows"))]
{
let _ = display;
unsupported()
}
}
async fn focus_window(&self, window: WindowHandle) -> Result<()> {
#[cfg(target_os = "windows")]
{
imp::focus_window(window)
}
#[cfg(not(target_os = "windows"))]
{
let _ = window;
unsupported()
}
}
async fn click(&self, request: ClickRequest) -> Result<()> {
#[cfg(target_os = "windows")]
{
imp::click(&request)
}
#[cfg(not(target_os = "windows"))]
{
let _ = request;
unsupported()
}
}
async fn type_text(&self, request: TypeRequest) -> Result<()> {
#[cfg(target_os = "windows")]
{
imp::type_text(&request)
}
#[cfg(not(target_os = "windows"))]
{
let _ = request;
unsupported()
}
}
async fn keypress(&self, request: KeyPressRequest) -> Result<()> {
#[cfg(target_os = "windows")]
{
imp::keypress(&request)
}
#[cfg(not(target_os = "windows"))]
{
let _ = request;
unsupported()
}
}
async fn permissions(&self) -> Result<PermissionSnapshot> {
Ok(PermissionSnapshot {
screen_recording: true,
accessibility: true,
needs_restart: false,
})
}
async fn request_permissions(&self, _needs: PermissionRequest) -> Result<PermissionSnapshot> {
self.permissions().await
}
}
#[cfg(all(test, target_os = "windows"))]
mod win_tests {
use super::*;
#[tokio::test]
async fn permissions_all_granted() {
let p = WindowsBackend::new().permissions().await.unwrap();
assert!(p.screen_recording && p.accessibility && !p.needs_restart);
}
#[tokio::test]
#[ignore = "needs an interactive desktop session; run explicitly on Windows"]
async fn lists_real_windows() {
let wins = WindowsBackend::new()
.list_windows(WindowFilter::default())
.await
.unwrap();
assert!(!wins.is_empty(), "expected some visible titled windows");
for w in wins.iter().take(8) {
eprintln!(
"win: '{}' [{}] pid={} {}x{}",
w.title,
w.bundle_id.as_deref().unwrap_or("?"),
w.handle.pid,
w.frame.width as i32,
w.frame.height as i32
);
}
}
#[tokio::test]
#[ignore = "needs an interactive desktop session; run explicitly on Windows"]
async fn captures_primary_display() {
let f = WindowsBackend::new()
.capture_display(DisplayId::PRIMARY)
.await
.unwrap();
assert!(f.width > 0 && f.height > 0);
assert_eq!(f.rgba.len(), (f.width * f.height * 4) as usize);
eprintln!("captured {}x{} ({} bytes)", f.width, f.height, f.rgba.len());
}
#[tokio::test]
#[ignore = "needs an interactive desktop session; run explicitly on Windows"]
async fn uia_a11y_tree_populates() {
let b = WindowsBackend::new();
let wins = b.list_windows(WindowFilter::default()).await.unwrap();
let mut found = false;
for w in wins.iter().filter(|w| !w.title.is_empty()).take(8) {
let map = b.observe_window(w.handle).await.unwrap();
eprintln!(
"'{}': a11y_empty={} nodes={} truncated={}",
w.title,
map.a11y_empty,
map.a11y_index.len(),
map.a11y_truncated
);
if map.a11y_empty {
continue;
}
found = true;
let root = map.a11y_root.as_ref().expect("populated map has a root");
assert!(!root.role.is_empty(), "root has a role");
assert!(!map.a11y_index.is_empty(), "index non-empty");
assert_eq!(
map.a11y_index.len(),
map.a11y_by_id.len(),
"index and by_id have matching cardinality"
);
for id in &map.a11y_index {
assert!(map.a11y_by_id.contains_key(id), "index id {id} in by_id");
}
for id in map.a11y_index.iter().take(6) {
let rec = &map.a11y_by_id[id];
eprintln!(
" {id}: role={} name={:?} focusable={} focused={}",
rec.role, rec.name, rec.focusable, rec.focused
);
}
if let Some(focusable_id) = map
.a11y_index
.iter()
.find(|id| map.a11y_by_id[*id].focusable)
{
let pt = super::imp::resolve_element_point(w.handle, focusable_id);
assert!(pt.is_some(), "focusable element_id resolves to a point");
eprintln!(" resolved {focusable_id} -> {pt:?}");
}
break;
}
assert!(
found,
"at least one visible titled window should expose a UIA tree"
);
}
}