use actl_core::keys::KeySpec;
use actl_core::target::Target;
use actl_core::{CtlError, ErrorCode};
use uiautomation::patterns::{
UIExpandCollapsePattern, UIInvokePattern, UISelectionItemPattern, UITogglePattern,
UIValuePattern,
};
use uiautomation::{UIAutomation, UIElement, UITreeWalker};
use crate::locate::{Located, locate};
use crate::read::set_clipboard_text;
use crate::window::{find_window, focus_window};
use crate::{input, internal, kbd, timing};
pub struct InputResult {
pub window_title: String,
pub role: String,
pub name: Option<String>,
}
impl From<&Located> for InputResult {
fn from(loc: &Located) -> Self {
Self {
window_title: loc.window_title.clone(),
role: loc.role.clone(),
name: loc.name.clone(),
}
}
}
pub fn click(
app: Option<&str>,
target: &Target,
near: Option<&str>,
) -> Result<(Located, &'static str), CtlError> {
let loc = locate(app, target, near)?;
let _t = actl_core::trace::scope("click.pattern");
if let Ok(invoke) = loc.element.get_pattern::<UIInvokePattern>() {
crate::feedback::semantic("click", &loc.element, &loc.window_title, || {
invoke.invoke().map_err(internal)
})?;
return Ok((loc, "invoke"));
}
if let Ok(sel) = loc.element.get_pattern::<UISelectionItemPattern>() {
crate::feedback::semantic("select", &loc.element, &loc.window_title, || {
sel.select().map_err(internal)
})?;
return Ok((loc, "selection-item"));
}
if let Ok(toggle) = loc.element.get_pattern::<UITogglePattern>() {
crate::feedback::semantic("toggle", &loc.element, &loc.window_title, || {
toggle.toggle().map_err(internal)
})?;
return Ok((loc, "toggle"));
}
if let Ok(exp) = loc.element.get_pattern::<UIExpandCollapsePattern>() {
crate::feedback::semantic("expand", &loc.element, &loc.window_title, || {
exp.expand().map_err(internal)
})?;
return Ok((loc, "expand-collapse"));
}
Err(CtlError::new(
ErrorCode::NotActionable,
format!(
"{} ({}) exposes none of Invoke/SelectionItem/Toggle/ExpandCollapse; \
scroll/focus first, try `set-value`, or `--physical` for a real click",
target.describe(),
loc.role
),
))
}
pub fn type_text(
app: Option<&str>,
target: &Target,
text: &str,
paste: bool,
near: Option<&str>,
verify_value: Option<&str>,
) -> Result<TypeOutcome, CtlError> {
let loc = locate(app, target, near)?;
let identity = crate::identity::identity(&loc.element);
let _lock = input::InputLock::acquire(timing().lock_wait_ms)?;
let mut action = crate::feedback::prepare(
"type",
crate::feedback::keyboard(paste, paste),
crate::feedback::target(&loc.element, &loc.window_title),
)?;
action.executing();
loc.element.set_focus().map_err(|_| {
CtlError::new(
ErrorCode::NotActionable,
format!(
"{} ({}) cannot take keyboard focus",
target.describe(),
loc.role
),
)
})?;
if app.is_some() {
std::thread::sleep(std::time::Duration::from_millis(timing().type_focus_ms));
let fg = kbd::foreground_title().unwrap_or_default();
if fg != loc.window_title {
return Err(actl_core::handoff::foreground_error(&loc.window_title, &fg));
}
}
let _focus = crate::identity::FocusGuard::enter(&loc.element)?;
let _anchor = crate::feedback::KeyboardAnchor::capture()?;
crate::feedback::check_foreground(&loc.window_title)?;
crate::feedback::check_keyboard_anchor()?;
action.executing();
let total = text.chars().count();
let outcome = |delivered: usize, stopped_early: bool| TypeOutcome {
located: (&loc).into(),
delivered,
total,
stopped_early,
};
if paste {
set_clipboard_text(text)?;
kbd::send_key_spec(&KeySpec {
modifiers: vec!["ctrl".into()],
keys: vec![actl_core::keys::Key::Char('v')],
})?;
action.delivered();
crate::input_verify::verify_value(&loc.element, &identity, verify_value, total)?;
Ok(outcome(total, false))
} else {
let chars: Vec<char> = text.chars().collect();
let pattern = Some(loc.window_title.clone());
let delivered = input::send_in_segments(
&chars,
timing().segment_chars,
|seg| {
kbd::send_key_spec(&KeySpec {
modifiers: Vec::new(),
keys: seg.iter().map(|c| actl_core::keys::Key::Char(*c)).collect(),
})
},
|| {
crate::identity::check_focus_anchor().is_ok()
&& pattern
.as_ref()
.map(|p| {
kbd::foreground_title()
.map(|t| t.contains(p.as_str()))
.unwrap_or(false)
})
.unwrap_or(false)
},
)?;
if delivered < total {
action.partial();
} else {
action.delivered();
}
crate::input_verify::verify_value(&loc.element, &identity, verify_value, delivered)?;
Ok(outcome(delivered, delivered < total))
}
}
pub struct TypeOutcome {
pub located: InputResult,
pub delivered: usize,
pub total: usize,
pub stopped_early: bool,
}
pub fn press_keys(spec: &KeySpec, app: Option<&str>) -> Result<Option<String>, CtlError> {
press_keys_targeted(spec, app, None)
}
pub fn press_keys_targeted(
spec: &KeySpec,
app: Option<&str>,
target: Option<&Target>,
) -> Result<Option<String>, CtlError> {
let _lock = input::InputLock::acquire(timing().lock_wait_ms)?;
let loc = target.map(|t| locate(app, t, None)).transpose()?;
let _focus = loc
.as_ref()
.map(|l| crate::identity::FocusGuard::enter(&l.element))
.transpose()?;
let focused = match &loc {
Some(l) => Some(l.window_title.clone()),
None => prepare_keyboard(app)?,
};
let _anchor = crate::feedback::KeyboardAnchor::capture()?;
let mut action = crate::feedback::prepare(
"press",
crate::feedback::keyboard(false, false),
actl_core::activity::ActionTarget {
window: focused.clone(),
..Default::default()
},
)?;
if let Some(title) = &focused {
crate::feedback::check_foreground(title)?;
}
crate::feedback::check_keyboard_anchor()?;
action.executing();
kbd::send_key_spec(spec)?;
action.delivered();
Ok(focused)
}
pub fn press_half(spec: &KeySpec, app: Option<&str>) -> Result<Option<String>, CtlError> {
press_half_with_direction(spec, app, true)
}
pub fn press_half_with_direction(
spec: &KeySpec,
app: Option<&str>,
down: bool,
) -> Result<Option<String>, CtlError> {
let _lock = input::InputLock::acquire(timing().lock_wait_ms)?;
let focused = prepare_keyboard(app)?;
let _anchor = crate::feedback::KeyboardAnchor::capture()?;
let mut action = crate::feedback::prepare(
if down { "key-down" } else { "key-up" },
crate::feedback::keyboard(false, false),
actl_core::activity::ActionTarget {
window: focused.clone(),
..Default::default()
},
)?;
if let Some(title) = &focused {
crate::feedback::check_foreground(title)?;
}
crate::feedback::check_keyboard_anchor()?;
action.executing();
kbd::send_key_partial(spec, down)?;
action.delivered();
Ok(focused)
}
fn prepare_keyboard(app: Option<&str>) -> Result<Option<String>, CtlError> {
Ok(match app {
Some(pattern) => {
let title = crate::window::focus_window(pattern)?;
std::thread::sleep(std::time::Duration::from_millis(timing().press_focus_ms));
ensure_element_focus(pattern)?;
let fg = kbd::foreground_title().unwrap_or_default();
if fg != title {
return Err(actl_core::handoff::foreground_error(&title, &fg));
}
Some(title)
}
None => None,
})
}
fn keyboard_focus_matches(pattern: &str, hwnd: isize, title: &str) -> bool {
if let Some(raw) = pattern.strip_prefix("hwnd:") {
crate::window::parse_hwnd(raw).is_ok_and(|expected| expected == hwnd)
} else {
title.contains(pattern)
}
}
fn ensure_element_focus(pattern: &str) -> Result<(), CtlError> {
if kbd::focused_root_window()
.map(|(hwnd, title)| keyboard_focus_matches(pattern, hwnd, &title))
.unwrap_or(false)
{
return Ok(());
}
for _ in 0..timing().focus_retries {
crate::feedback::check_stop()?;
if let Some(edit) = find_editable_element(pattern) {
crate::feedback::check_stop()?;
let _ = edit.set_focus();
}
std::thread::sleep(std::time::Duration::from_millis(timing().poll_ms));
if kbd::focused_root_window()
.map(|(hwnd, title)| keyboard_focus_matches(pattern, hwnd, &title))
.unwrap_or(false)
{
return Ok(());
}
}
Ok(())
}
fn find_editable_element(pattern: &str) -> Option<UIElement> {
let auto = UIAutomation::new().ok()?;
let walker = auto.create_tree_walker().ok()?;
let win = find_window(&auto, &walker, pattern).ok()?;
fn dfs(walker: &UITreeWalker, elem: &UIElement, depth: u32) -> Option<UIElement> {
if depth > 12 {
return None;
}
let role = format!("{:?}", elem.get_control_type().ok()?);
if role == "Document" || role == "Edit" {
return Some(elem.clone());
}
let mut child = walker.get_first_child(elem).ok();
while let Some(c) = child {
if let Some(hit) = dfs(walker, &c, depth + 1) {
return Some(hit);
}
child = walker.get_next_sibling(&c).ok();
}
None
}
dfs(&walker, &win, 0)
}
pub fn set_value(
app: Option<&str>,
target: &Target,
value: &str,
near: Option<&str>,
) -> Result<InputResult, CtlError> {
let loc = locate(app, target, near)?;
let pattern = loc.element.get_pattern::<UIValuePattern>().map_err(|_| {
CtlError::new(
ErrorCode::NotActionable,
format!(
"{} ({}) exposes no Value pattern; try `type` (keyboard path) instead",
target.describe(),
loc.role
),
)
})?;
crate::feedback::semantic("set-value", &loc.element, &loc.window_title, || {
pattern.set_value(value).map_err(internal)
})?;
Ok((&loc).into())
}
pub fn select_option(
app: Option<&str>,
container: &Target,
option: &str,
near: Option<&str>,
) -> Result<(Located, String, &'static str), CtlError> {
let loc = locate(app, container, near)?;
let hits = selection_items_under(&loc.element, option, false);
let (items, level) = if hits.len() == 1 {
(hits, "exact-name")
} else if hits.is_empty() {
let fuzzy = selection_items_under(&loc.element, option, true);
if fuzzy.is_empty() {
return Err(CtlError::new(
ErrorCode::NotFound,
format!(
"no selectable item named {option:?} under {} (exact and substring)",
container.describe()
),
));
}
(fuzzy, "substring-name")
} else {
(hits, "exact-name")
};
if items.len() > 1 {
let names: Vec<String> = items
.iter()
.filter_map(|e| e.get_name().ok())
.take(5)
.collect();
return Err(CtlError::with_evidence(
ErrorCode::Ambiguous,
format!("{option:?} matches {} selectable items", items.len()),
serde_json::json!({ "candidates": names }),
));
}
let item = &items[0];
let name = item.get_name().unwrap_or_default();
crate::feedback::semantic("select", item, &loc.window_title, || {
item.get_pattern::<UISelectionItemPattern>()
.map_err(|_| {
CtlError::new(
ErrorCode::NotActionable,
format!("{name:?} exposes no SelectionItem pattern"),
)
})?
.select()
.map_err(internal)
})?;
Ok((loc, name, level))
}
fn selection_items_under(elem: &UIElement, option: &str, substring: bool) -> Vec<UIElement> {
let auto = match UIAutomation::new() {
Ok(a) => a,
Err(_) => return Vec::new(),
};
let walker = match auto.create_tree_walker() {
Ok(w) => w,
Err(_) => return Vec::new(),
};
fn dfs(
walker: &UITreeWalker,
elem: &UIElement,
option: &str,
substring: bool,
out: &mut Vec<UIElement>,
) {
use uiautomation::patterns::UISelectionItemPattern;
if out.len() >= 32 {
return;
}
if let Ok(name) = elem.get_name() {
let hit = if substring {
name.contains(option)
} else {
name == option
};
if hit && elem.get_pattern::<UISelectionItemPattern>().is_ok() {
out.push(elem.clone());
}
}
let mut child = walker.get_first_child(elem).ok();
while let Some(c) = child {
dfs(walker, &c, option, substring, out);
child = walker.get_next_sibling(&c).ok();
}
}
let mut out = Vec::new();
dfs(&walker, elem, option, substring, &mut out);
out
}
pub fn toggle_element(
app: Option<&str>,
target: &Target,
near: Option<&str>,
) -> Result<(Located, String), CtlError> {
let loc = locate(app, target, near)?;
let pattern = loc.element.get_pattern::<UITogglePattern>().map_err(|_| {
CtlError::new(
ErrorCode::NotActionable,
format!(
"{} ({}) exposes no Toggle pattern; `click` dispatches toggle semantics for checkboxes",
target.describe(),
loc.role
),
)
})?;
crate::feedback::semantic("toggle", &loc.element, &loc.window_title, || {
pattern.toggle().map_err(internal)
})?;
let state = pattern
.get_toggle_state()
.map(|s| format!("{s:?}"))
.unwrap_or_else(|_| "unknown".into());
Ok((loc, state))
}
pub fn paste(app: &str) -> Result<String, CtlError> {
use actl_core::keys::Key;
let _lock = input::InputLock::acquire(timing().lock_wait_ms)?;
let title = focus_window(app)?;
std::thread::sleep(std::time::Duration::from_millis(timing().press_focus_ms));
let fg = kbd::foreground_title().unwrap_or_default();
if fg != title {
return Err(actl_core::handoff::foreground_error(&title, &fg));
}
let _anchor = crate::feedback::KeyboardAnchor::capture()?;
let mut action = crate::feedback::prepare(
"paste",
crate::feedback::keyboard(true, false),
actl_core::activity::ActionTarget {
window: Some(title.clone()),
..Default::default()
},
)?;
crate::feedback::check_foreground(&title)?;
crate::feedback::check_keyboard_anchor()?;
action.executing();
kbd::send_key_spec(&KeySpec {
modifiers: vec!["ctrl".into()],
keys: vec![Key::Char('v')],
})?;
action.delivered();
Ok(title)
}
#[cfg(test)]
mod focus_tests {
use super::keyboard_focus_matches;
#[test]
fn hwnd_focus_preserves_the_selected_field() {
assert!(keyboard_focus_matches("hwnd:123", 123, "Zotero"));
assert!(keyboard_focus_matches("hwnd:0x7b", 123, "Zotero"));
assert!(!keyboard_focus_matches("hwnd:123", 456, "Zotero"));
assert!(!keyboard_focus_matches("hwnd:invalid", 123, "hwnd:invalid"));
assert!(keyboard_focus_matches("Zotero", 123, "My Library - Zotero"));
assert!(!keyboard_focus_matches("Other", 123, "Zotero"));
}
}