actl-uia 0.1.4

Windows UIA backend: the ONLY crate allowed to touch COM/unsafe
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
//! action —— 语义动作执行:click(pattern 分发)、type/press(键盘注入 +
//! verify-then-inject 护栏)、set-value(Value pattern 直写)。
//! 物理指针动作见 `pointer`;读取与断言见 `read`。

use actl_core::keys::KeySpec;
use actl_core::target::Target;
use actl_core::{CtlError, ErrorCode};
use uiautomation::patterns::{
    UIExpandCollapsePattern, UIInvokePattern, UISelectionItemPattern, UITogglePattern,
    UIValuePattern,
};
use uiautomation::{UIAutomation, UIElement, UITreeWalker};

use crate::locate::{Located, locate};
use crate::read::set_clipboard_text;
use crate::window::{find_window, focus_window};
use crate::{input, internal, kbd, timing};

/// 键盘命令家族说明:type/press 走 SendInput 键盘事件(键盘语义命令的执行本体,
/// 非物理"兜底",ADR-007 的 --physical 约束针对 click/scroll 等 UIA 等价物的兜底路径)。
pub struct InputResult {
    pub window_title: String,
    pub role: String,
    pub name: Option<String>,
}

impl From<&Located> for InputResult {
    fn from(loc: &Located) -> Self {
        Self {
            window_title: loc.window_title.clone(),
            role: loc.role.clone(),
            name: loc.name.clone(),
        }
    }
}

/// click:语义执行分发(06 M1 范围:Invoke/Value/Selection)。
/// Invoke 优先;无 Invoke 但支持 SelectionItem(列表项/树项/选项卡)→ select(),
/// 语义同为"点击选中"。实际使用的 pattern 随结果上报,agent 可归因。
pub fn click(
    app: Option<&str>,
    target: &Target,
    near: Option<&str>,
) -> Result<(Located, &'static str), CtlError> {
    let loc = locate(app, target, near)?;
    let _t = actl_core::trace::scope("click.pattern");
    if let Ok(invoke) = loc.element.get_pattern::<UIInvokePattern>() {
        invoke.invoke().map_err(internal)?;
        return Ok((loc, "invoke"));
    }
    if let Ok(sel) = loc.element.get_pattern::<UISelectionItemPattern>() {
        sel.select().map_err(internal)?;
        return Ok((loc, "selection-item"));
    }
    // 复选框/开关:点击语义 = Toggle(状态翻转)
    if let Ok(toggle) = loc.element.get_pattern::<UITogglePattern>() {
        toggle.toggle().map_err(internal)?;
        return Ok((loc, "toggle"));
    }
    // 折叠面板/树节点:点击语义 = ExpandCollapse
    if let Ok(exp) = loc.element.get_pattern::<UIExpandCollapsePattern>() {
        exp.expand().map_err(internal)?;
        return Ok((loc, "expand-collapse"));
    }
    Err(CtlError::new(
        ErrorCode::NotActionable,
        format!(
            "{} ({}) exposes none of Invoke/SelectionItem/Toggle/ExpandCollapse; \
             scroll/focus first, try `set-value`, or `--physical` for a real click",
            target.describe(),
            loc.role
        ),
    ))
}

/// type:定位 → SetFocus →(护栏)→ 逐字键盘输入或剪贴板粘贴。
/// `--paste`:文本经剪贴板 + ctrl+v 送达——**中文模式 IME 下唯一可靠的文本通道**
/// (UNICODE 包在 WinUI/TSF 仍会被 IME 组合改写:空格吞成上屏、标点全角化,
/// 战记实测);代价是占用共享剪贴板,输出透明上报 via 字段,不静默。
pub fn type_text(
    app: Option<&str>,
    target: &Target,
    text: &str,
    paste: bool,
    near: Option<&str>,
) -> Result<TypeOutcome, CtlError> {
    let loc = locate(app, target, near)?;
    loc.element.set_focus().map_err(|_| {
        CtlError::new(
            ErrorCode::NotActionable,
            format!(
                "{} ({}) cannot take keyboard focus",
                target.describe(),
                loc.role
            ),
        )
    })?;
    // 护栏与 press 相同:元素置焦后仍要物理前台核对,文本才允许注入
    if let Some(pattern) = app {
        std::thread::sleep(std::time::Duration::from_millis(timing().type_focus_ms));
        let fg = kbd::foreground_title().unwrap_or_default();
        if !fg.contains(pattern) {
            return Err(CtlError::new(
                ErrorCode::PermDenied,
                format!(
                    "text injection ABORTED by verify-then-inject guard: \
                     physical foreground is {fg:?}, expected a window matching {pattern:?}; \
                     no text was sent"
                ),
            ));
        }
    }
    // 输入占用锁(doc 09 §6.2):多 actl 实例互斥;持锁覆盖整个注入动作段
    let _lock = input::InputLock::acquire(timing().lock_wait_ms)?;
    let total = text.chars().count();
    let outcome = |delivered: usize, stopped_early: bool| TypeOutcome {
        located: (&loc).into(),
        delivered,
        total,
        stopped_early,
    };
    if paste {
        set_clipboard_text(text)?;
        // ctrl+v 走 VK 组合通道(IME 不拦截带修饰键组合);单次原子动作不分批
        kbd::send_key_spec(&KeySpec {
            modifiers: vec!["ctrl".into()],
            keys: vec![actl_core::keys::Key::Char('v')],
        })?;
        Ok(outcome(total, false))
    } else {
        // 键盘逐字通道:分批注入,批间复核物理前台——焦点被抢(真人点走/弹窗
        // 夺焦)即停在批边界并如实上报已送达量(并发输入损坏注入的四次活体
        // 实证结论,docs/spike-findings)
        let chars: Vec<char> = text.chars().collect();
        let pattern = app.map(str::to_string);
        let delivered = input::send_in_segments(
            &chars,
            timing().segment_chars,
            |seg| {
                kbd::send_key_spec(&KeySpec {
                    modifiers: Vec::new(),
                    keys: seg.iter().map(|c| actl_core::keys::Key::Char(*c)).collect(),
                })
            },
            || {
                pattern
                    .as_ref()
                    .map(|p| {
                        kbd::foreground_title()
                            .map(|t| t.contains(p.as_str()))
                            .unwrap_or(false)
                    })
                    .unwrap_or(false)
            },
        )?;
        Ok(outcome(delivered, delivered < total))
    }
}

/// type 的结果:注入了多少、是否中途被焦点抢占截断(部分送达是如实上报,
/// 不是失败——ok:true + input_effects.keyboard="partial")。
pub struct TypeOutcome {
    pub located: InputResult,
    pub delivered: usize,
    pub total: usize,
    pub stopped_early: bool,
}

/// press:可选先置前台(--app)。实测链路(docs/spike-findings.md):窗口级 SetFocus
/// 只激活窗口、不建立键盘焦点(键会被丢);必须再对窗口内内容元素做**元素级** SetFocus。
/// 启发式:窗口内首个 Document/Edit;找不到(如纯按钮面板)退化为仅窗口前置。
///
/// **verify-then-inject 护栏(事故教训,docs/spike-findings.md #6)**:注入前用 Win32
/// 物理前台(GetForegroundWindow)验证前台确为目标窗口——UIA 焦点视图可能与物理前台
/// 不一致,"看起来聚焦了"不等于键会进目标。不匹配则拒绝注入,宁可失败不可错发。
pub fn press_keys(spec: &KeySpec, app: Option<&str>) -> Result<Option<String>, CtlError> {
    let focused = prepare_keyboard(app)?;
    // 输入占用锁(doc 09 §6.2):护栏之后、注入之前取得,覆盖整个按键序列
    let _lock = input::InputLock::acquire(timing().lock_wait_ms)?;
    kbd::send_key_spec(spec)?;
    Ok(focused)
}

/// key-down/key-up:与 press 相同的护栏链,但只发半边事件(见 kbd::send_key_partial)。
pub fn press_half(spec: &KeySpec, app: Option<&str>) -> Result<Option<String>, CtlError> {
    let focused = prepare_keyboard(app)?;
    let _lock = input::InputLock::acquire(timing().lock_wait_ms)?;
    // 半事件方向由调用方决定;此函数不持锁跨进程(key-up 是另一进程,锁不横跨)
    kbd::send_key_partial(spec, true)?;
    Ok(focused)
}

/// 键盘注入的前置链:置前台 → 元素级聚焦 → 物理前台护栏(verify-then-inject)。
fn prepare_keyboard(app: Option<&str>) -> Result<Option<String>, CtlError> {
    Ok(match app {
        Some(pattern) => {
            let title = crate::window::focus_window(pattern)?;
            std::thread::sleep(std::time::Duration::from_millis(timing().press_focus_ms));
            // 元素聚焦是异步请求且存在竞态(实测固定延时忽好忽坏):
            // 反复 SetFocus 并验证 UIA 焦点元素真的变为可编辑角色,最多 3 轮
            ensure_element_focus(pattern)?;
            // 护栏:物理前台必须是目标窗口,否则注入会打进别处(已发生过真实事故)
            let fg = kbd::foreground_title().unwrap_or_default();
            if !fg.contains(pattern) {
                return Err(CtlError::new(
                    ErrorCode::PermDenied,
                    format!(
                        "keyboard injection ABORTED by verify-then-inject guard: \
                         physical foreground is {fg:?}, expected a window matching {pattern:?}; \
                         no keys were sent"
                    ),
                ));
            }
            Some(title)
        }
        None => None,
    })
}

/// 反复对窗口内可编辑元素 SetFocus,直到**物理**键盘焦点的根窗口是目标。
/// 判据用 GetGUIThreadInfo 物理层而非 UIA focused element:后者在 WinUI 上
/// 实测会谎报(物理层 RichEditD2DPT 已聚焦时仍报别窗口编辑框,导致成批误拒绝
/// ——战记"信源必须用可靠层"的又一次实例)。重试耗尽未就绪 → 放行,由
/// verify-then-inject 物理前台护栏做最终裁决(它才是注入前的权威闸门)。
fn ensure_element_focus(pattern: &str) -> Result<(), CtlError> {
    // 前置免扰:物理键盘焦点已落在目标窗口时直接放行——多 Edit 应用
    // (Excel:首页搜索框/名称框/编辑栏)里"找首个 Edit 强行 SetFocus"会把
    // 已正确的焦点搅到搜索框,后续按键全被吞(Office 实测:ctrl+s/f12 失效)
    if kbd::focused_root_title()
        .map(|t| t.contains(pattern))
        .unwrap_or(false)
    {
        return Ok(());
    }
    for _ in 0..timing().focus_retries {
        if let Some(edit) = find_editable_element(pattern) {
            let _ = edit.set_focus();
        }
        std::thread::sleep(std::time::Duration::from_millis(timing().poll_ms));
        if kbd::focused_root_title()
            .map(|t| t.contains(pattern))
            .unwrap_or(false)
        {
            return Ok(());
        }
    }
    Ok(())
}

/// 在标题匹配的窗口内找首个 Document/Edit(元素级键盘焦点的落点)。
fn find_editable_element(pattern: &str) -> Option<UIElement> {
    let auto = UIAutomation::new().ok()?;
    let walker = auto.create_tree_walker().ok()?;
    let win = find_window(&auto, &walker, pattern).ok()?;
    fn dfs(walker: &UITreeWalker, elem: &UIElement, depth: u32) -> Option<UIElement> {
        if depth > 12 {
            return None;
        }
        let role = format!("{:?}", elem.get_control_type().ok()?);
        if role == "Document" || role == "Edit" {
            return Some(elem.clone());
        }
        let mut child = walker.get_first_child(elem).ok();
        while let Some(c) = child {
            if let Some(hit) = dfs(walker, &c, depth + 1) {
                return Some(hit);
            }
            child = walker.get_next_sibling(&c).ok();
        }
        None
    }
    dfs(&walker, &win, 0)
}

/// set-value:UIValue pattern 直写,不经键盘(与 type 的分工见 06 §3.6)。
pub fn set_value(
    app: Option<&str>,
    target: &Target,
    value: &str,
    near: Option<&str>,
) -> Result<InputResult, CtlError> {
    let loc = locate(app, target, near)?;
    let pattern = loc.element.get_pattern::<UIValuePattern>().map_err(|_| {
        CtlError::new(
            ErrorCode::NotActionable,
            format!(
                "{} ({}) exposes no Value pattern; try `type` (keyboard path) instead",
                target.describe(),
                loc.role
            ),
        )
    })?;
    pattern.set_value(value).map_err(internal)?;
    Ok((&loc).into())
}

/// select:在容器内按选项名选中一项(SelectionItem pattern;06 §3.6 与
/// set-value 的分工——离散选项 vs 文本值)。选项匹配:精确 Name,落空后
/// 子串兜底(与定位链 L1/L2 同哲学);多命中 → AMBIGUOUS(如实上报)。
pub fn select_option(
    app: Option<&str>,
    container: &Target,
    option: &str,
    near: Option<&str>,
) -> Result<(Located, String, &'static str), CtlError> {
    let loc = locate(app, container, near)?;
    let hits = selection_items_under(&loc.element, option, false);
    let (items, level) = if hits.len() == 1 {
        (hits, "exact-name")
    } else if hits.is_empty() {
        let fuzzy = selection_items_under(&loc.element, option, true);
        if fuzzy.is_empty() {
            return Err(CtlError::new(
                ErrorCode::NotFound,
                format!(
                    "no selectable item named {option:?} under {} (exact and substring)",
                    container.describe()
                ),
            ));
        }
        (fuzzy, "substring-name")
    } else {
        (hits, "exact-name")
    };
    if items.len() > 1 {
        let names: Vec<String> = items
            .iter()
            .filter_map(|e| e.get_name().ok())
            .take(5)
            .collect();
        return Err(CtlError::with_evidence(
            ErrorCode::Ambiguous,
            format!("{option:?} matches {} selectable items", items.len()),
            serde_json::json!({ "candidates": names }),
        ));
    }
    let item = &items[0];
    let name = item.get_name().unwrap_or_default();
    item.get_pattern::<UISelectionItemPattern>()
        .map_err(|_| {
            CtlError::new(
                ErrorCode::NotActionable,
                format!("{name:?} exposes no SelectionItem pattern"),
            )
        })?
        .select()
        .map_err(internal)?;
    Ok((loc, name, level))
}

/// 容器子树内收集支持 SelectionItem 且名字匹配的元素。
fn selection_items_under(elem: &UIElement, option: &str, substring: bool) -> Vec<UIElement> {
    let auto = match UIAutomation::new() {
        Ok(a) => a,
        Err(_) => return Vec::new(),
    };
    let walker = match auto.create_tree_walker() {
        Ok(w) => w,
        Err(_) => return Vec::new(),
    };
    fn dfs(
        walker: &UITreeWalker,
        elem: &UIElement,
        option: &str,
        substring: bool,
        out: &mut Vec<UIElement>,
    ) {
        use uiautomation::patterns::UISelectionItemPattern;
        if out.len() >= 32 {
            return;
        }
        if let Ok(name) = elem.get_name() {
            let hit = if substring {
                name.contains(option)
            } else {
                name == option
            };
            if hit && elem.get_pattern::<UISelectionItemPattern>().is_ok() {
                out.push(elem.clone());
            }
        }
        let mut child = walker.get_first_child(elem).ok();
        while let Some(c) = child {
            dfs(walker, &c, option, substring, out);
            child = walker.get_next_sibling(&c).ok();
        }
    }
    let mut out = Vec::new();
    dfs(&walker, elem, option, substring, &mut out);
    out
}

/// toggle:翻转 Toggle 状态并上报翻转后状态(与 click 的分工:toggle 无点击
/// 副作用、可读状态;复选框/开关的语义动作)。
pub fn toggle_element(
    app: Option<&str>,
    target: &Target,
    near: Option<&str>,
) -> Result<(Located, String), CtlError> {
    let loc = locate(app, target, near)?;
    let pattern = loc.element.get_pattern::<UITogglePattern>().map_err(|_| {
        CtlError::new(
            ErrorCode::NotActionable,
            format!(
                "{} ({}) exposes no Toggle pattern; `click` dispatches toggle semantics for checkboxes",
                target.describe(),
                loc.role
            ),
        )
    })?;
    pattern.toggle().map_err(internal)?;
    let state = pattern
        .get_toggle_state()
        .map(|s| format!("{s:?}"))
        .unwrap_or_else(|_| "unknown".into());
    Ok((loc, state))
}

/// paste:ctrl+v 注入到目标窗口的**当前焦点/选中处**(pack 路线的关键原语:
/// Excel 选中格、文件管理器选中项、重命名框等"无元素"落点)。护栏链与 press
/// 相同,但**刻意不做元素级聚焦**——SetFocus 会移动选中,违背本命令语义;
/// 只做窗口前置 + 物理前台护栏 + 输入锁。剪贴板内容只读不写(消费方)。
pub fn paste(app: &str) -> Result<String, CtlError> {
    use actl_core::keys::Key;
    let title = focus_window(app)?;
    std::thread::sleep(std::time::Duration::from_millis(timing().press_focus_ms));
    let fg = kbd::foreground_title().unwrap_or_default();
    if !fg.contains(app) {
        return Err(CtlError::new(
            ErrorCode::PermDenied,
            format!(
                "paste ABORTED by verify-then-inject guard: \
                 physical foreground is {fg:?}, expected a window matching {app:?}; \
                 clipboard untouched, nothing pasted"
            ),
        ));
    }
    let _lock = input::InputLock::acquire(timing().lock_wait_ms)?;
    kbd::send_key_spec(&KeySpec {
        modifiers: vec!["ctrl".into()],
        keys: vec![Key::Char('v')],
    })?;
    Ok(title)
}