Skip to main content

computer_use/
element.rs

1//! Element actions: revalidation, the perform allowlist, and secure-field refusal.
2//!
3//! Nothing here calls a platform API. The macOS backend and [`crate::test_backend`]
4//! both ask this module whether an action may run, then record or perform it.
5
6use crate::error::ComputerError;
7use crate::tree::{self, SavedFrame, SigNode, UiNode};
8use crate::types::MouseButton;
9
10/// Each edge may differ by this many screen points and still be the same element.
11///
12/// Two points is the tolerance P2 uses to join an AX window to a CG window.
13/// A larger shift is `stale_element`. AX rounding inside the tolerance is not.
14pub const FRAME_TOLERANCE: i32 = 2;
15
16/// Hint included on `element_not_found` and `stale_element`.
17pub const REFRESH_HINT: &str = "re-run `computer-use tree --app X` to refresh indices";
18
19/// `AXUIElementPerformAction` names `perform_action` accepts.
20pub const PERFORM_ALLOWLIST: &[&str] = &[
21    "AXPress",
22    "AXShowMenu",
23    "AXIncrement",
24    "AXDecrement",
25    "AXConfirm",
26    "AXCancel",
27    "AXRaise",
28    "AXScrollToVisible",
29    "AXScrollUpByPage",
30    "AXScrollDownByPage",
31    "AXScrollLeftByPage",
32    "AXScrollRightByPage",
33];
34
35/// What the caller asked to do to one element index.
36#[derive(Debug, Clone, PartialEq, Eq)]
37pub enum ElementOp {
38    /// `AXPress` for the left button. Right uses `AXShowMenu` when the element
39    /// advertises it, otherwise a coordinate click at the element center.
40    /// Middle, back, and forward always click the center.
41    Press {
42        /// Repeat count. `AXShowMenu` runs once.
43        count: u8,
44        /// Mouse button. Defaults to left at the call site.
45        button: MouseButton,
46    },
47    /// Set `AXFocused` to true.
48    Focus,
49    /// Set `AXValue`. Refused on secure fields.
50    SetValue {
51        /// New value.
52        value: String,
53    },
54    /// Set `AXSelectedTextRange`. Refused on secure fields.
55    SelectText {
56        /// First selected character.
57        start: i64,
58        /// Character after the selection.
59        end: i64,
60    },
61    /// `AXUIElementPerformAction` with an allowlisted name.
62    Perform {
63        /// Action name such as `AXShowMenu`.
64        name: String,
65    },
66    /// `AXScrollToVisible` only.
67    ScrollTo,
68    /// Prefer `AXValue` when the attribute is settable, otherwise focus and type.
69    Type {
70        /// Text to write.
71        text: String,
72    },
73}
74
75impl ElementOp {
76    /// JSON `action` name.
77    pub fn action_name(&self) -> &'static str {
78        match self {
79            Self::Press { .. } => "press",
80            Self::Focus => "focus",
81            Self::SetValue { .. } => "set_value",
82            Self::SelectText { .. } => "select_text",
83            Self::Perform { .. } => "perform_action",
84            Self::ScrollTo => "scroll_to",
85            Self::Type { .. } => "type",
86        }
87    }
88
89    /// Writes text or a selection. Secure fields refuse these.
90    pub fn writes_text(&self) -> bool {
91        matches!(
92            self,
93            Self::SetValue { .. } | Self::SelectText { .. } | Self::Type { .. }
94        )
95    }
96}
97
98/// Shared fields for one element command.
99#[derive(Debug, Clone, PartialEq, Eq)]
100pub struct ElementRequest {
101    /// Action to run after revalidation.
102    pub op: ElementOp,
103    /// Index in the cached revision.
104    pub index: u32,
105    /// Fail when the element is outside the window and scroll fails or is missing.
106    pub require_visible: bool,
107    /// Application selector. Empty uses the frontmost app.
108    pub app: Option<String>,
109    /// Window selector. Empty uses the key window.
110    pub window: Option<String>,
111    /// Session cache namespace.
112    pub session: Option<String>,
113    /// Requested input tier for a center click. Empty is auto.
114    pub tier: Option<String>,
115    /// Skip `foreground.lock` when a center click uses `activate`.
116    pub allow_foreground: bool,
117    /// Do not fall back to `activate` for a center click.
118    pub require_background: bool,
119    /// Skip the overlay helper for a center click.
120    pub no_overlay: bool,
121}
122
123/// One step the backend performs after [`gate`] allows it.
124#[derive(Debug, Clone, PartialEq, Eq)]
125pub enum Step {
126    /// Repeat `AXPress`.
127    Press {
128        /// Repeat count, already checked to be 1, 2, or 3.
129        count: u8,
130    },
131    /// Set `AXFocused` to true.
132    Focus,
133    /// Set `AXValue`.
134    SetValue {
135        /// New value.
136        value: String,
137    },
138    /// Set `AXSelectedTextRange`.
139    SelectText {
140        /// Range location.
141        start: i64,
142        /// Range end, exclusive.
143        end: i64,
144    },
145    /// Perform an allowlisted action.
146    Perform {
147        /// Action name.
148        name: String,
149    },
150    /// Perform `AXScrollToVisible`.
151    ScrollTo,
152    /// Set `AXValue` because the attribute is settable.
153    TypeSetValue {
154        /// Text to write.
155        text: String,
156    },
157    /// Focus the element, then post the text to its process. Does not activate the app.
158    TypeEnigo {
159        /// Text to type.
160        text: String,
161    },
162    /// Click the element center through the input tiers.
163    CenterClick {
164        /// Screen x of the center.
165        x: i32,
166        /// Screen y of the center.
167        y: i32,
168        /// Mouse button.
169        button: MouseButton,
170        /// Click count, 1, 2, or 3.
171        count: u8,
172    },
173}
174
175impl Step {
176    /// Name recorded by the stub: the AX action or attribute that runs.
177    pub fn record_name(&self) -> &str {
178        match self {
179            Self::Press { .. } => "AXPress",
180            Self::CenterClick { .. } => "center-click",
181            Self::Focus | Self::TypeEnigo { .. } => "AXFocused",
182            Self::SetValue { .. } | Self::TypeSetValue { .. } => "AXValue",
183            Self::SelectText { .. } => "AXSelectedTextRange",
184            Self::Perform { name } => name.as_str(),
185            Self::ScrollTo => "AXScrollToVisible",
186        }
187    }
188
189    /// How many times the stub records this step.
190    pub fn repeats(&self) -> u8 {
191        match self {
192            Self::Press { count } | Self::CenterClick { count, .. } => *count,
193            _ => 1,
194        }
195    }
196}
197
198/// Successful element action, before it is rendered as JSON.
199#[derive(Debug, Clone, PartialEq, Eq)]
200pub struct ElementOutcome {
201    /// Command name such as `press` or `set_value`.
202    pub action: String,
203    /// Index the caller passed.
204    pub element: u32,
205    /// Structural hash after the action.
206    pub revision: u64,
207    /// `ax` for an accessibility action. `event` posts to the pid. `activate` uses the foreground path.
208    pub tier: String,
209    /// True when this action moved the system cursor.
210    pub cursor_moved: bool,
211    /// Set when the action tried to restore the previous frontmost app.
212    pub focus_restored: Option<bool>,
213    /// Non-fatal notes. Scroll failure lands here unless `--require-visible` is set.
214    pub warnings: Vec<String>,
215    /// `set_value` or `enigo` for `type`. Empty for other commands.
216    pub via: Option<String>,
217    /// Allowlisted action name for `perform_action`.
218    pub name: Option<String>,
219    /// Press count, when the command was `press`.
220    pub count: Option<u8>,
221}
222
223/// Success value shared by the macOS backend and the stub.
224pub fn outcome(
225    request: &ElementRequest,
226    revision: u64,
227    warnings: Vec<String>,
228    via: Option<String>,
229) -> ElementOutcome {
230    let tier = match via.as_deref() {
231        Some("event") => "event",
232        Some("activate") => "activate",
233        Some("enigo") => "enigo",
234        _ => "ax",
235    };
236    let cursor_moved = matches!(via.as_deref(), Some("activate") | Some("enigo"));
237    let focus_restored = (via.as_deref() == Some("activate")).then_some(true);
238    ElementOutcome {
239        action: request.op.action_name().to_string(),
240        element: request.index,
241        revision,
242        tier: tier.to_string(),
243        cursor_moved,
244        focus_restored,
245        warnings,
246        via,
247        name: match &request.op {
248            ElementOp::Perform { name } => Some(name.clone()),
249            _ => None,
250        },
251        count: match request.op {
252            ElementOp::Press { count, .. } => Some(count),
253            _ => None,
254        },
255    }
256}
257
258/// Outcome for an action that the `activate` tier performed.
259///
260/// [`outcome`] reports `focus_restored: Some(true)` for every `activate` via,
261/// which is right on macOS (that tier puts the previous app back). Windows
262/// activates an element pattern as a *retry*, and passes what its own
263/// `SetForegroundWindow` restore actually achieved, so this constructor takes
264/// the flag instead of assuming it.
265pub fn activate_outcome(
266    request: &ElementRequest,
267    revision: u64,
268    warnings: Vec<String>,
269    focus_restored: bool,
270) -> ElementOutcome {
271    ElementOutcome {
272        action: request.op.action_name().to_string(),
273        element: request.index,
274        revision,
275        tier: "activate".to_string(),
276        cursor_moved: true,
277        focus_restored: Some(focus_restored),
278        warnings,
279        via: Some("activate".to_string()),
280        name: None,
281        count: match request.op {
282            ElementOp::Press { count, .. } => Some(count),
283            _ => None,
284        },
285    }
286}
287
288/// Failure that may carry the fresh tree for `stale_element`.
289#[derive(Debug)]
290pub struct ElementError {
291    /// Stable error.
292    pub error: ComputerError,
293    /// Numbered tree after the re-fetch. Present for `stale_element`.
294    pub tree: Option<String>,
295}
296
297impl ElementError {
298    /// An error without a replacement tree.
299    pub fn new(error: ComputerError) -> Self {
300        Self { error, tree: None }
301    }
302
303    /// `stale_element` plus the fresh numbered tree.
304    pub fn stale(message: impl Into<String>, tree: impl Into<String>) -> Self {
305        Self {
306            error: ComputerError::StaleElement(message.into()),
307            tree: Some(tree.into()),
308        }
309    }
310}
311
312impl From<ComputerError> for ElementError {
313    fn from(error: ComputerError) -> Self {
314        Self::new(error)
315    }
316}
317
318/// Cached title and secure bit used by the confirm hook before an element action.
319#[derive(Debug, Clone, PartialEq, Eq)]
320pub struct ElementPeek {
321    /// Title, identifier, or description. Empty when the node has none.
322    pub title: Option<String>,
323    /// `AXSecureTextField` or a secure subrole.
324    pub secure: bool,
325}
326
327/// Role and title of the element under a coordinate, when hit-testing finds one.
328#[derive(Debug, Clone, PartialEq, Eq)]
329pub struct HitInfo {
330    /// `AXRole`.
331    pub role: String,
332    /// `AXTitle`, when the element has one.
333    pub title: Option<String>,
334}
335
336/// Revalidation succeeded. The backend may read settable or character count next.
337#[derive(Debug, Clone, PartialEq, Eq)]
338pub struct Gate {
339    /// Child indices from the window to the element. Empty is the window itself.
340    pub path: Vec<usize>,
341    /// Try `AXScrollToVisible` before the action.
342    pub scroll_first: bool,
343    /// The element is outside the window and does not support scroll.
344    pub outside_warning: Option<String>,
345    /// `type` still needs `AXUIElementIsAttributeSettable` for `AXValue`.
346    pub needs_settable: bool,
347    /// `select_text` still needs `AXNumberOfCharacters` when the attribute exists.
348    pub needs_chars: bool,
349    /// Action names copied before the perform.
350    pub actions: Vec<String>,
351    /// Center of the element frame, in screen points. Missing when the frame is absent.
352    pub center: Option<(i32, i32)>,
353}
354
355/// Inputs [`gate`] needs. The live node is the re-fetched tree.
356pub struct GateInput<'a> {
357    /// Requested operation.
358    pub op: &'a ElementOp,
359    /// Caller index.
360    pub index: u32,
361    /// `--require-visible`.
362    pub require_visible: bool,
363    /// App name used in the refresh hint. Empty keeps the literal `X`.
364    pub app_label: &'a str,
365    /// Cached signature. `observed` wins over the diff baseline.
366    pub cached: &'a SigNode,
367    /// Revision stored with `cached`. It must equal [`tree::revision_of`].
368    pub cached_revision: u64,
369    /// Pid stored with the snapshot.
370    pub cached_pid: i32,
371    /// Window id stored with the snapshot.
372    pub cached_window: u32,
373    /// Pid of the resolved target.
374    pub resolved_pid: i32,
375    /// Window id of the resolved target.
376    pub resolved_window: u32,
377    /// Fresh tree.
378    pub live: &'a UiNode,
379    /// Pid of the fresh tree.
380    pub live_pid: i32,
381    /// Window id of the fresh tree.
382    pub live_window: u32,
383}
384
385/// `count` must be 1, 2, or 3.
386pub fn validate_count(count: u8) -> Result<(), ComputerError> {
387    if (1..=3).contains(&count) {
388        Ok(())
389    } else {
390        Err(ComputerError::InvalidArguments(
391            "count must be from 1 to 3".into(),
392        ))
393    }
394}
395
396/// Refuse names outside [`PERFORM_ALLOWLIST`].
397pub fn validate_perform_name(name: &str) -> Result<(), ComputerError> {
398    if PERFORM_ALLOWLIST.contains(&name) {
399        Ok(())
400    } else {
401        Err(ComputerError::InvalidArguments(format!(
402            "{name} is not an allowed action. Allowed: {}",
403            PERFORM_ALLOWLIST.join(", ")
404        )))
405    }
406}
407
408/// `start <= end`, both non-negative, and `end` within `char_count` when it was read.
409pub fn validate_range(start: i64, end: i64, char_count: Option<i64>) -> Result<(), ComputerError> {
410    if start < 0 || end < 0 {
411        return Err(ComputerError::InvalidArguments(
412            "start and end must be greater than or equal to 0".into(),
413        ));
414    }
415    if start > end {
416        return Err(ComputerError::InvalidArguments(
417            "start must be less than or equal to end".into(),
418        ));
419    }
420    if let Some(count) = char_count {
421        if end > count {
422            return Err(ComputerError::InvalidArguments(format!(
423                "end {end} is past AXNumberOfCharacters {count}"
424            )));
425        }
426    }
427    Ok(())
428}
429
430/// Refresh hint. A known app name replaces the literal `X`.
431pub fn refresh_hint(app: &str) -> String {
432    let app = app.trim();
433    if app.is_empty() || app == "X" {
434        REFRESH_HINT.to_string()
435    } else {
436        format!("re-run `computer-use tree --app {app}` to refresh indices")
437    }
438}
439
440/// Child-index path of preorder `index` in a cached signature.
441pub fn find_path(root: &SigNode, index: u32) -> Option<Vec<usize>> {
442    fn walk(node: &SigNode, index: u32, path: &mut Vec<usize>) -> bool {
443        if node.index == index {
444            return true;
445        }
446        for (child_index, child) in node.children.iter().enumerate() {
447            path.push(child_index);
448            if walk(child, index, path) {
449                return true;
450            }
451            path.pop();
452        }
453        false
454    }
455    let mut path = Vec::new();
456    walk(root, index, &mut path).then_some(path)
457}
458
459/// Live node at a child-index path. An empty path is the root.
460pub fn ui_at_path<'a>(root: &'a UiNode, path: &[usize]) -> Option<&'a UiNode> {
461    let mut node = root;
462    for &index in path {
463        node = node.children.get(index)?;
464    }
465    Some(node)
466}
467
468fn sig_at_path<'a>(root: &'a SigNode, path: &[usize]) -> Option<&'a SigNode> {
469    let mut node = root;
470    for &index in path {
471        node = node.children.get(index)?;
472    }
473    Some(node)
474}
475
476/// True when `element` is not fully inside `window`.
477pub fn frame_outside(element: SavedFrame, window: SavedFrame) -> bool {
478    let right = element.x.saturating_add_unsigned(element.width);
479    let bottom = element.y.saturating_add_unsigned(element.height);
480    let window_right = window.x.saturating_add_unsigned(window.width);
481    let window_bottom = window.y.saturating_add_unsigned(window.height);
482    element.x < window.x || element.y < window.y || right > window_right || bottom > window_bottom
483}
484
485/// True when every edge is within [`FRAME_TOLERANCE`]. Missing frames match only each other.
486pub fn frames_match(cached: Option<SavedFrame>, live: Option<SavedFrame>) -> bool {
487    match (cached, live) {
488        (None, None) => true,
489        (Some(cached), Some(live)) => {
490            near(cached.x, live.x)
491                && near(cached.y, live.y)
492                && near_u(cached.width, live.width)
493                && near_u(cached.height, live.height)
494        }
495        _ => false,
496    }
497}
498
499fn near(left: i32, right: i32) -> bool {
500    (i64::from(left) - i64::from(right)).abs() <= i64::from(FRAME_TOLERANCE)
501}
502
503fn near_u(left: u32, right: u32) -> bool {
504    (i64::from(left) - i64::from(right)).abs() <= i64::from(FRAME_TOLERANCE)
505}
506
507/// Decide whether the action may run.
508///
509/// A pid, window id, or cached revision that does not match the stored tree is
510/// `stale_element`. The live structural hash may differ when some other node
511/// changed: this function compares the element at the cached child path.
512pub fn gate(input: &GateInput<'_>) -> Result<Gate, ElementError> {
513    let hint = refresh_hint(input.app_label);
514    let live_text = tree::render(input.live).text;
515    if input.cached_pid != input.resolved_pid
516        || input.cached_window != input.resolved_window
517        || input.cached_pid != input.live_pid
518        || input.cached_window != input.live_window
519        || input.cached_revision != tree::revision_of(input.cached)
520    {
521        return Err(ElementError::stale(
522            format!("cached revision/pid/window does not match; {hint}"),
523            live_text,
524        ));
525    }
526    let Some(path) = find_path(input.cached, input.index) else {
527        return Err(ComputerError::ElementNotFound(format!(
528            "element {} is not in the cached tree; {hint}",
529            input.index
530        ))
531        .into());
532    };
533    let Some(cached) = sig_at_path(input.cached, &path) else {
534        return Err(ComputerError::ElementNotFound(format!(
535            "element {} is not in the cached tree; {hint}",
536            input.index
537        ))
538        .into());
539    };
540    let Some(live) = ui_at_path(input.live, &path) else {
541        return Err(ComputerError::ElementNotFound(format!(
542            "element {} is missing from the fresh tree; {hint}",
543            input.index
544        ))
545        .into());
546    };
547    let live_identity = tree::identity_text(live);
548    let live_frame = live.frame.map(SavedFrame::from);
549    if cached.role != live.role {
550        return Err(ElementError::stale(
551            format!("role changed from {} to {}; {hint}", cached.role, live.role),
552            live_text,
553        ));
554    }
555    if cached.identity != live_identity {
556        return Err(ElementError::stale(
557            format!(
558                "identifier changed from \"{}\" to \"{}\"; {hint}",
559                cached.identity, live_identity
560            ),
561            live_text,
562        ));
563    }
564    if !frames_match(cached.frame, live_frame) {
565        return Err(ElementError::stale(
566            format!(
567                "frame changed from {} to {}; {hint}",
568                frame_text(cached.frame),
569                frame_text(live_frame)
570            ),
571            live_text,
572        ));
573    }
574    let secure = cached.secure || tree::node_is_secure(live);
575    if secure && input.op.writes_text() {
576        return Err(ComputerError::PolicyDenied(
577            "refusing to read or write a secure text field".into(),
578        )
579        .into());
580    }
581    let window_frame = input.live.frame.map(SavedFrame::from);
582    let outside = if path.is_empty() {
583        false
584    } else {
585        match (live_frame, window_frame) {
586            (Some(element), Some(window)) => frame_outside(element, window),
587            _ => false,
588        }
589    };
590    let supports_scroll = live
591        .actions
592        .iter()
593        .any(|action| action == "AXScrollToVisible");
594    let already_scrolling = matches!(input.op, ElementOp::ScrollTo)
595        || matches!(input.op, ElementOp::Perform { name } if name == "AXScrollToVisible");
596    let (scroll_first, outside_warning) = if outside && !already_scrolling {
597        if supports_scroll {
598            (true, None)
599        } else if input.require_visible {
600            return Err(ComputerError::ExecutionFailed(
601                "element is outside the window and does not support AXScrollToVisible".into(),
602            )
603            .into());
604        } else {
605            (
606                false,
607                Some("element is outside the window and does not support AXScrollToVisible".into()),
608            )
609        }
610    } else {
611        (false, None)
612    };
613    let needs_settable = matches!(input.op, ElementOp::Type { .. });
614    let needs_chars = matches!(input.op, ElementOp::SelectText { .. });
615    Ok(Gate {
616        path,
617        scroll_first,
618        outside_warning,
619        needs_settable,
620        needs_chars,
621        actions: live.actions.clone(),
622        center: frame_center(live_frame),
623    })
624}
625
626fn frame_center(frame: Option<SavedFrame>) -> Option<(i32, i32)> {
627    let frame = frame?;
628    let half_w = i32::try_from(frame.width / 2).ok()?;
629    let half_h = i32::try_from(frame.height / 2).ok()?;
630    Some((
631        frame.x.saturating_add(half_w),
632        frame.y.saturating_add(half_h),
633    ))
634}
635
636/// Left press stays `AXPress`. Right uses `AXShowMenu` when that action is
637/// advertised. Every other button, and a right press without a menu, clicks
638/// the element center through the input tiers.
639pub fn press_route(
640    button: MouseButton,
641    count: u8,
642    actions: &[String],
643    center: Option<(i32, i32)>,
644) -> Result<Vec<Step>, ComputerError> {
645    validate_count(count)?;
646    if button == MouseButton::Left {
647        require_action(actions, "AXPress")?;
648        return Ok(vec![Step::Press { count }]);
649    }
650    if button == MouseButton::Right && actions.iter().any(|action| action == "AXShowMenu") {
651        return Ok(vec![Step::Perform {
652            name: "AXShowMenu".into(),
653        }]);
654    }
655    let Some((x, y)) = center else {
656        return Err(ComputerError::ExecutionFailed(
657            "element has no frame, so a coordinate click cannot be aimed".into(),
658        ));
659    };
660    Ok(vec![Step::CenterClick {
661        x,
662        y,
663        button,
664        count,
665    }])
666}
667
668/// Build the steps that run after settable and character-count reads.
669///
670/// `explicit_tier` is the raw value the caller passed for `--tier`. `None`
671/// or empty means "auto": the accessibility value is used when the element
672/// advertises it as settable; otherwise the keystroke fallback runs. A
673/// non-empty value (e.g. `"event"`) forces the keystroke path because
674/// `set_value` REPLACES the field contents and the caller explicitly asked
675/// for keystroke input.
676pub fn steps(
677    op: &ElementOp,
678    gate: &Gate,
679    settable: bool,
680    char_count: Option<i64>,
681    explicit_tier: Option<&str>,
682) -> Result<Vec<Step>, ElementError> {
683    match op {
684        ElementOp::Press { count, button } => {
685            press_route(*button, *count, &gate.actions, gate.center).map_err(ElementError::from)
686        }
687        ElementOp::Focus => Ok(vec![Step::Focus]),
688        ElementOp::SetValue { value } => {
689            validate_text(value, true)?;
690            Ok(vec![Step::SetValue {
691                value: value.clone(),
692            }])
693        }
694        ElementOp::SelectText { start, end } => {
695            validate_range(*start, *end, char_count)?;
696            Ok(vec![Step::SelectText {
697                start: *start,
698                end: *end,
699            }])
700        }
701        ElementOp::Perform { name } => {
702            validate_perform_name(name)?;
703            require_action(&gate.actions, name)?;
704            Ok(vec![Step::Perform { name: name.clone() }])
705        }
706        ElementOp::ScrollTo => {
707            require_action(&gate.actions, "AXScrollToVisible")?;
708            Ok(vec![Step::ScrollTo])
709        }
710        ElementOp::Type { text } => {
711            validate_text(text, false)?;
712            if settable && !explicit_non_auto_tier(explicit_tier) {
713                Ok(vec![Step::TypeSetValue { text: text.clone() }])
714            } else {
715                Ok(vec![Step::TypeEnigo { text: text.clone() }])
716            }
717        }
718    }
719}
720
721/// True when the caller passed an explicit, non-auto `--tier` flag. Used by
722/// `type` to keep `set_value` as an opt-in: `set_value` REPLACES the field
723/// contents, so an explicit `--tier event` (or any other named tier) must
724/// fall through to the keystroke path.
725fn explicit_non_auto_tier(raw: Option<&str>) -> bool {
726    matches!(
727        raw.map(str::trim).filter(|text| !text.is_empty()),
728        Some("event" | "synthetic" | "activate" | "ax")
729    )
730}
731
732/// `execution_failed` listing the actions CopyActionNames returned.
733pub fn require_action(actions: &[String], name: &str) -> Result<(), ComputerError> {
734    if actions.iter().any(|action| action == name) {
735        Ok(())
736    } else {
737        let listed = if actions.is_empty() {
738            "(none)".to_string()
739        } else {
740            actions.join(", ")
741        };
742        Err(ComputerError::ExecutionFailed(format!(
743            "element has no {name} action. actions: {listed}"
744        )))
745    }
746}
747
748/// Empty text is refused when `empty_ok` is false. A null byte is always refused.
749pub fn validate_text(text: &str, empty_ok: bool) -> Result<(), ComputerError> {
750    if !empty_ok && text.is_empty() {
751        return Err(ComputerError::InvalidArguments(
752            "text must not be empty".into(),
753        ));
754    }
755    reject_null(text)
756}
757
758fn reject_null(text: &str) -> Result<(), ComputerError> {
759    if text.contains('\0') {
760        Err(ComputerError::InvalidArguments(
761            "text must not contain a null byte".into(),
762        ))
763    } else {
764        Ok(())
765    }
766}
767
768fn frame_text(frame: Option<SavedFrame>) -> String {
769    match frame {
770        Some(frame) => format!("@{},{} {}x{}", frame.x, frame.y, frame.width, frame.height),
771        None => "none".into(),
772    }
773}
774
775#[cfg(test)]
776mod tests {
777    use super::*;
778    use crate::tree::NodeFrame;
779
780    fn frame(x: f64, y: f64, width: f64, height: f64) -> NodeFrame {
781        NodeFrame {
782            x,
783            y,
784            width,
785            height,
786        }
787    }
788
789    fn button(title: &str, origin_y: f64, actions: &[&str]) -> UiNode {
790        let mut node = UiNode::bare("AXButton");
791        node.title = Some(title.into());
792        node.frame = Some(frame(12.0, origin_y, 80.0, 24.0));
793        node.actions = actions.iter().map(|action| (*action).to_string()).collect();
794        node
795    }
796
797    fn window(child: UiNode) -> UiNode {
798        let mut root = UiNode::bare("AXWindow");
799        root.title = Some("Notes".into());
800        root.frame = Some(frame(0.0, 0.0, 800.0, 600.0));
801        root.child_count = Some(1);
802        root.children = vec![child];
803        root
804    }
805
806    fn input_for<'a>(
807        op: &'a ElementOp,
808        cached: &'a SigNode,
809        revision: u64,
810        live: &'a UiNode,
811        index: u32,
812    ) -> GateInput<'a> {
813        GateInput {
814            op,
815            index,
816            require_visible: false,
817            app_label: "Notes",
818            cached,
819            cached_revision: revision,
820            cached_pid: 10,
821            cached_window: 100,
822            resolved_pid: 10,
823            resolved_window: 100,
824            live,
825            live_pid: 10,
826            live_window: 100,
827        }
828    }
829
830    fn render_pair(root: &UiNode) -> (SigNode, u64) {
831        let rendered = tree::render(root);
832        let revision = tree::revision_of(&rendered.root);
833        (rendered.root, revision)
834    }
835
836    #[test]
837    fn same_role_allows_and_role_identifier_or_frame_changes_are_stale() {
838        let cached_ui = window(button("Save", 10.0, &["AXPress"]));
839        let (cached, revision) = render_pair(&cached_ui);
840        let op = ElementOp::Press {
841            count: 1,
842            button: MouseButton::Left,
843        };
844        let allowed = gate(&input_for(&op, &cached, revision, &cached_ui, 1)).unwrap();
845        assert_eq!(allowed.path, vec![0]);
846        assert!(!allowed.scroll_first);
847        let steps = steps(&op, &allowed, false, None, None).unwrap();
848        assert_eq!(steps, vec![Step::Press { count: 1 }]);
849
850        let mut changed_role = cached_ui.clone();
851        changed_role.children[0].role = "AXTextArea".into();
852        let error = gate(&input_for(&op, &cached, revision, &changed_role, 1)).unwrap_err();
853        assert_eq!(error.error.code(), "stale_element");
854        assert!(error.tree.unwrap().contains("AXTextArea"));
855
856        let mut changed_id = cached_ui.clone();
857        changed_id.children[0].title = Some("Delete".into());
858        let error = gate(&input_for(&op, &cached, revision, &changed_id, 1)).unwrap_err();
859        assert_eq!(error.error.code(), "stale_element");
860        assert!(error.error.to_string().contains("identifier changed"));
861
862        let mut changed_frame = cached_ui.clone();
863        changed_frame.children[0].frame = Some(frame(40.0, 10.0, 80.0, 24.0));
864        let error = gate(&input_for(&op, &cached, revision, &changed_frame, 1)).unwrap_err();
865        assert_eq!(error.error.code(), "stale_element");
866        assert!(error.error.to_string().contains("frame changed"));
867
868        let mut nudged = cached_ui.clone();
869        nudged.children[0].frame = Some(frame(14.0, 10.0, 80.0, 24.0));
870        assert!(gate(&input_for(&op, &cached, revision, &nudged, 1)).is_ok());
871    }
872
873    #[test]
874    fn secure_node_refuses_set_value_select_text_and_type() {
875        let mut field = UiNode::bare("AXSecureTextField");
876        field.title = Some("Password".into());
877        field.secure = true;
878        let root = window(field);
879        let (cached, revision) = render_pair(&root);
880        for op in [
881            ElementOp::SetValue {
882                value: "secret".into(),
883            },
884            ElementOp::SelectText { start: 0, end: 1 },
885            ElementOp::Type {
886                text: "secret".into(),
887            },
888        ] {
889            let error = gate(&input_for(&op, &cached, revision, &root, 1)).unwrap_err();
890            assert_eq!(error.error.code(), "policy_denied", "{op:?}");
891            assert!(error.tree.is_none());
892        }
893        let mut subrole = UiNode::bare("AXTextField");
894        subrole.subrole = Some("AXSecureTextField".into());
895        subrole.title = Some("Password".into());
896        let root = window(subrole);
897        let (cached, revision) = render_pair(&root);
898        let op = ElementOp::SetValue {
899            value: "secret".into(),
900        };
901        let error = gate(&input_for(&op, &cached, revision, &root, 1)).unwrap_err();
902        assert_eq!(error.error.code(), "policy_denied");
903    }
904
905    #[test]
906    fn allowlist_rejects_ax_delete_and_missing_action_lists_names() {
907        let error = validate_perform_name("AXDelete").unwrap_err();
908        assert_eq!(error.code(), "invalid_arguments");
909        assert!(error.to_string().contains("AXPress"));
910        let error = require_action(&["AXShowMenu".into()], "AXPress").unwrap_err();
911        assert_eq!(error.code(), "execution_failed");
912        assert!(error.to_string().contains("AXShowMenu"));
913    }
914
915    #[test]
916    fn scroll_to_is_requested_when_the_frame_is_outside_the_window() {
917        let cached_ui = window(button("Save", 700.0, &["AXPress", "AXScrollToVisible"]));
918        let (cached, revision) = render_pair(&cached_ui);
919        let press = ElementOp::Press {
920            count: 1,
921            button: MouseButton::Left,
922        };
923        let allowed = gate(&input_for(&press, &cached, revision, &cached_ui, 1)).unwrap();
924        assert!(allowed.scroll_first);
925        let scroll = ElementOp::ScrollTo;
926        let allowed = gate(&input_for(&scroll, &cached, revision, &cached_ui, 1)).unwrap();
927        assert!(!allowed.scroll_first);
928        let steps = steps(&scroll, &allowed, false, None, None).unwrap();
929        assert_eq!(steps, vec![Step::ScrollTo]);
930        let perform = ElementOp::Perform {
931            name: "AXScrollToVisible".into(),
932        };
933        let allowed = gate(&input_for(&perform, &cached, revision, &cached_ui, 1)).unwrap();
934        assert!(!allowed.scroll_first);
935    }
936
937    #[test]
938    fn require_visible_fails_when_the_element_cannot_scroll() {
939        let cached_ui = window(button("Save", 700.0, &["AXPress"]));
940        let (cached, revision) = render_pair(&cached_ui);
941        let press = ElementOp::Press {
942            count: 1,
943            button: MouseButton::Left,
944        };
945        let mut input = input_for(&press, &cached, revision, &cached_ui, 1);
946        input.require_visible = true;
947        let error = gate(&input).unwrap_err();
948        assert_eq!(error.error.code(), "execution_failed");
949        input.require_visible = false;
950        let allowed = gate(&input).unwrap();
951        assert!(!allowed.scroll_first);
952        assert!(allowed.outside_warning.is_some());
953    }
954
955    #[test]
956    fn mismatched_revision_or_window_is_stale_and_a_missing_index_is_not_found() {
957        let cached_ui = window(button("Save", 10.0, &["AXPress"]));
958        let (cached, revision) = render_pair(&cached_ui);
959        let op = ElementOp::Focus;
960        let mut input = input_for(&op, &cached, revision, &cached_ui, 1);
961        input.cached_revision = revision.wrapping_add(1);
962        let error = gate(&input).unwrap_err();
963        assert_eq!(error.error.code(), "stale_element");
964        assert!(error
965            .error
966            .to_string()
967            .contains("cached revision/pid/window does not match"));
968
969        input.cached_revision = revision;
970        input.live_window = 9;
971        let error = gate(&input).unwrap_err();
972        assert_eq!(error.error.code(), "stale_element");
973
974        input.live_window = 100;
975        input.index = 9;
976        let error = gate(&input).unwrap_err();
977        assert_eq!(error.error.code(), "element_not_found");
978        assert!(error.error.to_string().contains("refresh indices"));
979    }
980
981    #[test]
982    fn select_range_must_sit_inside_the_character_count() {
983        assert!(validate_range(0, 5, Some(5)).is_ok());
984        assert_eq!(
985            validate_range(4, 2, None).unwrap_err().code(),
986            "invalid_arguments"
987        );
988        assert_eq!(
989            validate_range(0, 6, Some(5)).unwrap_err().code(),
990            "invalid_arguments"
991        );
992    }
993
994    #[test]
995    fn explicit_event_tier_skips_set_value_on_a_settable_field() {
996        // The bug: `type --element <N> --tier event` still went down the
997        // set_value path. set_value REPLACES the field contents; the user
998        // asked for keystrokes. With --tier event set the path must fall
999        // through to TypeEnigo even when the element advertises settable.
1000        let op = ElementOp::Type {
1001            text: "hello".into(),
1002        };
1003        // `settable = true` would otherwise pick TypeSetValue; the explicit
1004        // tier forces TypeEnigo.
1005        let steps = steps(&op, &stub_gate_for_type(), true, Some(0), Some("event")).unwrap();
1006        assert_eq!(
1007            steps,
1008            vec![Step::TypeEnigo {
1009                text: "hello".into()
1010            }]
1011        );
1012    }
1013
1014    #[test]
1015    fn explicit_synthetic_tier_also_skips_set_value() {
1016        let op = ElementOp::Type { text: "x".into() };
1017        let steps = steps(&op, &stub_gate_for_type(), true, Some(1), Some("synthetic")).unwrap();
1018        assert_eq!(steps, vec![Step::TypeEnigo { text: "x".into() }]);
1019    }
1020
1021    #[test]
1022    fn no_explicit_tier_keeps_set_value_on_a_settable_field() {
1023        // The default ("auto") keeps the original behaviour: a settable
1024        // text field still goes through set_value. The explicit tier flag
1025        // is what flips the choice.
1026        let op = ElementOp::Type {
1027            text: "world".into(),
1028        };
1029        let steps = steps(&op, &stub_gate_for_type(), true, Some(0), None).unwrap();
1030        assert_eq!(
1031            steps,
1032            vec![Step::TypeSetValue {
1033                text: "world".into()
1034            }]
1035        );
1036    }
1037
1038    #[test]
1039    fn no_explicit_tier_falls_through_to_keystrokes_when_not_settable() {
1040        let op = ElementOp::Type { text: "x".into() };
1041        let steps = steps(&op, &stub_gate_for_type(), false, None, None).unwrap();
1042        assert_eq!(steps, vec![Step::TypeEnigo { text: "x".into() }]);
1043    }
1044
1045    /// A minimal gate for the type action. The `Type` branch in `steps`
1046    /// only inspects `gate.actions` for the ScrollTo path, and our tests
1047    /// here exercise the `Type` branch directly, so the gate contents are
1048    /// unused — but `steps` takes the gate by reference, so we still need
1049    /// to pass something that satisfies the type.
1050    fn stub_gate_for_type() -> Gate {
1051        Gate {
1052            path: Vec::new(),
1053            scroll_first: false,
1054            outside_warning: None,
1055            needs_settable: false,
1056            needs_chars: false,
1057            actions: Vec::new(),
1058            center: None,
1059        }
1060    }
1061}