computer_use/
element.rs

1use crate::geometry::Rect;
2
3bitflags::bitflags! {
4    /// Boolean accessibility states of an element.
5    #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
6    pub struct ElementStates: u16 {
7        /// Has keyboard focus.
8        const FOCUSED = 1 << 0;
9        /// Is selected (rows, tabs, list items).
10        const SELECTED = 1 << 1;
11        /// Is disabled and will ignore input.
12        const DISABLED = 1 << 2;
13        /// Is expanded (disclosure triangles, combo boxes, tree rows).
14        const EXPANDED = 1 << 3;
15        /// Is collapsed but can be expanded.
16        const COLLAPSED = 1 << 4;
17        /// Is checked or toggled on.
18        const CHECKED = 1 << 5;
19        /// Its value can be written with `set_value`.
20        const SETTABLE = 1 << 6;
21        /// Accepts text input.
22        const EDITABLE = 1 << 7;
23        /// Is not visible on screen (off-screen or scrolled away).
24        const OFFSCREEN = 1 << 8;
25    }
26}
27
28impl ElementStates {
29    /// Short labels used when rendering the tree.
30    #[must_use]
31    pub fn labels(self) -> Vec<&'static str> {
32        const LABELS: [(ElementStates, &str); 9] = [
33            (ElementStates::FOCUSED, "focused"),
34            (ElementStates::SELECTED, "selected"),
35            (ElementStates::DISABLED, "disabled"),
36            (ElementStates::EXPANDED, "expanded"),
37            (ElementStates::COLLAPSED, "collapsed"),
38            (ElementStates::CHECKED, "checked"),
39            (ElementStates::SETTABLE, "settable"),
40            (ElementStates::EDITABLE, "editable"),
41            (ElementStates::OFFSCREEN, "offscreen"),
42        ];
43        LABELS.iter().filter(|(flag, _)| self.contains(*flag)).map(|(_, label)| *label).collect()
44    }
45}
46
47/// An accessibility action an element exposes.
48#[derive(Debug, Clone, PartialEq, Eq)]
49pub struct Action {
50    /// The platform's raw action name, e.g. `AXPress`, `Invoke`, `click`.
51    pub name: String,
52    /// Human-readable label, e.g. `Press`, `Show Menu`.
53    pub label: String,
54    /// Whether this is the element's default "click" action.
55    pub primary: bool,
56}
57
58impl Action {
59    /// Creates an action; the label is derived from the raw name.
60    #[must_use]
61    pub fn new(name: impl Into<String>, primary: bool) -> Self {
62        let name = name.into();
63        let label = humanize_action_name(&name);
64        Self { name, label, primary }
65    }
66
67    /// True when `query` names this action by raw name or label, ignoring case,
68    /// spaces, and an `AX` prefix.
69    #[must_use]
70    pub fn matches(&self, query: &str) -> bool {
71        let q = normalize_action(query);
72        !q.is_empty() && (normalize_action(&self.name) == q || normalize_action(&self.label) == q)
73    }
74}
75
76fn normalize_action(value: &str) -> String {
77    let lowered: String = value.chars().filter(|c| c.is_alphanumeric()).flat_map(char::to_lowercase).collect();
78    lowered.strip_prefix("ax").map(str::to_owned).unwrap_or(lowered)
79}
80
81fn humanize_action_name(name: &str) -> String {
82    let trimmed = name.strip_prefix("AX").unwrap_or(name);
83    let mut out = String::with_capacity(trimmed.len() + 4);
84    let mut prev_lower = false;
85    let mut start_word = true;
86    for c in trimmed.chars() {
87        if c == '_' || c == '-' || c == ' ' {
88            if !out.is_empty() && !out.ends_with(' ') {
89                out.push(' ');
90            }
91            start_word = true;
92            prev_lower = false;
93            continue;
94        }
95        if c.is_uppercase() && prev_lower {
96            out.push(' ');
97            start_word = true;
98        }
99        if start_word {
100            out.extend(c.to_uppercase());
101        } else {
102            out.push(c);
103        }
104        start_word = false;
105        prev_lower = c.is_lowercase() || c.is_ascii_digit();
106    }
107    out
108}
109
110/// Raw attributes of an element, as reported by a [`Backend`](crate::Backend).
111///
112/// Backends fill this in; the crate assigns indexes, prunes uninteresting containers, and
113/// truncates text to produce [`Element`]s.
114#[derive(Debug, Clone, Default, PartialEq)]
115#[non_exhaustive]
116pub struct ElementData {
117    /// Display role, lowercase and human-readable (`button`, `text field`).
118    pub role: String,
119    /// The platform's raw role (`AXButton`, `ControlType.Button`, `push button`).
120    pub platform_role: String,
121    /// Primary label or name.
122    pub title: Option<String>,
123    /// Current value (text contents, slider position, checkbox state).
124    pub value: Option<String>,
125    /// Accessibility description, when different from the title.
126    pub description: Option<String>,
127    /// Tooltip or help text.
128    pub help: Option<String>,
129    /// Placeholder text of empty text fields.
130    pub placeholder: Option<String>,
131    /// Developer-assigned identifier.
132    pub identifier: Option<String>,
133    /// Link target.
134    pub url: Option<String>,
135    /// Frame in screen points (not window-relative).
136    pub frame: Option<Rect>,
137    /// States.
138    pub states: ElementStates,
139    /// Available actions.
140    pub actions: Vec<Action>,
141}
142
143impl ElementData {
144    /// Creates data for an element with the given display and platform role.
145    #[must_use]
146    pub fn new(role: impl Into<String>, platform_role: impl Into<String>) -> Self {
147        Self { role: role.into(), platform_role: platform_role.into(), ..Self::default() }
148    }
149
150    fn is_noise(&self) -> bool {
151        const CONTAINER_ROLES: [&str; 8] =
152            ["group", "unknown", "generic", "filler", "panel", "pane", "section", "layout table"];
153        CONTAINER_ROLES.contains(&self.role.as_str())
154            && self.title.is_none()
155            && self.value.is_none()
156            && self.description.is_none()
157            && self.identifier.is_none()
158            && self.url.is_none()
159            && !self.actions.iter().any(|a| !a.primary)
160            && !self.states.intersects(ElementStates::FOCUSED | ElementStates::SELECTED)
161    }
162}
163
164/// A node in a captured accessibility tree.
165#[derive(Debug, Clone, PartialEq)]
166#[non_exhaustive]
167pub struct Element {
168    /// Index of this element within its [`AppState`](crate::AppState); pass it to element actions.
169    pub index: usize,
170    /// Index of the nearest rendered ancestor.
171    pub parent: Option<usize>,
172    /// Indexes of rendered children.
173    pub children: Vec<usize>,
174    /// Nesting depth below the window.
175    pub depth: usize,
176    /// Display role.
177    pub role: String,
178    /// Raw platform role.
179    pub platform_role: String,
180    /// Primary label.
181    pub title: Option<String>,
182    /// Current value.
183    pub value: Option<String>,
184    /// Accessibility description.
185    pub description: Option<String>,
186    /// Help text.
187    pub help: Option<String>,
188    /// Placeholder text.
189    pub placeholder: Option<String>,
190    /// Developer identifier.
191    pub identifier: Option<String>,
192    /// Link target.
193    pub url: Option<String>,
194    /// Frame in screen points.
195    pub frame: Option<Rect>,
196    /// States.
197    pub states: ElementStates,
198    /// Available actions.
199    pub actions: Vec<Action>,
200}
201
202impl Element {
203    /// The default click action, if any.
204    #[must_use]
205    pub fn primary_action(&self) -> Option<&Action> {
206        self.actions.iter().find(|a| a.primary)
207    }
208
209    /// Finds an action by raw name or label.
210    #[must_use]
211    pub fn action(&self, query: &str) -> Option<&Action> {
212        self.actions.iter().find(|a| a.matches(query))
213    }
214
215    /// Text that best describes the element: title, then description, then value.
216    #[must_use]
217    pub fn label(&self) -> Option<&str> {
218        self.title.as_deref().or(self.description.as_deref()).or(self.value.as_deref())
219    }
220
221    pub(crate) fn text_fields(&self) -> impl Iterator<Item = &str> {
222        [
223            self.title.as_deref(),
224            self.description.as_deref(),
225            self.value.as_deref(),
226            self.placeholder.as_deref(),
227            self.identifier.as_deref(),
228            self.help.as_deref(),
229        ]
230        .into_iter()
231        .flatten()
232    }
233}
234
235/// A raw node returned by a backend, in depth-first pre-order.
236#[derive(Debug, Clone)]
237pub struct RawNode<H> {
238    /// Backend handle used to act on the element later.
239    pub handle: H,
240    /// Index of the parent within the same `Vec`, `None` for the root.
241    pub parent: Option<usize>,
242    /// The element's attributes.
243    pub data: ElementData,
244}
245
246pub(crate) fn build_tree<H>(
247    nodes: Vec<RawNode<H>>,
248    text_limit: Option<usize>,
249) -> (Vec<Element>, Vec<H>, Vec<Option<usize>>) {
250    let mut elements: Vec<Element> = Vec::with_capacity(nodes.len());
251    let mut handles = Vec::with_capacity(nodes.len());
252    let mut kept_ancestor: Vec<Option<usize>> = Vec::with_capacity(nodes.len());
253    let mut raw_to_element: Vec<Option<usize>> = Vec::with_capacity(nodes.len());
254
255    for (raw_index, node) in nodes.into_iter().enumerate() {
256        let parent = node.parent.filter(|&p| p < raw_index).and_then(|p| kept_ancestor[p]);
257        let is_root = node.parent.is_none();
258        if !is_root && node.data.is_noise() {
259            kept_ancestor.push(parent);
260            raw_to_element.push(None);
261            continue;
262        }
263        let index = elements.len();
264        let depth = parent.map_or(0, |p| elements[p].depth + 1);
265        if let Some(p) = parent {
266            elements[p].children.push(index);
267        }
268        let d = node.data;
269        let clip = |s: Option<String>| s.map(|s| sanitize(&s, text_limit)).filter(|s| !s.is_empty());
270        elements.push(Element {
271            index,
272            parent,
273            children: Vec::new(),
274            depth,
275            role: d.role,
276            platform_role: d.platform_role,
277            title: clip(d.title),
278            value: clip(d.value),
279            description: clip(d.description),
280            help: clip(d.help),
281            placeholder: clip(d.placeholder),
282            identifier: clip(d.identifier),
283            url: clip(d.url),
284            frame: d.frame,
285            states: d.states,
286            actions: d.actions,
287        });
288        handles.push(node.handle);
289        kept_ancestor.push(Some(index));
290        raw_to_element.push(Some(index));
291    }
292    (elements, handles, raw_to_element)
293}
294
295pub(crate) fn sanitize(text: &str, limit: Option<usize>) -> String {
296    let mut out = String::with_capacity(text.len().min(limit.unwrap_or(usize::MAX)));
297    let mut count = 0usize;
298    let mut pending_space = false;
299    for c in text.chars() {
300        if c.is_whitespace() || c.is_control() {
301            pending_space = !out.is_empty();
302            continue;
303        }
304        let needed = usize::from(pending_space) + 1;
305        if limit.is_some_and(|l| count + needed > l) {
306            out.push('…');
307            return out;
308        }
309        if pending_space {
310            out.push(' ');
311            count += 1;
312            pending_space = false;
313        }
314        out.push(c);
315        count += 1;
316    }
317    out
318}
319
320#[cfg(test)]
321mod tests {
322    use super::*;
323
324    fn node(parent: Option<usize>, role: &str, title: Option<&str>) -> RawNode<u32> {
325        let mut data = ElementData::new(role, role);
326        data.title = title.map(str::to_owned);
327        RawNode { handle: 0, parent, data }
328    }
329
330    #[test]
331    fn humanizes_actions() {
332        assert_eq!(humanize_action_name("AXShowMenu"), "Show Menu");
333        assert_eq!(humanize_action_name("show_menu"), "Show Menu");
334        assert_eq!(humanize_action_name("AXScrollDownByPage"), "Scroll Down By Page");
335        assert!(Action::new("AXShowMenu", false).matches("show menu"));
336        assert!(Action::new("AXShowMenu", false).matches("ShowMenu"));
337        assert!(!Action::new("AXPress", true).matches("show menu"));
338    }
339
340    #[test]
341    fn prunes_anonymous_groups_and_reparents() {
342        let nodes = vec![
343            node(None, "window", Some("Main")),
344            node(Some(0), "group", None),
345            node(Some(1), "button", Some("OK")),
346            node(Some(0), "group", Some("Named")),
347            node(Some(3), "text", Some("hi")),
348        ];
349        let (elements, _, map) = build_tree(nodes, None);
350        let roles: Vec<_> = elements.iter().map(|e| (e.index, e.parent, e.depth, e.role.as_str())).collect();
351        assert_eq!(
352            roles,
353            vec![(0, None, 0, "window"), (1, Some(0), 1, "button"), (2, Some(0), 1, "group"), (3, Some(2), 2, "text"),]
354        );
355        assert_eq!(map, vec![Some(0), None, Some(1), Some(2), Some(3)]);
356        assert_eq!(elements[0].children, vec![1, 2]);
357    }
358
359    #[test]
360    fn sanitize_collapses_and_truncates() {
361        assert_eq!(sanitize("  a \n\t b  ", None), "a b");
362        assert_eq!(sanitize("hello world", Some(5)), "hello…");
363        assert_eq!(sanitize("héllo", Some(10)), "héllo");
364    }
365}
computer_use/element.rs

computer_use/
element.rs
