computer_use/
state.rs

1use std::fmt::{self, Write as _};
2use std::time::SystemTime;
3
4use crate::app::AppInfo;
5use crate::element::{Element, ElementStates};
6use crate::geometry::Rect;
7use crate::screenshot::Screenshot;
8
9/// The window an [`AppState`] was captured from.
10#[derive(Debug, Clone, PartialEq, Default)]
11#[non_exhaustive]
12pub struct WindowInfo {
13    /// Platform window ID, when known: the `CGWindowID` on macOS, the `HWND` on Windows, and an
14    /// opaque ID derived from the window's accessibility object on Linux. Pass it to
15    /// [`WindowSelector::Id`] to capture this window again.
16    pub id: Option<u64>,
17    /// Window title.
18    pub title: Option<String>,
19    /// Frame in screen points.
20    pub frame: Rect,
21}
22
23impl WindowInfo {
24    /// Creates window info.
25    #[must_use]
26    pub const fn new(id: Option<u64>, title: Option<String>, frame: Rect) -> Self {
27        Self { id, title, frame }
28    }
29}
30
31/// Which of an app's windows to capture.
32///
33/// [`Computer::list_windows`](crate::Computer::list_windows) lists the candidates.
34#[derive(Debug, Clone, PartialEq, Eq, Hash)]
35#[non_exhaustive]
36pub enum WindowSelector {
37    /// A [`WindowInfo::id`].
38    Id(u64),
39    /// A window title, matched case-insensitively: an exact match wins, then the first window
40    /// whose title contains it.
41    Title(String),
42}
43
44/// Limits and toggles for [`Computer::get_app_state`](crate::Computer::get_app_state).
45#[derive(Debug, Clone, PartialEq, Eq)]
46#[non_exhaustive]
47#[allow(clippy::struct_excessive_bools)] // independent toggles, each with a builder
48pub struct CaptureOptions {
49    /// Walk the accessibility tree. Disabling this invalidates element indexes.
50    pub include_tree: bool,
51    /// Capture a screenshot. Disabling this prevents coordinate-based actions.
52    pub include_screenshot: bool,
53    /// Maximum characters kept per text attribute; `None` keeps everything.
54    pub text_limit: Option<usize>,
55    /// Maximum number of raw accessibility nodes visited.
56    pub max_nodes: usize,
57    /// Maximum tree depth visited.
58    pub max_depth: usize,
59    /// Screenshots larger than this on either side are downscaled; `None` keeps full resolution.
60    pub max_screenshot_dimension: Option<u32>,
61    /// Launch the app (without activating it) if it is installed but not running.
62    pub launch_if_needed: bool,
63    /// The window to capture; `None` picks the focused or main window.
64    pub window: Option<WindowSelector>,
65    /// Also capture the app's menu bar, as a second root after the window's tree.
66    ///
67    /// Only macOS keeps the menu bar outside the window; on Windows and Linux it is already
68    /// part of the window's tree, so this has no effect there.
69    pub include_menu_bar: bool,
70}
71
72impl Default for CaptureOptions {
73    fn default() -> Self {
74        Self {
75            include_tree: true,
76            include_screenshot: true,
77            text_limit: Some(500),
78            max_nodes: 1200,
79            max_depth: 64,
80            max_screenshot_dimension: Some(1280),
81            launch_if_needed: true,
82            window: None,
83            include_menu_bar: false,
84        }
85    }
86}
87
88impl CaptureOptions {
89    /// Sets [`include_tree`](Self::include_tree).
90    #[must_use]
91    pub const fn include_tree(mut self, yes: bool) -> Self {
92        self.include_tree = yes;
93        self
94    }
95
96    /// Sets [`include_screenshot`](Self::include_screenshot).
97    #[must_use]
98    pub const fn include_screenshot(mut self, yes: bool) -> Self {
99        self.include_screenshot = yes;
100        self
101    }
102
103    /// Sets [`text_limit`](Self::text_limit).
104    #[must_use]
105    pub const fn text_limit(mut self, limit: Option<usize>) -> Self {
106        self.text_limit = limit;
107        self
108    }
109
110    /// Sets [`max_nodes`](Self::max_nodes).
111    #[must_use]
112    pub const fn max_nodes(mut self, max: usize) -> Self {
113        self.max_nodes = max;
114        self
115    }
116
117    /// Sets [`max_depth`](Self::max_depth).
118    #[must_use]
119    pub const fn max_depth(mut self, max: usize) -> Self {
120        self.max_depth = max;
121        self
122    }
123
124    /// Sets [`max_screenshot_dimension`](Self::max_screenshot_dimension).
125    #[must_use]
126    pub const fn max_screenshot_dimension(mut self, max: Option<u32>) -> Self {
127        self.max_screenshot_dimension = max;
128        self
129    }
130
131    /// Sets [`launch_if_needed`](Self::launch_if_needed).
132    #[must_use]
133    pub const fn launch_if_needed(mut self, yes: bool) -> Self {
134        self.launch_if_needed = yes;
135        self
136    }
137
138    /// Sets [`window`](Self::window).
139    #[must_use]
140    pub fn window(mut self, window: WindowSelector) -> Self {
141        self.window = Some(window);
142        self
143    }
144
145    /// Sets [`include_menu_bar`](Self::include_menu_bar).
146    #[must_use]
147    pub const fn include_menu_bar(mut self, yes: bool) -> Self {
148        self.include_menu_bar = yes;
149        self
150    }
151}
152
153/// A snapshot of one app window.
154#[derive(Debug, Clone)]
155#[non_exhaustive]
156pub struct AppState {
157    /// The app.
158    pub app: AppInfo,
159    /// The captured window.
160    pub window: WindowInfo,
161    /// Accessibility elements in depth-first order; `elements[i].index == i`.
162    pub elements: Vec<Element>,
163    /// Whether node or depth limits cut the tree short.
164    pub truncated: bool,
165    /// Whether the tree was requested at all.
166    pub tree_included: bool,
167    /// Index of the focused element, if it is part of the tree.
168    pub focused: Option<usize>,
169    /// Currently selected text in the focused element.
170    pub selected_text: Option<String>,
171    /// Screenshot of the window.
172    pub screenshot: Option<Screenshot>,
173    /// When the state was captured.
174    pub captured_at: SystemTime,
175    /// Non-fatal problems encountered while capturing, e.g. why the screenshot is missing.
176    pub notes: Vec<String>,
177}
178
179impl AppState {
180    /// Looks up an element by index.
181    #[must_use]
182    pub fn element(&self, index: usize) -> Option<&Element> {
183        self.elements.get(index)
184    }
185
186    /// The focused element.
187    #[must_use]
188    pub fn focused_element(&self) -> Option<&Element> {
189        self.focused.and_then(|i| self.element(i))
190    }
191
192    /// Elements matching `query`, in tree order.
193    pub fn find(&self, query: &ElementQuery) -> impl Iterator<Item = &Element> + use<'_> {
194        let query = query.clone();
195        let limit = query.limit;
196        self.elements.iter().filter(move |e| query.matches(e)).take(limit)
197    }
198
199    /// The deepest element whose frame contains the screen point, preferring actionable ones.
200    #[must_use]
201    pub fn element_at(&self, point: crate::geometry::Point) -> Option<&Element> {
202        self.elements.iter().filter(|e| e.frame.is_some_and(|f| f.contains(point))).min_by(|a, b| {
203            let rank = |e: &Element| (e.primary_action().is_none(), e.frame.map_or(f64::MAX, |f| f.area()));
204            let (ra, rb) = (rank(a), rank(b));
205            ra.0.cmp(&rb.0).then(ra.1.total_cmp(&rb.1))
206        })
207    }
208
209    /// Renders the state as compact, human-readable text (also handy as language-model input).
210    ///
211    /// Each element is one tab-indented line beginning with its index:
212    ///
213    /// ```text
214    /// App=com.apple.TextEdit (pid 812)
215    /// Window: "Untitled", App: TextEdit.
216    /// 0 standard window Untitled
217    ///     1 text area (focused, settable, editable) Value: Hello
218    ///     2 button Close Secondary Actions: Show Menu
219    /// ```
220    #[must_use]
221    pub fn render(&self) -> String {
222        let mut out = String::new();
223        let pid = self.app.pid.map_or_else(String::new, |p| format!(" (pid {p})"));
224        let _ = writeln!(out, "App={}{pid}", self.app.reference());
225        let title = self.window.title.as_deref().filter(|t| !t.is_empty()).unwrap_or(&self.app.name);
226        let _ = write!(out, "Window: {title:?}, App: {}.", self.app.name);
227        for element in &self.elements {
228            out.push('\n');
229            for _ in 0..element.depth {
230                out.push('\t');
231            }
232            render_element(&mut out, element);
233        }
234        if !self.tree_included {
235            out.push_str("\nAccessibility tree omitted (include_tree = false); element indexes are unavailable.");
236        } else if self.truncated {
237            out.push_str("\n… tree truncated by node or depth limits.");
238        }
239        for note in &self.notes {
240            let _ = write!(out, "\n{note}");
241        }
242        if let Some(selected) = self.selected_text.as_deref().filter(|s| !s.is_empty()) {
243            let _ = write!(out, "\n\nSelected text: [{selected}]");
244        } else if let Some(focused) = self.focused_element() {
245            out.push_str("\n\nThe focused UI element is ");
246            render_element(&mut out, focused);
247            out.push('.');
248        }
249        out
250    }
251}
252
253impl fmt::Display for AppState {
254    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
255        f.write_str(&self.render())
256    }
257}
258
259fn render_element(out: &mut String, e: &Element) {
260    let _ = write!(out, "{} {}", e.index, e.role);
261    let states = (e.states - ElementStates::OFFSCREEN).labels();
262    if !states.is_empty() {
263        let _ = write!(out, " ({})", states.join(", "));
264    }
265    if let Some(title) = &e.title {
266        let _ = write!(out, " {title}");
267    }
268    let mut field = |name: &str, value: Option<&String>| {
269        if let Some(v) = value.filter(|v| Some(*v) != e.title.as_ref()) {
270            let _ = write!(out, " {name}: {v}");
271        }
272    };
273    field("Description", e.description.as_ref());
274    field("Help", e.help.as_ref().filter(|h| Some(*h) != e.description.as_ref()));
275    field("URL", e.url.as_ref());
276    field("ID", e.identifier.as_ref());
277    field("Value", e.value.as_ref());
278    if e.value.is_none() {
279        field("Placeholder", e.placeholder.as_ref());
280    }
281    let secondary: Vec<&str> = e.actions.iter().filter(|a| !a.primary).map(|a| a.label.as_str()).collect();
282    if !secondary.is_empty() {
283        let _ = write!(out, " Secondary Actions: {}", secondary.join(", "));
284    }
285}
286
287/// Criteria for [`Computer::query`](crate::Computer::query) and [`AppState::find`].
288#[derive(Debug, Clone, PartialEq, Eq)]
289pub struct ElementQuery {
290    /// Text matched against title, description, value, placeholder, identifier, and help.
291    pub text: Option<String>,
292    /// Role matched against the display role or platform role (an `AX` prefix is optional).
293    pub role: Option<String>,
294    /// Require an exact (case-insensitive) text match instead of a substring.
295    pub exact: bool,
296    /// Maximum number of matches.
297    pub limit: usize,
298}
299
300impl Default for ElementQuery {
301    fn default() -> Self {
302        Self { text: None, role: None, exact: false, limit: 20 }
303    }
304}
305
306impl ElementQuery {
307    /// Matches elements containing `text`.
308    #[must_use]
309    pub fn text(text: impl Into<String>) -> Self {
310        Self { text: Some(text.into()), ..Self::default() }
311    }
312
313    /// Matches elements with `role`.
314    #[must_use]
315    pub fn role(role: impl Into<String>) -> Self {
316        Self { role: Some(role.into()), ..Self::default() }
317    }
318
319    /// Adds a role constraint.
320    #[must_use]
321    pub fn with_role(mut self, role: impl Into<String>) -> Self {
322        self.role = Some(role.into());
323        self
324    }
325
326    /// Requires exact text matches.
327    #[must_use]
328    pub const fn exact(mut self, exact: bool) -> Self {
329        self.exact = exact;
330        self
331    }
332
333    /// Caps the number of results.
334    #[must_use]
335    pub const fn limit(mut self, limit: usize) -> Self {
336        self.limit = limit;
337        self
338    }
339
340    /// Whether `element` satisfies the query.
341    #[must_use]
342    pub fn matches(&self, element: &Element) -> bool {
343        if let Some(role) = &self.role {
344            let want = normalize_role(role);
345            if normalize_role(&element.role) != want && normalize_role(&element.platform_role) != want {
346                return false;
347            }
348        }
349        if let Some(text) = &self.text {
350            let needle = text.to_lowercase();
351            let hit = element.text_fields().any(|field| {
352                let field = field.to_lowercase();
353                if self.exact { field == needle } else { field.contains(&needle) }
354            });
355            if !hit {
356                return false;
357            }
358        }
359        true
360    }
361}
362
363fn normalize_role(role: &str) -> String {
364    let lowered: String = role.chars().filter(|c| c.is_alphanumeric()).flat_map(char::to_lowercase).collect();
365    lowered.strip_prefix("ax").map(str::to_owned).unwrap_or(lowered)
366}
367
368#[cfg(test)]
369mod tests {
370    use super::*;
371    use crate::element::{Action, ElementData, RawNode, build_tree};
372
373    pub(crate) fn sample_state() -> AppState {
374        let mut window = ElementData::new("standard window", "AXWindow");
375        window.title = Some("Doc".into());
376        let mut text = ElementData::new("text area", "AXTextArea");
377        text.value = Some("Hello world".into());
378        text.states = ElementStates::FOCUSED | ElementStates::SETTABLE;
379        text.frame = Some(Rect::new(0.0, 20.0, 200.0, 100.0));
380        let mut button = ElementData::new("button", "AXButton");
381        button.title = Some("Save".into());
382        button.actions = vec![Action::new("AXPress", true), Action::new("AXShowMenu", false)];
383        button.frame = Some(Rect::new(10.0, 130.0, 50.0, 20.0));
384        let nodes = vec![
385            RawNode { handle: (), parent: None, data: window },
386            RawNode { handle: (), parent: Some(0), data: text },
387            RawNode { handle: (), parent: Some(0), data: button },
388        ];
389        let (elements, _, _) = build_tree(nodes, Some(500));
390        AppState {
391            app: AppInfo::new("TextEdit").with_bundle_id("com.apple.TextEdit").with_pid(9),
392            window: WindowInfo::new(Some(1), Some("Doc".into()), Rect::new(0.0, 0.0, 300.0, 200.0)),
393            elements,
394            truncated: false,
395            tree_included: true,
396            focused: Some(1),
397            selected_text: None,
398            screenshot: None,
399            captured_at: SystemTime::UNIX_EPOCH,
400            notes: Vec::new(),
401        }
402    }
403
404    #[test]
405    fn renders_tree() {
406        let text = sample_state().render();
407        let expected = "App=com.apple.TextEdit (pid 9)\n\
408            Window: \"Doc\", App: TextEdit.\n\
409            0 standard window Doc\n\
410            \t1 text area (focused, settable) Value: Hello world\n\
411            \t2 button Save Secondary Actions: Show Menu\n\
412            \n\
413            The focused UI element is 1 text area (focused, settable) Value: Hello world.";
414        assert_eq!(text, expected);
415    }
416
417    #[test]
418    fn queries_by_text_and_role() {
419        let state = sample_state();
420        let q = ElementQuery::text("save");
421        assert_eq!(state.find(&q).map(|e| e.index).collect::<Vec<_>>(), vec![2]);
422        let q = ElementQuery::role("AXTextArea");
423        assert_eq!(state.find(&q).map(|e| e.index).collect::<Vec<_>>(), vec![1]);
424        let q = ElementQuery::role("button").exact(true);
425        assert_eq!(state.find(&q).count(), 1);
426        let q = ElementQuery::text("hello").exact(true);
427        assert_eq!(state.find(&q).count(), 0);
428    }
429
430    #[test]
431    fn hit_tests_prefer_actionable_elements() {
432        let state = sample_state();
433        let hit = state.element_at(crate::geometry::Point::new(20.0, 140.0));
434        assert_eq!(hit.map(|e| e.index), Some(2));
435    }
436}
computer_use/state.rs

computer_use/
state.rs
