computer_use/
computer.rs

1use std::collections::HashMap;
2use std::thread;
3use std::time::{Duration, SystemTime};
4
5use crate::app::{self, AppInfo};
6use crate::backend::{Backend, CaptureRequest, Delivery, InputTarget, MouseButton, Permissions, ScrollDirection};
7use crate::element::{Element, build_tree};
8use crate::error::{Error, Result};
9use crate::geometry::Point;
10use crate::keys::KeyChord;
11use crate::screenshot::Screenshot;
12use crate::state::{AppState, CaptureOptions, ElementQuery};
13use crate::text::TextSelection;
14
15/// What to click.
16#[derive(Debug, Clone, Copy, PartialEq)]
17#[non_exhaustive]
18pub enum ClickTarget {
19    /// An element index from the latest [`AppState`].
20    Element(usize),
21    /// A pixel position in the latest screenshot.
22    Pixel {
23        /// Horizontal pixel.
24        x: f64,
25        /// Vertical pixel.
26        y: f64,
27    },
28}
29
30/// How a click is performed.
31#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
32#[non_exhaustive]
33pub enum ClickMethod {
34    /// Use the element's accessibility action when that is equivalent, otherwise synthesize
35    /// pointer events with the computer's default [`Delivery`].
36    #[default]
37    Auto,
38    /// Only use accessibility actions; requires an element target.
39    Accessibility,
40    /// Synthesize pointer events delivered to the app in the background.
41    Background,
42    /// Like `Background`, with the target window given synthetic focus first
43    /// (see [`Delivery::FocusedBackground`]).
44    FocusedBackground,
45    /// Synthesize pointer events through the global input stream.
46    Global,
47}
48
49impl std::str::FromStr for ClickMethod {
50    type Err = Error;
51
52    fn from_str(s: &str) -> Result<Self> {
53        match s.trim().to_ascii_lowercase().as_str() {
54            "auto" => Ok(Self::Auto),
55            "accessibility" | "ax" => Ok(Self::Accessibility),
56            "background" => Ok(Self::Background),
57            "focused_background" | "focused-background" => Ok(Self::FocusedBackground),
58            "global" => Ok(Self::Global),
59            other => Err(Error::invalid(format!("unknown click method {other:?}"))),
60        }
61    }
62}
63
64/// Options for [`Computer::click`].
65#[derive(Debug, Clone, Copy, PartialEq, Eq)]
66pub struct ClickOptions {
67    /// Button to press.
68    pub button: MouseButton,
69    /// Number of clicks (2 for a double click).
70    pub count: u32,
71    /// Strategy.
72    pub method: ClickMethod,
73}
74
75impl Default for ClickOptions {
76    fn default() -> Self {
77        Self { button: MouseButton::Left, count: 1, method: ClickMethod::Auto }
78    }
79}
80
81impl ClickOptions {
82    /// Sets the button.
83    #[must_use]
84    pub const fn button(mut self, button: MouseButton) -> Self {
85        self.button = button;
86        self
87    }
88
89    /// Sets the click count.
90    #[must_use]
91    pub const fn count(mut self, count: u32) -> Self {
92        self.count = count;
93        self
94    }
95
96    /// Sets the method.
97    #[must_use]
98    pub const fn method(mut self, method: ClickMethod) -> Self {
99        self.method = method;
100        self
101    }
102}
103
104/// How a click was ultimately delivered.
105#[derive(Debug, Clone, PartialEq, Eq)]
106#[non_exhaustive]
107pub enum ClickOutcome {
108    /// An accessibility action was invoked.
109    Action(String),
110    /// Pointer events were synthesized.
111    Pointer(Delivery),
112}
113
114struct Session<H> {
115    state: AppState,
116    handles: Vec<H>,
117}
118
119/// A computer-use session over a [`Backend`].
120///
121/// Observe an app with [`get_app_state`](Self::get_app_state), then act on it by element index or by
122/// screenshot pixel. It remembers the latest state per app so indexes stay valid until the
123/// next capture of that app.
124///
125/// ```no_run
126/// # #[cfg(any(target_os = "macos", target_os = "windows", target_os = "linux"))]
127/// # fn main() -> computer_use::Result<()> {
128/// use computer_use::{CaptureOptions, ClickOptions, ClickTarget, Computer};
129///
130/// let mut computer = Computer::new()?;
131/// let state = computer.get_app_state("TextEdit", &CaptureOptions::default())?;
132/// println!("{}", state.render());
133///
134/// computer.click("TextEdit", ClickTarget::Element(3), ClickOptions::default())?;
135/// computer.type_text("TextEdit", "Hello from Rust")?;
136/// computer.press_key("TextEdit", "super+s")?;
137/// # Ok(())
138/// # }
139/// # #[cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))]
140/// # fn main() {}
141/// ```
142pub struct Computer<B: Backend> {
143    backend: B,
144    sessions: HashMap<String, Session<B::Handle>>,
145    aliases: HashMap<String, String>,
146    delivery: Delivery,
147    settle_delay: Duration,
148}
149
150impl<B: Backend> std::fmt::Debug for Computer<B> {
151    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
152        f.debug_struct("Computer")
153            .field("sessions", &self.sessions.keys().collect::<Vec<_>>())
154            .field("delivery", &self.delivery)
155            .field("settle_delay", &self.settle_delay)
156            .finish_non_exhaustive()
157    }
158}
159
160#[cfg(any(target_os = "macos", target_os = "windows", target_os = "linux"))]
161impl Computer<crate::NativeBackend> {
162    /// Creates a session over this platform's native backend.
163    ///
164    /// # Errors
165    ///
166    /// Fails if the platform's accessibility stack cannot be initialized.
167    pub fn new() -> Result<Self> {
168        Ok(Self::with_backend(crate::NativeBackend::new()?))
169    }
170}
171
172impl<B: Backend> Computer<B> {
173    /// Creates a session over a custom backend.
174    #[must_use]
175    pub fn with_backend(backend: B) -> Self {
176        Self {
177            backend,
178            sessions: HashMap::new(),
179            aliases: HashMap::new(),
180            delivery: Delivery::Background,
181            settle_delay: Duration::ZERO,
182        }
183    }
184
185    /// Sets the default input delivery for synthesized events.
186    #[must_use]
187    pub const fn delivery(mut self, delivery: Delivery) -> Self {
188        self.delivery = delivery;
189        self
190    }
191
192    /// Sleeps this long after every action so the UI can settle before the next capture.
193    #[must_use]
194    pub const fn settle_delay(mut self, delay: Duration) -> Self {
195        self.settle_delay = delay;
196        self
197    }
198
199    /// The underlying backend, for platform-specific calls.
200    pub const fn backend(&self) -> &B {
201        &self.backend
202    }
203
204    /// Permissions currently held by this process.
205    pub fn permissions(&self) -> Permissions {
206        self.backend.permissions()
207    }
208
209    /// Prompts for missing permissions where the OS supports it.
210    pub fn request_permissions(&self) -> Permissions {
211        self.backend.request_permissions()
212    }
213
214    /// Lists running apps first (frontmost first), then installed apps, by name.
215    ///
216    /// # Errors
217    ///
218    /// Fails if the platform cannot enumerate apps.
219    pub fn list_apps(&self) -> Result<Vec<AppInfo>> {
220        let mut apps = self.backend.list_apps()?;
221        apps.sort_by(|a, b| {
222            (!a.is_running, !a.is_active, a.name.to_lowercase()).cmp(&(
223                !b.is_running,
224                !b.is_active,
225                b.name.to_lowercase(),
226            ))
227        });
228        Ok(apps)
229    }
230
231    /// Resolves an app query (name, bundle ID, path, or `pid:<n>`), preferring running apps.
232    ///
233    /// # Errors
234    ///
235    /// Returns [`Error::AppNotFound`] when nothing matches.
236    pub fn resolve_app(&self, query: &str) -> Result<AppInfo> {
237        let apps = self.backend.list_apps()?;
238        app::best_match(&apps, query).cloned().ok_or_else(|| Error::AppNotFound(query.to_owned()))
239    }
240
241    /// Captures the app's current window, replacing any earlier state (and element indexes)
242    /// for that app.
243    ///
244    /// # Errors
245    ///
246    /// Fails if the app cannot be found or launched, has no window, or permissions are missing.
247    pub fn get_app_state(&mut self, app: &str, options: &CaptureOptions) -> Result<&AppState> {
248        let info = self.running_app(app, options.launch_if_needed)?;
249        let request =
250            CaptureRequest::new(options.include_tree, options.include_screenshot, options.max_nodes, options.max_depth);
251        let capture = self.backend.capture(&info, &request)?;
252        let (elements, handles, raw_to_element) = build_tree(capture.nodes, options.text_limit);
253        let focused = capture.focused.and_then(|raw| raw_to_element.get(raw).copied().flatten());
254        let screenshot = capture
255            .screenshot
256            .map(|image| Screenshot::encode(image, capture.window.frame, options.max_screenshot_dimension))
257            .transpose()?;
258        let state = AppState {
259            app: info,
260            window: capture.window,
261            elements,
262            truncated: capture.truncated,
263            tree_included: options.include_tree,
264            focused,
265            selected_text: capture.selected_text.map(|s| crate::element::sanitize(&s, options.text_limit)),
266            screenshot,
267            captured_at: SystemTime::now(),
268            notes: capture.notes,
269        };
270        let key = self.remember(app, &state.app);
271        let session = Session { state, handles };
272        Ok(&self.sessions.entry(key).insert_entry(session).into_mut().state)
273    }
274
275    /// The latest captured state of an app, if any.
276    #[must_use]
277    pub fn state(&self, app: &str) -> Option<&AppState> {
278        self.session_for(app).ok().map(|s| &s.state)
279    }
280
281    /// Discards the remembered state of one app.
282    pub fn forget(&mut self, app: &str) {
283        if let Some(key) = self.aliases.remove(&alias_key(app)) {
284            self.sessions.remove(&key);
285            self.aliases.retain(|_, v| *v != key);
286        }
287    }
288
289    /// Discards all remembered state.
290    pub fn clear(&mut self) {
291        self.sessions.clear();
292        self.aliases.clear();
293    }
294
295    /// Finds elements in the app's current window without taking a screenshot.
296    ///
297    /// The fresh tree replaces the app's remembered state, so the returned indexes can be used
298    /// with element actions. A previous screenshot is kept when the window has not moved.
299    ///
300    /// # Errors
301    ///
302    /// Fails like [`get_app_state`](Self::get_app_state) or if the query is empty.
303    pub fn query(&mut self, app: &str, query: &ElementQuery, max_nodes: usize) -> Result<Vec<Element>> {
304        if query.text.as_deref().is_none_or(str::is_empty) && query.role.as_deref().is_none_or(str::is_empty) {
305            return Err(Error::invalid("query requires text and/or role"));
306        }
307        let previous = self.session_for(app).ok().map(|s| (s.state.window.frame, s.state.screenshot.clone()));
308        let options = CaptureOptions::default().include_screenshot(false).max_nodes(max_nodes).launch_if_needed(false);
309        let state = self.get_app_state(app, &options)?;
310        let matches: Vec<Element> = state.find(query).cloned().collect();
311        let frame = state.window.frame;
312        if let Some((prev_frame, Some(shot))) = previous
313            && prev_frame == frame
314            && let Ok(session) = self.session_for_mut(app)
315        {
316            session.state.screenshot = Some(shot);
317        }
318        Ok(matches)
319    }
320
321    /// Clicks an element or a screenshot pixel.
322    ///
323    /// # Errors
324    ///
325    /// Fails if there is no state for the app, the target is invalid, or delivery fails.
326    pub fn click(&mut self, app: &str, target: ClickTarget, options: ClickOptions) -> Result<ClickOutcome> {
327        if options.count == 0 {
328            return Err(Error::invalid("click count must be at least 1"));
329        }
330        let session = self.session_for(app)?;
331        let point = match target {
332            ClickTarget::Element(index) => {
333                let element = element(session, app, index)?;
334                if matches!(options.method, ClickMethod::Auto | ClickMethod::Accessibility)
335                    && let Some(action) = equivalent_action(element, options)
336                {
337                    let name = action.name.clone();
338                    self.backend.perform_action(&session.handles[index], &name)?;
339                    self.settle();
340                    return Ok(ClickOutcome::Action(name));
341                }
342                if options.method == ClickMethod::Accessibility {
343                    return Err(Error::action(format!(
344                        "element {index} has no accessibility action for a {:?} click",
345                        options.button
346                    )));
347                }
348                element
349                    .frame
350                    .filter(|f| !f.is_empty())
351                    .map(|f| f.center())
352                    .ok_or_else(|| Error::StateUnavailable(format!("element {index} has no on-screen frame")))?
353            }
354            ClickTarget::Pixel { x, y } => {
355                if options.method == ClickMethod::Accessibility {
356                    return Err(Error::invalid("accessibility clicks require an element target"));
357                }
358                screenshot_point(&session.state, x, y)?
359            }
360        };
361        let delivery = match options.method {
362            ClickMethod::Global => Delivery::Global,
363            ClickMethod::Background => Delivery::Background,
364            ClickMethod::FocusedBackground => Delivery::FocusedBackground,
365            ClickMethod::Auto | ClickMethod::Accessibility => self.delivery,
366        };
367        let input = input_target(&session.state, delivery);
368        self.backend.click(input, point, options.button, options.count)?;
369        self.settle();
370        Ok(ClickOutcome::Pointer(delivery))
371    }
372
373    /// Invokes a named accessibility action on an element (e.g. `Show Menu`, `AXRaise`).
374    ///
375    /// # Errors
376    ///
377    /// Fails if the element does not expose the action.
378    pub fn perform_secondary_action(&mut self, app: &str, index: usize, action: &str) -> Result<()> {
379        let session = self.session_for(app)?;
380        let element = element(session, app, index)?;
381        let found = element.action(action).ok_or_else(|| {
382            let available: Vec<&str> = element.actions.iter().map(|a| a.label.as_str()).collect();
383            Error::invalid(format!(
384                "{action:?} is not a valid action for element {index}; available: [{}]",
385                available.join(", ")
386            ))
387        })?;
388        self.backend.perform_action(&session.handles[index], &found.name)?;
389        self.settle();
390        Ok(())
391    }
392
393    /// Scrolls an element by a number of pages (fractions allowed).
394    ///
395    /// Whole pages use the element's own scroll-by-page action when it has one; otherwise
396    /// wheel events are posted at the element's center.
397    ///
398    /// # Errors
399    ///
400    /// Fails if `pages` is not positive or the element has no frame.
401    pub fn scroll(&mut self, app: &str, index: usize, direction: ScrollDirection, pages: f64) -> Result<()> {
402        if !pages.is_finite() || pages <= 0.0 {
403            return Err(Error::invalid("pages must be a positive number"));
404        }
405        let session = self.session_for(app)?;
406        let element = element(session, app, index)?;
407        let by_page = format!("Scroll {} By Page", direction.as_str());
408        if pages.fract() == 0.0
409            && pages <= 100.0
410            && let Some(action) = element.action(&by_page)
411        {
412            for _ in 0..whole_pages(pages) {
413                self.backend.perform_action(&session.handles[index], &action.name)?;
414            }
415            self.settle();
416            return Ok(());
417        }
418        let frame = element
419            .frame
420            .filter(|f| !f.is_empty())
421            .ok_or_else(|| Error::StateUnavailable(format!("element {index} has no scrollable frame")))?;
422        let page_x = (frame.size.width * 0.8).max(40.0);
423        let page_y = (frame.size.height * 0.8).max(40.0);
424        let (dx, dy) = match direction {
425            ScrollDirection::Up => (0.0, -page_y * pages),
426            ScrollDirection::Down => (0.0, page_y * pages),
427            ScrollDirection::Left => (-page_x * pages, 0.0),
428            ScrollDirection::Right => (page_x * pages, 0.0),
429        };
430        let input = input_target(&session.state, self.delivery);
431        self.backend.scroll(input, frame.center(), dx, dy)?;
432        self.settle();
433        Ok(())
434    }
435
436    /// Drags between two screenshot pixels.
437    ///
438    /// # Errors
439    ///
440    /// Fails if there is no screenshot or a point lies outside it.
441    pub fn drag(&mut self, app: &str, from: (f64, f64), to: (f64, f64)) -> Result<()> {
442        let session = self.session_for(app)?;
443        let start = screenshot_point(&session.state, from.0, from.1)?;
444        let end = screenshot_point(&session.state, to.0, to.1)?;
445        let input = input_target(&session.state, self.delivery);
446        self.backend.drag(input, start, end)?;
447        self.settle();
448        Ok(())
449    }
450
451    /// Types literal text into the app's focused element.
452    ///
453    /// With background delivery, text is inserted through accessibility when the focused
454    /// element supports it, and synthesized as keystrokes otherwise.
455    ///
456    /// # Errors
457    ///
458    /// Fails if the app is not running or delivery fails.
459    pub fn type_text(&mut self, app: &str, text: &str) -> Result<()> {
460        if text.is_empty() {
461            return Ok(());
462        }
463        let (info, window) = self.input_context(app)?;
464        if self.delivery != Delivery::Global && self.backend.insert_text(&info, text)? {
465            self.settle();
466            return Ok(());
467        }
468        let target = InputTarget { app: &info, window: window.as_ref(), delivery: self.delivery };
469        self.backend.type_text(target, text)?;
470        self.settle();
471        Ok(())
472    }
473
474    /// Presses a key chord in xdotool syntax, e.g. `Return`, `super+c`, `KP_0`.
475    ///
476    /// # Errors
477    ///
478    /// Fails if the chord cannot be parsed or delivered.
479    pub fn press_key(&mut self, app: &str, key: &str) -> Result<()> {
480        let chord: KeyChord = key.parse()?;
481        let (info, window) = self.input_context(app)?;
482        let target = InputTarget { app: &info, window: window.as_ref(), delivery: self.delivery };
483        self.backend.press_key(target, &chord)?;
484        self.settle();
485        Ok(())
486    }
487
488    /// Sets the value of a settable element.
489    ///
490    /// # Errors
491    ///
492    /// Fails if the element's value is read-only.
493    pub fn set_value(&mut self, app: &str, index: usize, value: &str) -> Result<()> {
494        let session = self.session_for(app)?;
495        element(session, app, index)?;
496        self.backend.set_value(&session.handles[index], value)?;
497        self.settle();
498        Ok(())
499    }
500
501    /// Selects text inside a text element, or places the caret before or after it.
502    ///
503    /// # Errors
504    ///
505    /// Fails if the text is missing or ambiguous, or the element does not support selection.
506    pub fn select_text(&mut self, app: &str, index: usize, selection: &TextSelection) -> Result<()> {
507        let session = self.session_for(app)?;
508        let cached = element(session, app, index)?;
509        let handle = &session.handles[index];
510        let value = match self.backend.value(handle) {
511            Ok(Some(value)) => value,
512            Ok(None) => return Err(Error::StateUnavailable(format!("element {index} has no text value"))),
513            Err(Error::Unsupported(_)) => cached
514                .value
515                .clone()
516                .ok_or_else(|| Error::StateUnavailable(format!("element {index} has no text value")))?,
517            Err(e) => return Err(e),
518        };
519        let range = selection.resolve(&value)?;
520        self.backend.select_text(handle, &value, range)?;
521        self.settle();
522        Ok(())
523    }
524
525    fn running_app(&self, query: &str, launch: bool) -> Result<AppInfo> {
526        let info = self.resolve_app(query)?;
527        if info.is_running {
528            Ok(info)
529        } else if launch {
530            self.backend.launch_app(&info)
531        } else {
532            Err(Error::AppNotFound(format!("{query} is not running")))
533        }
534    }
535
536    fn input_context(&self, app: &str) -> Result<(AppInfo, Option<crate::state::WindowInfo>)> {
537        if let Ok(session) = self.session_for(app) {
538            return Ok((session.state.app.clone(), Some(session.state.window.clone())));
539        }
540        Ok((self.running_app(app, false)?, None))
541    }
542
543    fn remember(&mut self, query: &str, info: &AppInfo) -> String {
544        let key = info.pid.map_or_else(|| info.reference().to_lowercase(), |pid| format!("pid:{pid}"));
545        self.aliases.insert(alias_key(query), key.clone());
546        self.aliases.insert(alias_key(info.reference()), key.clone());
547        self.aliases.insert(alias_key(&info.name), key.clone());
548        if let Some(pid) = info.pid {
549            self.aliases.insert(format!("pid:{pid}"), key.clone());
550        }
551        key
552    }
553
554    fn session_for(&self, app: &str) -> Result<&Session<B::Handle>> {
555        self.aliases
556            .get(&alias_key(app))
557            .and_then(|key| self.sessions.get(key))
558            .ok_or_else(|| Error::NoAppState(app.to_owned()))
559    }
560
561    fn session_for_mut(&mut self, app: &str) -> Result<&mut Session<B::Handle>> {
562        self.aliases
563            .get(&alias_key(app))
564            .and_then(|key| self.sessions.get_mut(key))
565            .ok_or_else(|| Error::NoAppState(app.to_owned()))
566    }
567
568    fn settle(&self) {
569        if !self.settle_delay.is_zero() {
570            thread::sleep(self.settle_delay);
571        }
572    }
573}
574
575fn alias_key(query: &str) -> String {
576    query.trim().to_lowercase()
577}
578
579fn element<'a, H>(session: &'a Session<H>, app: &str, index: usize) -> Result<&'a Element> {
580    session.state.element(index).ok_or_else(|| Error::ElementNotFound { app: app.to_owned(), index })
581}
582
583fn equivalent_action(element: &Element, options: ClickOptions) -> Option<&crate::element::Action> {
584    match (options.button, options.count) {
585        (MouseButton::Left, 1) => element.primary_action(),
586        (MouseButton::Right, 1) => element.action("Show Menu"),
587        _ => None,
588    }
589}
590
591fn screenshot_point(state: &AppState, x: f64, y: f64) -> Result<Point> {
592    let shot = state.screenshot.as_ref().ok_or_else(|| {
593        Error::StateUnavailable("no screenshot in the latest state; capture one before coordinate actions".into())
594    })?;
595    if !shot.contains_pixel(x, y) {
596        return Err(Error::invalid(format!("({x}, {y}) is outside the {}x{} screenshot", shot.width(), shot.height())));
597    }
598    Ok(shot.to_screen_point(x, y))
599}
600
601const fn input_target(state: &AppState, delivery: Delivery) -> InputTarget<'_> {
602    InputTarget { app: &state.app, window: Some(&state.window), delivery }
603}
604
605#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
606const fn whole_pages(pages: f64) -> u32 {
607    pages as u32
608}
computer_use/computer.rs

computer_use/
computer.rs
