computer-use-rs 0.3.1

Cross-platform computer-use primitives: read app accessibility trees, take window screenshots, and drive apps with accessibility actions and synthetic input.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121

use std::fmt;

use crate::app::AppInfo;
use crate::state::WindowInfo;

/// Convenience alias used throughout the crate.
pub type Result<T, E = Error> = std::result::Result<T, E>;

/// An OS permission that computer use depends on.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[non_exhaustive]
pub enum Permission {
    /// Reading and driving other apps through the accessibility API.
    Accessibility,
    /// Capturing pixels of other apps' windows.
    ScreenRecording,
}

impl fmt::Display for Permission {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.write_str(match self {
            Self::Accessibility => "accessibility",
            Self::ScreenRecording => "screen recording",
        })
    }
}

/// Everything that can go wrong while observing or driving an app.
#[derive(Debug, thiserror::Error)]
#[non_exhaustive]
pub enum Error {
    /// No running or installed app matched the query.
    #[error("app not found: {0}")]
    AppNotFound(String),

    /// Several running apps matched the query equally well.
    #[error("{query:?} matches more than one running app: {}", list(candidates, |a| format!("{} (pid {})", a.name, a.pid.unwrap_or_default())))]
    AmbiguousApp {
        /// The query.
        query: String,
        /// The matching apps.
        candidates: Vec<AppInfo>,
    },

    /// Several of an app's windows matched a title.
    #[error("{title:?} matches more than one window: {}", list(candidates, |w| format!("{:?} (id {})", w.title.as_deref().unwrap_or_default(), w.id.unwrap_or_default())))]
    AmbiguousWindow {
        /// The title.
        title: String,
        /// The matching windows.
        candidates: Vec<WindowInfo>,
    },

    /// The app is running but has no window that can be observed.
    #[error("no usable window for app {0}")]
    WindowNotFound(String),

    /// An action referenced app state that was never captured (or was discarded).
    #[error("no app state for {0}; call get_app_state first")]
    NoAppState(String),

    /// An element index did not exist in the latest captured state.
    #[error("element {index} not found in the latest state of {app}")]
    ElementNotFound {
        /// App the lookup was made against.
        app: String,
        /// The requested element index.
        index: usize,
    },

    /// A caller-supplied argument was invalid.
    #[error("invalid argument: {0}")]
    InvalidArgument(String),

    /// A required OS permission has not been granted to this process.
    #[error("{0} permission has not been granted to this process")]
    PermissionDenied(Permission),

    /// The operation is not available on this platform or backend.
    #[error("unsupported: {0}")]
    Unsupported(String),

    /// The platform accepted the request but reported that it failed.
    #[error("action failed: {0}")]
    ActionFailed(String),

    /// The state the action needs is missing (for example, no screenshot was captured).
    #[error("state unavailable: {0}")]
    StateUnavailable(String),

    /// A platform API returned an unexpected error.
    #[error("platform error: {0}")]
    Platform(String),

    /// Encoding a screenshot failed.
    #[error("image encoding failed: {0}")]
    Image(#[from] image::ImageError),

    /// An I/O error, for example while launching an app.
    #[error(transparent)]
    Io(#[from] std::io::Error),
}

impl Error {
    pub(crate) fn invalid(message: impl Into<String>) -> Self {
        Self::InvalidArgument(message.into())
    }

    #[cfg_attr(not(target_os = "macos"), allow(dead_code))]
    pub(crate) fn platform(message: impl Into<String>) -> Self {
        Self::Platform(message.into())
    }

    pub(crate) fn action(message: impl Into<String>) -> Self {
        Self::ActionFailed(message.into())
    }
}

fn list<T>(items: &[T], describe: impl Fn(&T) -> String) -> String {
    items.iter().map(describe).collect::<Vec<_>>().join(", ")
}