computer_use/
state.rs

1//! Best-effort desktop snapshot. Subsystem failures become null fields and a note.
2
3use std::panic::{catch_unwind, AssertUnwindSafe};
4
5use serde_json::{json, Value};
6
7use crate::backend::DesktopBackend;
8use crate::error::ComputerError;
9use crate::tree::{TreeQuery, TreeReport};
10use crate::types::{DisplayInfo, FrontmostInfo};
11
12/// Collect screen, cursor, frontmost window, and accessibility information.
13///
14/// This function does not panic. A backend panic is caught and reported in `note`.
15pub fn collect_state(backend: &dyn DesktopBackend) -> Value {
16    collect_state_for(backend, &TreeQuery::default())
17}
18
19/// Same snapshot as [`collect_state`], with the tree query from the request.
20pub fn collect_state_for(backend: &dyn DesktopBackend, query: &TreeQuery) -> Value {
21    match catch_unwind(AssertUnwindSafe(|| collect_state_inner(backend, query))) {
22        Ok(value) => value,
23        Err(_) => recovered_state(),
24    }
25}
26
27fn recovered_state() -> Value {
28    json!({
29        "platform": platform_name(),
30        "displays": [],
31        "screen": null,
32        "cursor": null,
33        "frontmost_app": null,
34        "frontmost_window": null,
35        "tree": null,
36        "revision": null,
37        "window_id": null,
38        "pid": null,
39        "source": null,
40        "truncated": null,
41        "accessibility_legacy": null,
42        "note": "get-state panicked internally and was recovered"
43    })
44}
45
46fn collect_state_inner(backend: &dyn DesktopBackend, query: &TreeQuery) -> Value {
47    let mut notes = Vec::new();
48
49    let displays = match guard("displays", || backend.list_displays()) {
50        Ok(displays) => displays,
51        Err(note) => {
52            notes.push(note);
53            Vec::new()
54        }
55    };
56    let screen = screen_value(primary_display(&displays));
57
58    let cursor = match guard("cursor", || backend.cursor_position()) {
59        Ok((x, y)) => json!({"x": x, "y": y}),
60        Err(note) => {
61            notes.push(note);
62            Value::Null
63        }
64    };
65
66    let front = match catch_unwind(AssertUnwindSafe(|| backend.frontmost())) {
67        Ok(info) => info,
68        Err(_) => FrontmostInfo {
69            note: Some("frontmost lookup panicked and was recovered".into()),
70            ..FrontmostInfo::default()
71        },
72    };
73    if let Some(note) = front.note {
74        notes.push(note);
75    }
76
77    let tree = match catch_unwind(AssertUnwindSafe(|| backend.tree(query))) {
78        Ok(Ok(report)) => tree_value(&report, &mut notes),
79        Ok(Err(error)) => {
80            notes.push(error.to_string());
81            empty_tree()
82        }
83        Err(_) => {
84            notes.push("accessibility inspection panicked and was recovered".into());
85            empty_tree()
86        }
87    };
88
89    let mut body = json!({
90        "platform": platform_name(),
91        "displays": displays,
92        "screen": screen,
93        "cursor": cursor,
94        "frontmost_app": front.app,
95        "frontmost_window": front.window,
96        "note": if notes.is_empty() { Value::Null } else { Value::String(notes.join("; ")) }
97    });
98    if let (Some(body), Value::Object(fields)) = (body.as_object_mut(), tree) {
99        for (key, value) in fields {
100            body.insert(key, value);
101        }
102    }
103    body
104}
105
106fn empty_tree() -> Value {
107    json!({
108        "tree": null,
109        "revision": null,
110        "window_id": null,
111        "pid": null,
112        "source": null,
113        "truncated": null,
114        "accessibility_legacy": null,
115    })
116}
117
118fn tree_value(report: &TreeReport, notes: &mut Vec<String>) -> Value {
119    if let Some(note) = report.note.as_deref() {
120        notes.push(note.to_string());
121    }
122    let legacy = match report.legacy.as_deref() {
123        Some(text) => serde_json::from_str::<Value>(text.trim()).unwrap_or(Value::Null),
124        None => Value::Null,
125    };
126    json!({
127        "tree": report.tree,
128        "revision": report.revision,
129        "window_id": report.window_id,
130        "pid": report.pid,
131        "source": report.source,
132        "truncated": report.truncated,
133        "accessibility_legacy": legacy,
134    })
135}
136
137fn primary_display(displays: &[DisplayInfo]) -> Option<&DisplayInfo> {
138    if let Some(display) = displays.iter().find(|display| display.is_primary) {
139        return Some(display);
140    }
141    displays.first()
142}
143
144fn screen_value(display: Option<&DisplayInfo>) -> Value {
145    match display {
146        Some(display) => json!({
147            "id": display.id,
148            "name": display.name,
149            "x": display.x,
150            "y": display.y,
151            "width": display.width,
152            "height": display.height,
153            "scale": display.scale,
154        }),
155        None => Value::Null,
156    }
157}
158
159fn guard<T>(label: &str, function: impl FnOnce() -> Result<T, ComputerError>) -> Result<T, String> {
160    match catch_unwind(AssertUnwindSafe(function)) {
161        Ok(Ok(value)) => Ok(value),
162        Ok(Err(error)) => Err(format!("{label}: {error}")),
163        Err(_) => Err(format!("{label} panicked and was recovered")),
164    }
165}
166
167fn platform_name() -> &'static str {
168    #[cfg(target_os = "macos")]
169    {
170        "macos"
171    }
172    #[cfg(target_os = "linux")]
173    {
174        "linux"
175    }
176    #[cfg(target_os = "windows")]
177    {
178        "windows"
179    }
180}
181
182#[cfg(test)]
183mod tests {
184    use super::*;
185    use crate::test_backend::StubBackend;
186
187    #[test]
188    fn state_shape_from_a_stub_backend() {
189        let backend = StubBackend::default();
190        let value = collect_state(&backend);
191        assert_eq!(value["displays"][0]["name"], "Stub");
192        assert_eq!(value["screen"]["width"], 800);
193        assert_eq!(value["screen"]["scale"], 2.0);
194        assert_eq!(value["cursor"]["x"], 4);
195        assert_eq!(value["cursor"]["y"], 5);
196        assert_eq!(value["frontmost_app"], "Finder");
197        assert_eq!(value["frontmost_window"], "Desktop");
198        assert_eq!(value["source"], "stub");
199        assert!(value["tree"].as_str().unwrap().contains("AXWindow"));
200        assert!(value["accessibility_legacy"].is_null());
201        assert!(value["note"].is_null());
202        assert!(value["platform"].is_string());
203    }
204
205    #[test]
206    fn accessibility_failure_is_null_with_a_note() {
207        let backend = StubBackend {
208            fail_accessibility: true,
209            ..StubBackend::default()
210        };
211        let value = collect_state(&backend);
212        assert!(value["tree"].is_null());
213        assert!(value["accessibility_legacy"].is_null());
214        assert!(value["note"].as_str().unwrap().contains("accessibility"));
215        assert_eq!(value["frontmost_app"], "Finder");
216    }
217}
computer_use/state.rs

computer_use/
state.rs
