punch_runtime/automation/
common.rs

1//! Common types for the desktop automation system.
2//!
3//! These types are used by the `AutomationBackend` trait and all platform
4//! implementations (macOS, Linux, Windows).
5
6use serde::{Deserialize, Serialize};
7
8/// Information about an on-screen window.
9#[derive(Debug, Clone, Serialize, Deserialize)]
10pub struct WindowInfo {
11    /// Window title (may be empty for unnamed windows).
12    pub title: String,
13    /// Name of the application owning this window.
14    pub app_name: String,
15    /// Screen position (x, y) of the top-left corner.
16    pub position: Option<(i32, i32)>,
17    /// Window size (width, height) in pixels.
18    pub size: Option<(u32, u32)>,
19    /// Whether the window is currently minimized.
20    pub is_minimized: bool,
21}
22
23/// A UI element discovered via the accessibility tree.
24#[derive(Debug, Clone, Serialize, Deserialize)]
25pub struct UiElement {
26    /// Stable-ish identifier: "AppName:role:index" (e.g. "Messages:button:3").
27    /// Generated by `find_ui_elements`, consumed by `click_element` etc.
28    pub element_id: String,
29    /// Accessibility role: "button", "text_field", "menu_item", "row", etc.
30    pub role: String,
31    /// Accessibility label (human-readable name), if available.
32    pub label: Option<String>,
33    /// Current value of the element, if applicable.
34    pub value: Option<String>,
35    /// Whether the element is enabled for interaction.
36    pub enabled: bool,
37}
38
39/// Selector used to filter UI elements when querying the accessibility tree.
40#[derive(Debug, Clone, Default, Serialize, Deserialize)]
41pub struct UiSelector {
42    /// Filter by accessibility role (e.g. "button", "text field").
43    pub role: Option<String>,
44    /// Filter by accessibility label (substring match).
45    pub label: Option<String>,
46    /// Filter by current value.
47    pub value: Option<String>,
48}
49
50/// Result of a screenshot capture.
51#[derive(Debug, Clone, Serialize, Deserialize)]
52pub struct ScreenshotResult {
53    /// Base64-encoded PNG image data.
54    pub png_base64: String,
55    /// Image width in pixels.
56    pub width: u32,
57    /// Image height in pixels.
58    pub height: u32,
59}
60
61/// Result of an OCR (optical character recognition) operation.
62#[derive(Debug, Clone, Serialize, Deserialize)]
63pub struct OcrResult {
64    /// Full extracted text.
65    pub text: String,
66    /// Individual text regions with positions and confidence.
67    pub regions: Vec<OcrRegion>,
68}
69
70/// A single text region from OCR.
71#[derive(Debug, Clone, Serialize, Deserialize)]
72pub struct OcrRegion {
73    /// The text recognized in this region.
74    pub text: String,
75    /// Bounding box: (x, y, width, height). None if position unavailable.
76    pub bounds: Option<(i32, i32, u32, u32)>,
77    /// Confidence score (0.0–1.0).
78    pub confidence: f32,
79}
80
81#[cfg(test)]
82mod tests {
83    use super::*;
84
85    #[test]
86    fn test_window_info_serde_roundtrip() {
87        let info = WindowInfo {
88            title: "My Window".to_string(),
89            app_name: "Safari".to_string(),
90            position: Some((100, 200)),
91            size: Some((1920, 1080)),
92            is_minimized: false,
93        };
94        let json = serde_json::to_string(&info).expect("serialize");
95        let deser: WindowInfo = serde_json::from_str(&json).expect("deserialize");
96        assert_eq!(deser.title, "My Window");
97        assert_eq!(deser.size, Some((1920, 1080)));
98    }
99
100    #[test]
101    fn test_window_info_none_optionals() {
102        let info = WindowInfo {
103            title: String::new(),
104            app_name: "Finder".to_string(),
105            position: None,
106            size: None,
107            is_minimized: true,
108        };
109        let json = serde_json::to_string(&info).expect("serialize");
110        let deser: WindowInfo = serde_json::from_str(&json).expect("deserialize");
111        assert!(deser.position.is_none());
112        assert!(deser.is_minimized);
113    }
114
115    #[test]
116    fn test_ui_element_serde_roundtrip() {
117        let elem = UiElement {
118            element_id: "Safari:button:3".to_string(),
119            role: "button".to_string(),
120            label: Some("Close".to_string()),
121            value: None,
122            enabled: true,
123        };
124        let json = serde_json::to_string(&elem).expect("serialize");
125        let deser: UiElement = serde_json::from_str(&json).expect("deserialize");
126        assert_eq!(deser.element_id, "Safari:button:3");
127        assert_eq!(deser.role, "button");
128        assert!(deser.enabled);
129    }
130
131    #[test]
132    fn test_ui_selector_defaults() {
133        let sel = UiSelector::default();
134        assert!(sel.role.is_none());
135        assert!(sel.label.is_none());
136        assert!(sel.value.is_none());
137    }
138
139    #[test]
140    fn test_screenshot_result_serde() {
141        let result = ScreenshotResult {
142            png_base64: "iVBORw0KGgo=".to_string(),
143            width: 1920,
144            height: 1080,
145        };
146        let json = serde_json::to_string(&result).expect("serialize");
147        let deser: ScreenshotResult = serde_json::from_str(&json).expect("deserialize");
148        assert_eq!(deser.width, 1920);
149        assert_eq!(deser.height, 1080);
150    }
151
152    #[test]
153    fn test_ocr_result_serde() {
154        let result = OcrResult {
155            text: "Hello World".to_string(),
156            regions: vec![OcrRegion {
157                text: "Hello".to_string(),
158                bounds: Some((10, 20, 50, 15)),
159                confidence: 0.95,
160            }],
161        };
162        let json = serde_json::to_string(&result).expect("serialize");
163        let deser: OcrResult = serde_json::from_str(&json).expect("deserialize");
164        assert_eq!(deser.regions.len(), 1);
165        assert!((deser.regions[0].confidence - 0.95).abs() < f32::EPSILON);
166    }
167
168    #[test]
169    fn test_ocr_empty_regions() {
170        let result = OcrResult {
171            text: String::new(),
172            regions: Vec::new(),
173        };
174        let json = serde_json::to_string(&result).expect("serialize");
175        let deser: OcrResult = serde_json::from_str(&json).expect("deserialize");
176        assert!(deser.text.is_empty());
177        assert!(deser.regions.is_empty());
178    }
179
180    #[test]
181    fn test_ocr_region_no_bounds() {
182        let region = OcrRegion {
183            text: "test".to_string(),
184            bounds: None,
185            confidence: 0.5,
186        };
187        let json = serde_json::to_string(&region).expect("serialize");
188        let deser: OcrRegion = serde_json::from_str(&json).expect("deserialize");
189        assert!(deser.bounds.is_none());
190    }
191}
punch_runtime/automation/common.rs

punch_runtime/automation/
common.rs
