droidrun_core/ui/
state.rs

1/// UIState — parsed UI elements with element resolution and coordinate conversion.
2use serde::{Deserialize, Serialize};
3
4use crate::error::{DroidrunError, Result};
5use crate::ui::coord;
6use crate::ui::geometry::{find_clear_point, Bounds};
7
8/// A UI element from the accessibility tree.
9#[derive(Debug, Clone, Serialize, Deserialize)]
10pub struct Element {
11    pub index: usize,
12    #[serde(default)]
13    pub class_name: String,
14    #[serde(default)]
15    pub resource_id: String,
16    #[serde(default)]
17    pub text: String,
18    #[serde(default)]
19    pub bounds: String,
20    #[serde(default)]
21    pub checked_state: String,
22    #[serde(default)]
23    pub children: Vec<Element>,
24}
25
26/// Phone state information.
27#[derive(Debug, Clone, Serialize, Deserialize, Default)]
28pub struct PhoneState {
29    #[serde(default, rename = "currentApp")]
30    pub current_app: String,
31    #[serde(default, rename = "packageName")]
32    pub package_name: String,
33    #[serde(default, rename = "isEditable")]
34    pub is_editable: bool,
35    #[serde(default, rename = "focusedElement")]
36    pub focused_element: Option<serde_json::Value>,
37}
38
39/// Screen dimensions.
40#[derive(Debug, Clone, Copy)]
41pub struct ScreenDimensions {
42    pub width: i32,
43    pub height: i32,
44}
45
46/// A snapshot of the device UI state.
47#[derive(Debug, Clone)]
48pub struct UIState {
49    pub elements: Vec<Element>,
50    pub formatted_text: String,
51    pub focused_text: String,
52    pub phone_state: PhoneState,
53    pub screen: ScreenDimensions,
54    pub use_normalized: bool,
55}
56
57impl UIState {
58    pub fn new(
59        elements: Vec<Element>,
60        formatted_text: String,
61        focused_text: String,
62        phone_state: PhoneState,
63        screen: ScreenDimensions,
64        use_normalized: bool,
65    ) -> Self {
66        Self {
67            elements,
68            formatted_text,
69            focused_text,
70            phone_state,
71            screen,
72            use_normalized,
73        }
74    }
75
76    /// Find an element by index (recursive tree search).
77    pub fn get_element(&self, index: usize) -> Option<&Element> {
78        find_by_index(&self.elements, index)
79    }
80
81    /// Get the center (x, y) of an element by index.
82    pub fn get_element_coords(&self, index: usize) -> Result<(i32, i32)> {
83        let el = self
84            .get_element(index)
85            .ok_or(DroidrunError::ElementNotFound(index))?;
86
87        if el.bounds.is_empty() {
88            return Err(DroidrunError::ElementNoBounds(index));
89        }
90
91        let bounds = Bounds::from_str(&el.bounds)
92            .ok_or_else(|| DroidrunError::InvalidBounds(el.bounds.clone()))?;
93
94        Ok(bounds.center())
95    }
96
97    /// Get element info for display.
98    pub fn get_element_info(&self, index: usize) -> Option<ElementInfo> {
99        let el = self.get_element(index)?;
100        Some(ElementInfo {
101            text: el.text.clone(),
102            class_name: el.class_name.clone(),
103            bounds: el.bounds.clone(),
104        })
105    }
106
107    /// Find a tap point that avoids overlapping elements.
108    pub fn get_clear_point(&self, index: usize) -> Result<(i32, i32)> {
109        let el = self
110            .get_element(index)
111            .ok_or(DroidrunError::ElementNotFound(index))?;
112
113        if el.bounds.is_empty() {
114            return Err(DroidrunError::ElementNoBounds(index));
115        }
116
117        let target = Bounds::from_str(&el.bounds)
118            .ok_or_else(|| DroidrunError::InvalidBounds(el.bounds.clone()))?;
119
120        let all_elements = collect_all(&self.elements);
121        let blockers: Vec<Bounds> = all_elements
122            .iter()
123            .filter(|e| e.index > index && !e.bounds.is_empty())
124            .filter_map(|e| Bounds::from_str(&e.bounds))
125            .filter(|b| target.overlaps(b))
126            .collect();
127
128        find_clear_point(&target, &blockers).ok_or(DroidrunError::ElementObscured(index))
129    }
130
131    /// Convert point to absolute pixels if normalized mode is active.
132    pub fn convert_point(&self, x: i32, y: i32) -> Result<(i32, i32)> {
133        if self.use_normalized {
134            coord::to_absolute(x, y, self.screen.width, self.screen.height)
135        } else {
136            Ok((x, y))
137        }
138    }
139
140    /// Get all element indices (flattened).
141    pub fn all_indices(&self) -> Vec<usize> {
142        collect_indices(&self.elements)
143    }
144}
145
146/// Basic element info for display.
147#[derive(Debug, Clone)]
148pub struct ElementInfo {
149    pub text: String,
150    pub class_name: String,
151    pub bounds: String,
152}
153
154// ── Internal helpers ─────────────────────────────────────────────
155
156fn find_by_index(elements: &[Element], target: usize) -> Option<&Element> {
157    for el in elements {
158        if el.index == target {
159            return Some(el);
160        }
161        if let Some(found) = find_by_index(&el.children, target) {
162            return Some(found);
163        }
164    }
165    None
166}
167
168fn collect_indices(elements: &[Element]) -> Vec<usize> {
169    let mut indices = Vec::new();
170    for el in elements {
171        indices.push(el.index);
172        indices.extend(collect_indices(&el.children));
173    }
174    indices.sort();
175    indices
176}
177
178fn collect_all(elements: &[Element]) -> Vec<&Element> {
179    let mut result = Vec::new();
180    for el in elements {
181        result.push(el);
182        result.extend(collect_all(&el.children));
183    }
184    result
185}
186
187#[cfg(test)]
188mod tests {
189    use super::*;
190
191    fn sample_elements() -> Vec<Element> {
192        vec![
193            Element {
194                index: 1,
195                class_name: "Button".into(),
196                resource_id: "btn_ok".into(),
197                text: "OK".into(),
198                bounds: "100,200,300,400".into(),
199                checked_state: String::new(),
200                children: vec![],
201            },
202            Element {
203                index: 2,
204                class_name: "TextView".into(),
205                resource_id: "".into(),
206                text: "Hello".into(),
207                bounds: "0,0,1080,100".into(),
208                checked_state: String::new(),
209                children: vec![Element {
210                    index: 3,
211                    class_name: "ImageView".into(),
212                    resource_id: "icon".into(),
213                    text: "".into(),
214                    bounds: "10,10,50,50".into(),
215                    checked_state: String::new(),
216                    children: vec![],
217                }],
218            },
219        ]
220    }
221
222    fn sample_state() -> UIState {
223        UIState::new(
224            sample_elements(),
225            "formatted".into(),
226            "focused".into(),
227            PhoneState::default(),
228            ScreenDimensions {
229                width: 1080,
230                height: 2400,
231            },
232            false,
233        )
234    }
235
236    #[test]
237    fn test_get_element() {
238        let state = sample_state();
239        assert!(state.get_element(1).is_some());
240        assert_eq!(state.get_element(1).unwrap().text, "OK");
241    }
242
243    #[test]
244    fn test_get_element_nested() {
245        let state = sample_state();
246        let el = state.get_element(3).unwrap();
247        assert_eq!(el.class_name, "ImageView");
248    }
249
250    #[test]
251    fn test_get_element_not_found() {
252        let state = sample_state();
253        assert!(state.get_element(999).is_none());
254    }
255
256    #[test]
257    fn test_get_element_coords() {
258        let state = sample_state();
259        let (x, y) = state.get_element_coords(1).unwrap();
260        assert_eq!((x, y), (200, 300)); // center of 100,200,300,400
261    }
262
263    #[test]
264    fn test_all_indices() {
265        let state = sample_state();
266        assert_eq!(state.all_indices(), vec![1, 2, 3]);
267    }
268
269    #[test]
270    fn test_convert_point_absolute() {
271        let state = sample_state();
272        let (x, y) = state.convert_point(540, 1200).unwrap();
273        assert_eq!((x, y), (540, 1200));
274    }
275
276    #[test]
277    fn test_convert_point_normalized() {
278        let mut state = sample_state();
279        state.use_normalized = true;
280        let (x, y) = state.convert_point(500, 500).unwrap();
281        assert_eq!((x, y), (540, 1200));
282    }
283}
droidrun_core/ui/state.rs

droidrun_core/ui/
state.rs
